From 3e6cd3b4b1ca1a9be731715fa0cad1a5357b86e0 Mon Sep 17 00:00:00 2001 From: Ryan Kistner Date: Sun, 19 Apr 2020 13:25:43 -0600 Subject: [PATCH] Limit unnecessary concurrent CDN token requests. Combine CDN token requests for steamcontent.com. Added a diagnostic timeout for lost CDN token requests. --- DepotDownloader/CDNClientPool.cs | 25 +++++++++---------------- DepotDownloader/ContentDownloader.cs | 4 ++-- DepotDownloader/Steam3Session.cs | 22 ++++++++++++++-------- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/DepotDownloader/CDNClientPool.cs b/DepotDownloader/CDNClientPool.cs index 6dc1c3e0..1f174972 100644 --- a/DepotDownloader/CDNClientPool.cs +++ b/DepotDownloader/CDNClientPool.cs @@ -20,8 +20,6 @@ namespace DepotDownloader public CDNClient CDNClient { get; } - private readonly ConcurrentDictionary, string> depotCdnKeys; - private readonly ConcurrentBag activeConnectionPool; private readonly BlockingCollection availableServerEndpoints; @@ -34,7 +32,6 @@ namespace DepotDownloader { this.steamSession = steamSession; CDNClient = new CDNClient(steamSession.steamClient); - depotCdnKeys = new ConcurrentDictionary, string>(); activeConnectionPool = new ConcurrentBag(); availableServerEndpoints = new BlockingCollection(); @@ -125,15 +122,17 @@ namespace DepotDownloader } } - private string AuthenticateConnection(uint appId, uint depotId, CDNClient.Server server) + private async Task AuthenticateConnection(uint appId, uint depotId, CDNClient.Server server) { - steamSession.RequestCDNAuthToken(appId, depotId, server.Host); + var host = steamSession.ResolveCDNTopLevelHost(server.Host); + var cdnKey = $"{depotId:D}:{host}"; - var cdnKey = $"{depotId:D}:{steamSession.ResolveCDNTopLevelHost(server.Host)}"; + steamSession.RequestCDNAuthToken(appId, depotId, host, cdnKey); - if (steamSession.CDNAuthTokens.TryGetValue(cdnKey, out var authTokenCallback)) + if (steamSession.CDNAuthTokens.TryGetValue(cdnKey, out var authTokenCallbackPromise)) { - return authTokenCallback.Token; + var result = await authTokenCallbackPromise.Task; + return result.Token; } else { @@ -151,7 +150,7 @@ namespace DepotDownloader return availableServerEndpoints.Take(token); } - public Tuple GetConnectionForDepot(uint appId, uint depotId, CancellationToken token) + public async Task> GetConnectionForDepot(uint appId, uint depotId, CancellationToken token) { // Take a free connection from the connection pool // If there were no free connections, create a new one from the server list @@ -160,14 +159,8 @@ namespace DepotDownloader server = BuildConnection(token); } - var topLevelHost = steamSession.ResolveCDNTopLevelHost(server.Host); - var depotKey = Tuple.Create(depotId, topLevelHost); - // If we don't have a CDN token yet for this server and depot, fetch one now - if (!depotCdnKeys.TryGetValue(depotKey, out var cdnToken)) - { - depotCdnKeys[depotKey] = cdnToken = AuthenticateConnection(appId, depotId, server); - } + var cdnToken = await AuthenticateConnection(appId, depotId, server); return Tuple.Create(server, cdnToken); } diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader/ContentDownloader.cs index a09877d3..1d8559d3 100644 --- a/DepotDownloader/ContentDownloader.cs +++ b/DepotDownloader/ContentDownloader.cs @@ -665,7 +665,7 @@ namespace DepotDownloader Tuple connection = null; try { - connection = cdnPool.GetConnectionForDepot( appId, depot.id, CancellationToken.None ); + connection = await cdnPool.GetConnectionForDepot( appId, depot.id, CancellationToken.None ); depotManifest = await cdnPool.CDNClient.DownloadManifestAsync( depot.id, depot.manifestId, connection.Item1, connection.Item2, depot.depotKey ).ConfigureAwait(false); @@ -890,7 +890,7 @@ namespace DepotDownloader Tuple connection; try { - connection = cdnPool.GetConnectionForDepot( appId, depot.id, cts.Token ); + connection = await cdnPool.GetConnectionForDepot( appId, depot.id, cts.Token ); } catch ( OperationCanceledException ) { diff --git a/DepotDownloader/Steam3Session.cs b/DepotDownloader/Steam3Session.cs index 2ab394a8..e6d5313b 100644 --- a/DepotDownloader/Steam3Session.cs +++ b/DepotDownloader/Steam3Session.cs @@ -7,6 +7,7 @@ using System.Collections.ObjectModel; using System.IO; using System.Linq; using System.Threading; +using System.Threading.Tasks; namespace DepotDownloader { @@ -33,7 +34,7 @@ namespace DepotDownloader public Dictionary AppTickets { get; private set; } public Dictionary AppTokens { get; private set; } public Dictionary DepotKeys { get; private set; } - public ConcurrentDictionary CDNAuthTokens { get; private set; } + public ConcurrentDictionary> CDNAuthTokens { get; private set; } public Dictionary AppInfo { get; private set; } public Dictionary PackageInfo { get; private set; } public Dictionary AppBetaPasswords { get; private set; } @@ -82,7 +83,7 @@ namespace DepotDownloader this.AppTickets = new Dictionary(); this.AppTokens = new Dictionary(); this.DepotKeys = new Dictionary(); - this.CDNAuthTokens = new ConcurrentDictionary(); + this.CDNAuthTokens = new ConcurrentDictionary>(); this.AppInfo = new Dictionary(); this.PackageInfo = new Dictionary(); this.AppBetaPasswords = new Dictionary(); @@ -328,19 +329,24 @@ namespace DepotDownloader { return "steampipe.steamcontent.com"; } + else if (host.EndsWith(".steamcontent.com")) + { + return "steamcontent.com"; + } return host; } - public void RequestCDNAuthToken( uint appid, uint depotid, string host ) + public void RequestCDNAuthToken( uint appid, uint depotid, string host, string cdnKey ) { - host = ResolveCDNTopLevelHost( host ); - var cdnKey = string.Format( "{0:D}:{1}", depotid, host ); - if ( CDNAuthTokens.ContainsKey( cdnKey ) || bAborted ) return; + if ( !CDNAuthTokens.TryAdd( cdnKey, new TaskCompletionSource() ) ) + return; + bool completed = false; + var timeoutDate = DateTime.Now.AddSeconds( 10 ); Action cbMethod = ( cdnAuth ) => { completed = true; @@ -352,13 +358,13 @@ namespace DepotDownloader return; } - CDNAuthTokens.TryAdd( cdnKey, cdnAuth ); + CDNAuthTokens[cdnKey].TrySetResult( cdnAuth ); }; WaitUntilCallback( () => { callbacks.Subscribe( steamApps.GetCDNAuthToken( appid, depotid, host ), cbMethod ); - }, () => { return completed; } ); + }, () => { return completed || DateTime.Now >= timeoutDate; } ); } public void CheckAppBetaPassword( uint appid, string password )