Process download chunks as tasks to increase concurrency

DepotDownloader/CDNClientPool.cs

@@ -17,10 +17,11 @@ namespace DepotDownloader
         private const int ServerEndpointMinimumSize = 8;
 
         private readonly Steam3Session steamSession;
+        private readonly uint appId;
 
         public CDNClient CDNClient { get; }
 
-        private readonly ConcurrentBag<CDNClient.Server> activeConnectionPool;
+        private readonly ConcurrentStack<CDNClient.Server> activeConnectionPool;
         private readonly BlockingCollection<CDNClient.Server> availableServerEndpoints;
 
         private readonly AutoResetEvent populatePoolEvent;
@@ -28,12 +29,13 @@ namespace DepotDownloader
         private readonly CancellationTokenSource shutdownToken;
         public CancellationTokenSource ExhaustedToken { get; set; }
 
-        public CDNClientPool(Steam3Session steamSession)
+        public CDNClientPool(Steam3Session steamSession, uint appId)
         {
             this.steamSession = steamSession;
+            this.appId = appId;
             CDNClient = new CDNClient(steamSession.steamClient);
 
-            activeConnectionPool = new ConcurrentBag<CDNClient.Server>();
+            activeConnectionPool = new ConcurrentStack<CDNClient.Server>();
             availableServerEndpoints = new BlockingCollection<CDNClient.Server>();
 
             populatePoolEvent = new AutoResetEvent(true);
@@ -97,12 +99,23 @@ namespace DepotDownloader
                         return;
                     }
 
-                    var weightedCdnServers = servers.Where(x => x.Type == "SteamCache" || x.Type == "CDN").Select(x =>
+                    var weightedCdnServers = servers
+                        .Where(x =>
+                        {
+#if STEAMKIT_UNRELEASED
+                            var isEligibleForApp = x.AllowedAppIds == null || x.AllowedAppIds.Contains(appId);
+                            return isEligibleForApp && (x.Type == "SteamCache" || x.Type == "CDN");
+#else
+                            return x.Type == "SteamCache" || x.Type == "CDN";
+#endif
+                        })
+                        .Select(x =>
                         {
                             AccountSettingsStore.Instance.ContentServerPenalty.TryGetValue(x.Host, out var penalty);
 
                             return Tuple.Create(x, penalty);
-                    }).OrderBy(x => x.Item2).ThenBy(x => x.Item1.WeightedLoad);
+                        })
+                        .OrderBy(x => x.Item2).ThenBy(x => x.Item1.WeightedLoad);
 
                     foreach (var (server, weight) in weightedCdnServers)
                     {
@@ -122,24 +135,6 @@ namespace DepotDownloader
             }
         }
 
-        private async Task<string> AuthenticateConnection(uint appId, uint depotId, CDNClient.Server server)
-        {
-            var host = steamSession.ResolveCDNTopLevelHost(server.Host);
-            var cdnKey = $"{depotId:D}:{host}";
-
-            steamSession.RequestCDNAuthToken(appId, depotId, host, cdnKey);
-
-            if (steamSession.CDNAuthTokens.TryGetValue(cdnKey, out var authTokenCallbackPromise))
-            {
-                var result = await authTokenCallbackPromise.Task;
-                return result.Token;
-            }
-            else
-            {
-                throw new Exception($"Failed to retrieve CDN token for server {server.Host} depot {depotId}");
-            }
-        }
-
         private CDNClient.Server BuildConnection(CancellationToken token)
         {
             if (availableServerEndpoints.Count < ServerEndpointMinimumSize)
@@ -150,29 +145,42 @@ namespace DepotDownloader
             return availableServerEndpoints.Take(token);
         }
 
-        public async Task<Tuple<CDNClient.Server, string>> GetConnectionForDepot(uint appId, uint depotId, CancellationToken token)
+        public CDNClient.Server GetConnection(CancellationToken token)
         {
-            // Take a free connection from the connection pool
+            if (!activeConnectionPool.TryPop(out var connection))
-            // If there were no free connections, create a new one from the server list
-            if (!activeConnectionPool.TryTake(out var server))
             {
-                server = BuildConnection(token);
+                connection = BuildConnection(token);
             }
 
-            // If we don't have a CDN token yet for this server and depot, fetch one now
+            return connection;
-            var cdnToken = await AuthenticateConnection(appId, depotId, server);
+        }
 
-            return Tuple.Create(server, cdnToken);
+        public async Task<string> AuthenticateConnection(uint appId, uint depotId, CDNClient.Server server)
+        {
+            var host = steamSession.ResolveCDNTopLevelHost(server.Host);
+            var cdnKey = $"{depotId:D}:{host}";
+
+            steamSession.RequestCDNAuthToken(appId, depotId, host, cdnKey);
+
+            if (steamSession.CDNAuthTokens.TryGetValue(cdnKey, out var authTokenCallbackPromise))
+            {
+                var result = await authTokenCallbackPromise.Task;
+                return result.Token;
+            }
+            else
+            {
+                throw new Exception($"Failed to retrieve CDN token for server {server.Host} depot {depotId}");
+            }
         }
 
-        public void ReturnConnection(Tuple<CDNClient.Server, string> server)
+        public void ReturnConnection(CDNClient.Server server)
         {
             if (server == null) return;
 
-            activeConnectionPool.Add(server.Item1);
+            activeConnectionPool.Push(server);
         }
 
-        public void ReturnBrokenConnection(Tuple<CDNClient.Server, string> server)
+        public void ReturnBrokenConnection(CDNClient.Server server)
         {
             if (server == null) return;
 

DepotDownloader/ContentDownloader.cs

@@ -1,5 +1,6 @@
 using SteamKit2;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -361,7 +362,6 @@ namespace DepotDownloader
                 return false;
             }
 
-            cdnPool = new CDNClientPool( steam3 );
             return true;
         }
 
@@ -396,6 +396,8 @@ namespace DepotDownloader
 
         public static async Task DownloadAppAsync( uint appId, uint depotId, ulong manifestId, string branch, string os, string arch, string language, bool lv, bool isUgc )
         {
+            cdnPool = new CDNClientPool(steam3, appId);
+
             // Load our configuration data containing the depots currently installed
             string configPath = ContentDownloader.Config.InstallDirectory;
             if (string.IsNullOrWhiteSpace(configPath))
@@ -595,22 +597,78 @@ namespace DepotDownloader
             public ProtoManifest.ChunkData NewChunk { get; private set; }
         }
 
-        private static async Task DownloadSteam3Async( uint appId, List<DepotDownloadInfo> depots )
+        private class DepotFilesData
         {
-            ulong TotalBytesCompressed = 0;
+            public DepotDownloadInfo depotDownloadInfo;
-            ulong TotalBytesUncompressed = 0;
+            public DepotDownloadCounter depotCounter;
-            var previousFiles = new List<ProtoManifest.FileData>();
+            public string stagingDir;
+            public ProtoManifest manifest;
+            public ProtoManifest previousManifest;
+            public List<ProtoManifest.FileData> filteredFiles;
+            public HashSet<string> allFileNames;
+        }
 
-            foreach ( var depot in depots )
+        private class FileStreamData
+        {
+            public FileStream fileStream;
+            public SemaphoreSlim fileLock;
+            public int chunksToDownload;
+        }
+
+        private class GlobalDownloadCounter
         {
-                ulong DepotBytesCompressed = 0;
+            public ulong TotalBytesCompressed;
-                ulong DepotBytesUncompressed = 0;
+            public ulong TotalBytesUncompressed;
+        }
 
-                Console.WriteLine( "Downloading depot {0} - {1}", depot.id, depot.contentName );
+        private class DepotDownloadCounter
+        {
+            public ulong CompleteDownloadSize;
+            public ulong SizeDownloaded;
+            public ulong DepotBytesCompressed;
+            public ulong DepotBytesUncompressed;
 
+        }
+
+        private static async Task DownloadSteam3Async(uint appId, List<DepotDownloadInfo> depots)
+        {
             CancellationTokenSource cts = new CancellationTokenSource();
             cdnPool.ExhaustedToken = cts;
 
+            GlobalDownloadCounter downloadCounter = new GlobalDownloadCounter();
+            var depotsToDownload = new List<DepotFilesData>(depots.Count);
+            var allFileNames = new HashSet<String>();
+
+            // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
+            foreach (var depot in depots)
+            {
+                var depotFileData = await ProcessDepotManifestAndFiles(cts, appId, depot);
+
+                if (depotFileData != null)
+                {
+                    depotsToDownload.Add(depotFileData);
+                    allFileNames.UnionWith(depotFileData.allFileNames);
+                }
+
+                cts.Token.ThrowIfCancellationRequested();
+            }
+
+            foreach (var depotFileData in depotsToDownload)
+            {
+                await DownloadSteam3AsyncDepotFiles(cts, appId, downloadCounter, depotFileData, allFileNames);
+            }
+
+            Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots",
+                downloadCounter.TotalBytesCompressed, downloadCounter.TotalBytesUncompressed, depots.Count);
+        }
+
+        private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts, 
+            uint appId, DepotDownloadInfo depot)
+        {
+            DepotDownloadCounter depotCounter = new DepotDownloadCounter();
+
+            Console.WriteLine("Processing depot {0} - {1}", depot.id, depot.contentName);
+
             ProtoManifest oldProtoManifest = null;
             ProtoManifest newProtoManifest = null;
             string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
@@ -691,18 +749,26 @@ namespace DepotDownloader
 
                     DepotManifest depotManifest = null;
 
-                        while ( depotManifest == null )
+                    do
                     {
-                            Tuple<CDNClient.Server, string> connection = null;
+                        cts.Token.ThrowIfCancellationRequested();
+
+                        CDNClient.Server connection = null;
+
                         try
                         {
-                                connection = await cdnPool.GetConnectionForDepot( appId, depot.id, CancellationToken.None );
+                            connection = cdnPool.GetConnection(cts.Token);
+                            var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);
 
                             depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
-                                    connection.Item1, connection.Item2, depot.depotKey ).ConfigureAwait(false);
+                                connection, cdnToken, depot.depotKey).ConfigureAwait(false);
 
                             cdnPool.ReturnConnection(connection);
                         }
+                        catch (TaskCanceledException)
+                        {
+                            Console.WriteLine("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId);
+                        }
                         catch (SteamKitWebRequestException e)
                         {
                             cdnPool.ReturnBrokenConnection(connection);
@@ -717,19 +783,27 @@ namespace DepotDownloader
                                 Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode);
                             }
                         }
+                        catch (OperationCanceledException)
+                        {
+                            break;
+                        }
                         catch (Exception e)
                         {
                             cdnPool.ReturnBrokenConnection(connection);
                             Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
                         }
                     }
+                    while (depotManifest == null);
 
                     if (depotManifest == null)
                     {
                         Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
-                            return;
+                        cts.Cancel();
                     }
 
+                    // Throw the cancellation exception if requested so that this task is marked failed
+                    cts.Token.ThrowIfCancellationRequested();
+
                     byte[] checksum;
 
                     newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
@@ -761,18 +835,19 @@ namespace DepotDownloader
                 }
 
                 File.WriteAllText(txtManifest, manifestBuilder.ToString());
-                    continue;
+                return null;
             }
 
-                ulong complete_download_size = 0;
-                ulong size_downloaded = 0;
             string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
 
             var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
+            var allFileNames = new HashSet<string>(filesAfterExclusions.Count);
 
             // Pre-process
             filesAfterExclusions.ForEach(file =>
             {
+                allFileNames.Add(file.FileName);
+
                 var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
                 var fileStagingPath = Path.Combine(stagingDir, file.FileName);
 
@@ -787,25 +862,85 @@ namespace DepotDownloader
                     Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
                     Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
 
-                        complete_download_size += file.TotalSize;
+                    depotCounter.CompleteDownloadSize += file.TotalSize;
                 }
             });
 
-                var semaphore = new SemaphoreSlim( Config.MaxDownloads );
+            return new DepotFilesData
-                var files = filesAfterExclusions.Where( f => !f.Flags.HasFlag( EDepotFileFlag.Directory ) ).ToArray();
-                var tasks = new Task[ files.Length ];
-                for ( var i = 0; i < files.Length; i++ )
             {
-                    var file = files[ i ];
+                depotDownloadInfo = depot,
-                    var task = Task.Run( async () =>
+                depotCounter = depotCounter,
+                stagingDir = stagingDir,
+                manifest = newProtoManifest,
+                previousManifest = oldProtoManifest,
+                filteredFiles = filesAfterExclusions,
+                allFileNames = allFileNames
+            };
+        }
+
+        private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource cts, uint appId,
+            GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, HashSet<String> allFileNames)
         {
-                        cts.Token.ThrowIfCancellationRequested();
+            var depot = depotFilesData.depotDownloadInfo;
+            var depotCounter = depotFilesData.depotCounter;
 
-                        try
+            Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
+
+            var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
+            var networkChunkQueue = new ConcurrentQueue<Tuple<FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData>>();
+
+            await Util.InvokeAsync(
+                files.Select(file => new Func<Task>(async () =>
+                    await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, depotFilesData, file, networkChunkQueue)))),
+                maxDegreeOfParallelism: Config.MaxDownloads
+            );
+
+            await Util.InvokeAsync(
+                networkChunkQueue.Select((x) => new Func<Task>(async () =>
+                    await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, appId, downloadCounter, depotFilesData,
+                        x.Item2, x.Item1, x.Item3)))),
+                maxDegreeOfParallelism: Config.MaxDownloads
+            );
+
+            // Check for deleted files if updating the depot.
+            if (depotFilesData.previousManifest != null)
+            {
+                var previousFilteredFiles = depotFilesData.previousManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).Select(f => f.FileName).ToHashSet();
+
+                // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names across all depots being downloaded
+                previousFilteredFiles.ExceptWith(allFileNames);
+
+                foreach(var existingFileName in previousFilteredFiles)
+                {
+                    string fileFinalPath = Path.Combine(depot.installDir, existingFileName);
+
+                    if (!File.Exists(fileFinalPath))
+                        continue;
+
+                    File.Delete(fileFinalPath);
+                    Console.WriteLine("Deleted {0}", fileFinalPath);
+                }
+            }
+
+            DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId;
+            DepotConfigStore.Save();
+
+            Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed);
+        }
+
+        private static void DownloadSteam3AsyncDepotFile(
+            CancellationTokenSource cts,
+            DepotFilesData depotFilesData,
+            ProtoManifest.FileData file,
+            ConcurrentQueue<Tuple<FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData>> networkChunkQueue)
         {
-                            await semaphore.WaitAsync().ConfigureAwait( false );
             cts.Token.ThrowIfCancellationRequested();
 
+            var depot = depotFilesData.depotDownloadInfo;
+            var stagingDir = depotFilesData.stagingDir;
+            var depotDownloadCounter = depotFilesData.depotCounter;
+            var oldProtoManifest = depotFilesData.previousManifest;
+
             string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
             string fileStagingPath = Path.Combine(stagingDir, file.FileName);
 
@@ -820,6 +955,8 @@ namespace DepotDownloader
             FileInfo fi = new FileInfo(fileFinalPath);
             if (!fi.Exists)
             {
+                Console.WriteLine("Pre-allocating {0}", fileFinalPath);
+
                 // create new file. need all chunks
                 fs = File.Create(fileFinalPath);
                 fs.SetLength((long)file.TotalSize);
@@ -841,6 +978,10 @@ namespace DepotDownloader
                     if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
                     {
                         // we have a version of this file, but it doesn't fully match what we want
+                        if (Config.VerifyAll)
+                        {
+                            Console.WriteLine("Validating {0}", fileFinalPath);
+                        }
 
                         var matchingChunks = new List<ChunkMatch>();
 
@@ -857,6 +998,8 @@ namespace DepotDownloader
                             }
                         }
 
+                        var orderedChunks = matchingChunks.OrderBy(x => x.OldChunk.Offset);
+
                         File.Move(fileFinalPath, fileStagingPath);
 
                         fs = File.Open(fileFinalPath, FileMode.Create);
@@ -864,7 +1007,7 @@ namespace DepotDownloader
 
                         using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
                         {
-                                            foreach ( var match in matchingChunks )
+                            foreach (var match in orderedChunks)
                             {
                                 fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
 
@@ -897,42 +1040,60 @@ namespace DepotDownloader
                         fs.SetLength((long)file.TotalSize);
                     }
 
+                    Console.WriteLine("Validating {0}", fileFinalPath);
                     neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
                 }
 
                 if (neededChunks.Count() == 0)
                 {
-                                    size_downloaded += file.TotalSize;
+                    lock (depotDownloadCounter)
-                                    Console.WriteLine( "{0,6:#00.00}% {1}", ( ( float )size_downloaded / ( float )complete_download_size ) * 100.0f, fileFinalPath );
+                    {
+                        depotDownloadCounter.SizeDownloaded += (ulong)file.TotalSize;
+                        Console.WriteLine("{0,6:#00.00}% {1}", ((float)depotDownloadCounter.SizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath);
+                    }
+
                     if (fs != null)
                         fs.Dispose();
                     return;
                 }
                 else
                 {
-                                    size_downloaded += ( file.TotalSize - ( ulong )neededChunks.Select( x => ( long )x.UncompressedLength ).Sum() );
+                    var sizeOnDisk = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum());
+                    lock (depotDownloadCounter)
+                    {
+                        depotDownloadCounter.SizeDownloaded += sizeOnDisk;
+                    }
                 }
             }
 
-                            foreach ( var chunk in neededChunks )
+            FileStreamData fileStreamData = new FileStreamData
             {
-                                if ( cts.IsCancellationRequested ) break;
+                fileStream = fs,
-
+                fileLock = new SemaphoreSlim(1),
-                                string chunkID = Util.EncodeHexString( chunk.ChunkID );
+                chunksToDownload = neededChunks.Count
-                                CDNClient.DepotChunk chunkData = null;
+            };
 
-                                while ( !cts.IsCancellationRequested )
+            foreach (var chunk in neededChunks)
-                                {
-                                    Tuple<CDNClient.Server, string> connection;
-                                    try
             {
-                                        connection = await cdnPool.GetConnectionForDepot( appId, depot.id, cts.Token );
+                networkChunkQueue.Enqueue(Tuple.Create(fileStreamData, file, chunk));
             }
-                                    catch ( OperationCanceledException )
-                                    {
-                                        break;
         }
 
+        private static async Task DownloadSteam3AsyncDepotFileChunk(
+            CancellationTokenSource cts, uint appId,
+            GlobalDownloadCounter downloadCounter,
+            DepotFilesData depotFilesData,
+            ProtoManifest.FileData file, 
+            FileStreamData fileStreamData, 
+            ProtoManifest.ChunkData chunk)
+        {
+            cts.Token.ThrowIfCancellationRequested();
+
+            var depot = depotFilesData.depotDownloadInfo;
+            var depotDownloadCounter = depotFilesData.depotCounter;
+
+            string chunkID = Util.EncodeHexString(chunk.ChunkID);
+
             DepotManifest.ChunkData data = new DepotManifest.ChunkData();
             data.ChunkID = chunk.ChunkID;
             data.Checksum = chunk.Checksum;
@@ -940,12 +1101,27 @@ namespace DepotDownloader
             data.CompressedLength = chunk.CompressedLength;
             data.UncompressedLength = chunk.UncompressedLength;
 
+            CDNClient.DepotChunk chunkData = null;
+
+            do
+            {
+                cts.Token.ThrowIfCancellationRequested();
+
+                CDNClient.Server connection = null;
+
                 try
                 {
+                    connection = cdnPool.GetConnection(cts.Token);
+                    var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);
+
                     chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data,
-                                            connection.Item1, connection.Item2, depot.depotKey ).ConfigureAwait( false );
+                        connection, cdnToken, depot.depotKey).ConfigureAwait(false);
+
                     cdnPool.ReturnConnection(connection);
-                                        break;
+                }
+                catch (TaskCanceledException)
+                {
+                    Console.WriteLine("Connection timeout downloading chunk {0}", chunkID);
                 }
                 catch (SteamKitWebRequestException e)
                 {
@@ -954,7 +1130,6 @@ namespace DepotDownloader
                     if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
                     {
                         Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
-                                            cts.Cancel();
                         break;
                     }
                     else
@@ -962,9 +1137,9 @@ namespace DepotDownloader
                         Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
                     }
                 }
-                                    catch ( TaskCanceledException )
+                catch (OperationCanceledException)
                 {
-                                        Console.WriteLine( "Connection timeout downloading chunk {0}", chunkID );
+                    break;
                 }
                 catch (Exception e)
                 {
@@ -972,6 +1147,7 @@ namespace DepotDownloader
                     Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
                 }
             }
+            while (chunkData == null);
 
             if (chunkData == null)
             {
@@ -982,67 +1158,46 @@ namespace DepotDownloader
             // Throw the cancellation exception if requested so that this task is marked failed
             cts.Token.ThrowIfCancellationRequested();
 
-                                TotalBytesCompressed += chunk.CompressedLength;
+            try
-                                DepotBytesCompressed += chunk.CompressedLength;
+            {
-                                TotalBytesUncompressed += chunk.UncompressedLength;
+                await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false);
-                                DepotBytesUncompressed += chunk.UncompressedLength;
-
-                                fs.Seek( ( long )chunk.Offset, SeekOrigin.Begin );
-                                fs.Write( chunkData.Data, 0, chunkData.Data.Length );
-
-                                size_downloaded += chunk.UncompressedLength;
-                            }
-
-                            fs.Dispose();
 
-                            Console.WriteLine( "{0,6:#00.00}% {1}", ( ( float )size_downloaded / ( float )complete_download_size ) * 100.0f, fileFinalPath );
+                fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin);
+                await fileStreamData.fileStream.WriteAsync(chunkData.Data, 0, chunkData.Data.Length);
             }
             finally
             {
-                            semaphore.Release();
+                fileStreamData.fileLock.Release();
             }
-                    } );
 
-                    tasks[ i ] = task;
+            int remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload);
+            if (remainingChunks == 0)
+            {
+                fileStreamData.fileStream.Dispose();
+                fileStreamData.fileLock.Dispose();
             }
 
-                await Task.WhenAll( tasks ).ConfigureAwait( false );
+            ulong sizeDownloaded = 0;
-
+            lock (depotDownloadCounter)
-                // Check for deleted files if updating the depot.
-                if ( oldProtoManifest != null )
             {
-                    var oldfilesAfterExclusions = oldProtoManifest.Files.AsParallel().Where( f => TestIsFileIncluded( f.FileName ) ).ToList();
+                sizeDownloaded = depotDownloadCounter.SizeDownloaded + (ulong)chunkData.Data.Length;
+                depotDownloadCounter.SizeDownloaded = sizeDownloaded;
+                depotDownloadCounter.DepotBytesCompressed += chunk.CompressedLength;
+                depotDownloadCounter.DepotBytesUncompressed += chunk.UncompressedLength;
+            }
 
-                    foreach ( var file in oldfilesAfterExclusions )
+            lock (downloadCounter)
             {
-                        // Delete it if it's in the old manifest AND not in the new manifest AND not in any of the previous depots.
+                downloadCounter.TotalBytesCompressed += chunk.CompressedLength;
-                        var newManifestFile = filesAfterExclusions.SingleOrDefault( f => f.FileName == file.FileName );
+                downloadCounter.TotalBytesUncompressed += chunk.UncompressedLength;
-                        if ( newManifestFile == null )
-                            continue;
-
-                        var previousFile = previousFiles.SingleOrDefault( f => f.FileName == file.FileName );
-                        if ( previousFile == null )
-                            continue;
-
-                        string fileFinalPath = Path.Combine( depot.installDir, file.FileName );
-                        if ( !File.Exists( fileFinalPath ) )
-                            continue;
-
-                        File.Delete( fileFinalPath );
-                        Console.WriteLine( "Deleted {0}", fileFinalPath );
-                    }
             }
             
-                // Remember files we processed for later.
+            if (remainingChunks == 0)
-                previousFiles.AddRange( filesAfterExclusions );
+            {
-
+                var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
-                DepotConfigStore.Instance.InstalledManifestIDs[ depot.id ] = depot.manifestId;
+                Console.WriteLine("{0,6:#00.00}% {1}", ((float)sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath);
-                DepotConfigStore.Save();
-
-                Console.WriteLine( "Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed );
             }
 
-            Console.WriteLine( "Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count );
         }
     }
 }

DepotDownloader/Program.cs

@@ -108,7 +108,7 @@ namespace DepotDownloader
 
             ContentDownloader.Config.VerifyAll = HasParameter( args, "-verify-all" ) || HasParameter( args, "-verify_all" ) || HasParameter( args, "-validate" );
             ContentDownloader.Config.MaxServers = GetParameter<int>( args, "-max-servers", 20 );
-            ContentDownloader.Config.MaxDownloads = GetParameter<int>( args, "-max-downloads", 4 );
+            ContentDownloader.Config.MaxDownloads = GetParameter<int>( args, "-max-downloads", 8 );
             ContentDownloader.Config.MaxServers = Math.Max( ContentDownloader.Config.MaxServers, ContentDownloader.Config.MaxDownloads );
             ContentDownloader.Config.LoginID = HasParameter( args, "-loginid" ) ? (uint?)GetParameter<uint>( args, "-loginid" ) : null;
 
@@ -341,8 +341,8 @@ namespace DepotDownloader
             Console.WriteLine();
             Console.WriteLine( "\t-manifest-only\t\t\t- downloads a human readable manifest for any depots that would be downloaded." );
             Console.WriteLine( "\t-cellid <#>\t\t\t\t- the overridden CellID of the content server to download from." );
-            Console.WriteLine( "\t-max-servers <#>\t\t- maximum number of content servers to use. (default: 8)." );
+            Console.WriteLine( "\t-max-servers <#>\t\t- maximum number of content servers to use. (default: 20)." );
-            Console.WriteLine( "\t-max-downloads <#>\t\t- maximum number of chunks to download concurrently. (default: 4)." );
+            Console.WriteLine( "\t-max-downloads <#>\t\t- maximum number of chunks to download concurrently. (default: 8)." );
             Console.WriteLine( "\t-loginid <#>\t\t- a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently." );
         }
     }

DepotDownloader/Steam3Session.cs

@@ -129,17 +129,25 @@ namespace DepotDownloader
         }
 
         public delegate bool WaitCondition();
+        private object steamLock = new object();
+
         public bool WaitUntilCallback( Action submitter, WaitCondition waiter )
         {
             while ( !bAborted && !waiter() )
+            {
+                lock (steamLock)
                 {
                     submitter();
+                }
 
                 int seq = this.seq;
                 do
+                {
+                    lock (steamLock)
                     {
                         WaitForCallbacks();
                     }
+                }
                 while ( !bAborted && this.seq == seq && !waiter() );
             }
 
@@ -473,7 +481,7 @@ namespace DepotDownloader
 
         public void TryWaitForLoginKey()
         {
-            if ( logonDetails.Username == null || !ContentDownloader.Config.RememberPassword ) return;
+            if ( logonDetails.Username == null || !credentials.LoggedOn || !ContentDownloader.Config.RememberPassword ) return;
 
             var totalWaitPeriod = DateTime.Now.AddSeconds( 3 );
 
@@ -584,7 +592,7 @@ namespace DepotDownloader
                     }
                     else
                     {
-                        Console.WriteLine( "Login key was expired. Please enter your password: " );
+                        Console.Write( "Login key was expired. Please enter your password: " );
                         logonDetails.Password = Util.ReadPassword();
                     }
                 }

DepotDownloader/Util.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace DepotDownloader
 {
@@ -132,5 +133,38 @@ namespace DepotDownloader
                 ( sb, v ) => sb.Append( v.ToString( "x2" ) )
                 ).ToString();
         }
+        
+        public static async Task InvokeAsync(IEnumerable<Func<Task>> taskFactories, int maxDegreeOfParallelism)
+        {
+            if (taskFactories == null) throw new ArgumentNullException(nameof(taskFactories));
+            if (maxDegreeOfParallelism <= 0) throw new ArgumentException(nameof(maxDegreeOfParallelism));
+
+            Func<Task>[] queue = taskFactories.ToArray();
+
+            if (queue.Length == 0)
+            {
+                return;
+            }
+
+            List<Task> tasksInFlight = new List<Task>(maxDegreeOfParallelism);
+            int index = 0;
+
+            do
+            {
+                while (tasksInFlight.Count < maxDegreeOfParallelism && index < queue.Length)
+                {
+                    Func<Task> taskFactory = queue[index++];
+
+                    tasksInFlight.Add(taskFactory());
+                }
+
+                Task completedTask = await Task.WhenAny(tasksInFlight).ConfigureAwait(false);
+
+                await completedTask.ConfigureAwait(false);
+
+                tasksInFlight.Remove(completedTask);
+            }
+            while (index < queue.Length || tasksInFlight.Count != 0);
+        }
     }
 }

README.md

@@ -50,6 +50,6 @@ Parameter | Description
 -validate				| Include checksum verification of files already downloaded
 -manifest-only			| downloads a human readable manifest for any depots that would be downloaded.
 -cellid \<#>			| the overridden CellID of the content server to download from.
--max-servers \<#>		| maximum number of content servers to use. (default: 8).
+-max-servers \<#>		| maximum number of content servers to use. (default: 20).
--max-downloads \<#>		| maximum number of chunks to download concurrently. (default: 4).
+-max-downloads \<#>		| maximum number of chunks to download concurrently. (default: 8).
 -loginid \<#>			| a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently.