Submit chunks to be downloaded as independent tasks

DepotDownloader/ContentDownloader.cs

@@ -787,9 +787,11 @@ namespace DepotDownloader
                     }
                 } );
 
-                var semaphore = new SemaphoreSlim( Config.MaxDownloads );
+                var ioSemaphore = new SemaphoreSlim( Config.MaxDownloads );
+                var networkSemaphore = new SemaphoreSlim( Config.MaxDownloads );
+
                 var files = filesAfterExclusions.Where( f => !f.Flags.HasFlag( EDepotFileFlag.Directory ) ).ToArray();
-                var tasks = new Task[ files.Length ];
+                var ioTasks = new Task[ files.Length ];
                 for ( var i = 0; i < files.Length; i++ )
                 {
                     var file = files[ i ];
@@ -799,7 +801,7 @@ namespace DepotDownloader
                         
                         try
                         {
-                            await semaphore.WaitAsync().ConfigureAwait( false );
+                            await ioSemaphore.WaitAsync().ConfigureAwait( false );
                             cts.Token.ThrowIfCancellationRequested();
 
                             string fileFinalPath = Path.Combine( depot.installDir, file.FileName );
@@ -910,9 +912,21 @@ namespace DepotDownloader
                                 }
                             }
 
-                            foreach ( var chunk in neededChunks )
+                            var fileSemaphore = new SemaphoreSlim(1);
+
+                            var downloadTasks = new Task<DepotManifest.ChunkData>[neededChunks.Count];
+                            for (var x = 0; x < neededChunks.Count; x++)
+                            {
+                                var chunk = neededChunks[x];
+
+                                var downloadTask = Task.Run(async () =>
+                                {
+                                    cts.Token.ThrowIfCancellationRequested();
+
+                                    try
                                     {
-                                if ( cts.IsCancellationRequested ) break;
+                                        await networkSemaphore.WaitAsync().ConfigureAwait(false);
+                                        cts.Token.ThrowIfCancellationRequested();
 
                                         string chunkID = Util.EncodeHexString(chunk.ChunkID);
                                         CDNClient.DepotChunk chunkData = null;
@@ -941,6 +955,7 @@ namespace DepotDownloader
                                                 chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data,
                                                     connection.Item1, connection.Item2, depot.depotKey).ConfigureAwait(false);
                                                 cdnPool.ReturnConnection(connection);
+
                                                 break;
                                             }
                                             catch (SteamKitWebRequestException e)
@@ -978,31 +993,55 @@ namespace DepotDownloader
                                         // Throw the cancellation exception if requested so that this task is marked failed
                                         cts.Token.ThrowIfCancellationRequested();
 
-                                TotalBytesCompressed += chunk.CompressedLength;
+                                        try
-                                DepotBytesCompressed += chunk.CompressedLength;
+                                        {
-                                TotalBytesUncompressed += chunk.UncompressedLength;
+                                            await fileSemaphore.WaitAsync().ConfigureAwait(false);
-                                DepotBytesUncompressed += chunk.UncompressedLength;
 
-                                fs.Seek( ( long )chunk.Offset, SeekOrigin.Begin );
+                                            fs.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin);
                                             fs.Write(chunkData.Data, 0, chunkData.Data.Length);
 
-                                size_downloaded += chunk.UncompressedLength;
+                                            return chunkData.ChunkInfo;
                                         }
+                                        finally
+                                        {
+                                            fileSemaphore.Release();
+                                        }
+                                    }
+                                    finally
+                                    {
+                                        networkSemaphore.Release();
+                                    }
+                                });
+
+                                downloadTasks[x] = downloadTask;
+                            }
+
+                            var completedDownloads = await Task.WhenAll(downloadTasks).ConfigureAwait(false);
 
                             fs.Dispose();
 
+                            foreach (var chunkInfo in completedDownloads)
+                            {
+                                TotalBytesCompressed += chunkInfo.CompressedLength;
+                                DepotBytesCompressed += chunkInfo.CompressedLength;
+                                TotalBytesUncompressed += chunkInfo.UncompressedLength;
+                                DepotBytesUncompressed += chunkInfo.UncompressedLength;
+
+                                size_downloaded += chunkInfo.UncompressedLength;
+                            }
+
                             Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath);
                         }
                         finally
                         {
-                            semaphore.Release();
+                            ioSemaphore.Release();
                         }
                     } );
 
-                    tasks[ i ] = task;
+                    ioTasks[ i ] = task;
                 }
 
-                await Task.WhenAll( tasks ).ConfigureAwait( false );
+                await Task.WhenAll(ioTasks).ConfigureAwait(false);
 
                 DepotConfigStore.Instance.InstalledManifestIDs[ depot.id ] = depot.manifestId;
                 DepotConfigStore.Save();

DepotDownloader/Program.cs

@@ -108,7 +108,7 @@ namespace DepotDownloader
 
             ContentDownloader.Config.VerifyAll = HasParameter( args, "-verify-all" ) || HasParameter( args, "-verify_all" ) || HasParameter( args, "-validate" );
             ContentDownloader.Config.MaxServers = GetParameter<int>( args, "-max-servers", 20 );
-            ContentDownloader.Config.MaxDownloads = GetParameter<int>( args, "-max-downloads", 4 );
+            ContentDownloader.Config.MaxDownloads = GetParameter<int>( args, "-max-downloads", 8 );
             ContentDownloader.Config.MaxServers = Math.Max( ContentDownloader.Config.MaxServers, ContentDownloader.Config.MaxDownloads );
             ContentDownloader.Config.LoginID = HasParameter( args, "-loginid" ) ? (uint?)GetParameter<uint>( args, "-loginid" ) : null;
 
@@ -341,7 +341,7 @@ namespace DepotDownloader
             Console.WriteLine();
             Console.WriteLine( "\t-manifest-only\t\t\t- downloads a human readable manifest for any depots that would be downloaded." );
             Console.WriteLine( "\t-cellid <#>\t\t\t\t- the overridden CellID of the content server to download from." );
-            Console.WriteLine( "\t-max-servers <#>\t\t- maximum number of content servers to use. (default: 8)." );
+            Console.WriteLine( "\t-max-servers <#>\t\t- maximum number of content servers to use. (default: 20)." );
             Console.WriteLine( "\t-max-downloads <#>\t\t- maximum number of chunks to download concurrently. (default: 4)." );
             Console.WriteLine( "\t-loginid <#>\t\t- a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently." );
         }

DepotDownloader/Steam3Session.cs

@@ -129,15 +129,18 @@ namespace DepotDownloader
         }
 
         public delegate bool WaitCondition();
+        private object steamLock = new object();
         public bool WaitUntilCallback( Action submitter, WaitCondition waiter )
         {
             while ( !bAborted && !waiter() )
             {
+                lock (steamLock)
                     submitter();
 
                 int seq = this.seq;
                 do
                 {
+                    lock (steamLock)
                         WaitForCallbacks();
                 }
                 while ( !bAborted && this.seq == seq && !waiter() );
@@ -473,7 +476,7 @@ namespace DepotDownloader
 
         public void TryWaitForLoginKey()
         {
-            if ( logonDetails.Username == null || !ContentDownloader.Config.RememberPassword ) return;
+            if ( logonDetails.Username == null || !credentials.LoggedOn || !ContentDownloader.Config.RememberPassword ) return;
 
             var totalWaitPeriod = DateTime.Now.AddSeconds( 3 );
 
@@ -584,7 +587,7 @@ namespace DepotDownloader
                     }
                     else
                     {
-                        Console.WriteLine( "Login key was expired. Please enter your password: " );
+                        Console.Write( "Login key was expired. Please enter your password: " );
                         logonDetails.Password = Util.ReadPassword();
                     }
                 }

README.md

@@ -51,5 +51,5 @@ Parameter | Description
 -manifest-only			| downloads a human readable manifest for any depots that would be downloaded.
 -cellid \<#>			| the overridden CellID of the content server to download from.
 -max-servers \<#>		| maximum number of content servers to use. (default: 8).
--max-downloads \<#>		| maximum number of chunks to download concurrently. (default: 4).
+-max-downloads \<#>		| maximum number of chunks to download concurrently. (default: 8).
 -loginid \<#>			| a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently.