From a0065344429873afd54f40e1bbdf11a619ce45f1 Mon Sep 17 00:00:00 2001 From: ddavidov-nv Date: Thu, 5 Jun 2025 10:18:20 +0300 Subject: [PATCH] Improvements and Contributions to DepotDownloader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This contribution enhances DepotDownloader with several significant improvements: 1. DepotLayout: Separated depot files into private folders for easier deployment in 3rd party systems. 2. Bandwidth Optimization: On high speed connections , Re-download outdated files instead of patching , reducing CPU/memory load during patching. 3. Robust Utilities: Added retry logic to Utils. Used for GetDepotDecryptionKey, to prevent process failures. 4 Error Handling: entitlement/License checks — now reports issues and continues downloading remaining depots. 5. DownloadMonitor: Introduced real-time download status updates to prevent silent periods during large file handling. --- .gitattributes | 2 +- .gitignore | 4 +- DepotDownloader/ContentDownloader.cs | 179 ++++++++++++++++++++----- DepotDownloader/DepotDownloader.csproj | 2 +- DepotDownloader/DownloadConfig.cs | 2 + DepotDownloader/Program.cs | 11 +- DepotDownloader/Steam3Session.cs | 19 +-- DepotDownloader/Util.cs | 35 +++++ README.md | 2 + global.json | 2 +- 10 files changed, 207 insertions(+), 51 deletions(-) diff --git a/.gitattributes b/.gitattributes index ff739929..48b55300 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,4 @@ *.cs text eol=lf *.csproj text eol=lf *.config eol=lf -*.json eol=lf +*.json eol=lf \ No newline at end of file diff --git a/.gitignore b/.gitignore index ce528650..5d12f982 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ # mstest test results TestResults - +.history ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. @@ -118,4 +118,6 @@ cryptopp/ # misc Thumbs.db +.vscode +DepotDownloader/Properties launchSettings.json diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader/ContentDownloader.cs index 967cc499..bd637154 100644 --- a/DepotDownloader/ContentDownloader.cs +++ b/DepotDownloader/ContentDownloader.cs @@ -68,9 +68,8 @@ namespace DepotDownloader else { Directory.CreateDirectory(Config.InstallDirectory); - - installDir = Config.InstallDirectory; - + installDir = Config.DepotLayout ? Path.Combine(Config.InstallDirectory, depotId.ToString()) : Config.InstallDirectory; + Directory.CreateDirectory(installDir); Directory.CreateDirectory(Path.Combine(installDir, CONFIG_DIR)); Directory.CreateDirectory(Path.Combine(installDir, STAGING_DIR)); } @@ -435,7 +434,9 @@ namespace DepotDownloader else { var contentName = GetAppName(appId); - throw new ContentDownloaderException(string.Format("App {0} ({1}) is not available from this account.", appId, contentName)); + Console.WriteLine(string.Format("Error: App {0} ({1}) is not available from this account.", appId, contentName)); + // Skip this app and continue processing other apps in the list since we don't have access to download it + return; } } @@ -600,8 +601,7 @@ namespace DepotDownloader Console.WriteLine("Error: Unable to create install directories!"); return null; } - - // For depots that are proxied through depotfromapp, we still need to resolve the proxy app id, unless the app is freetodownload + // For depots that are proxied through depotfromapp, we still need to resolve the proxy app id var containingAppId = appId; var proxyAppId = GetSteam3DepotProxyAppId(depotId, appId); if (proxyAppId != INVALID_APP_ID) @@ -655,6 +655,80 @@ namespace DepotDownloader public ulong depotBytesUncompressed; } + /// + /// Monitors and reports download progress at regular intervals + /// + private sealed class DownloadMonitor : IDisposable + { + private readonly GlobalDownloadCounter globalDownloadCounter; + private readonly int reportIntervalMs; + private Thread monitorThread; + private volatile bool running = false; + private long lastDownloadSize = 0; + private readonly AutoResetEvent threadWait = new(false); + private readonly object lockObject = new(); + + public DownloadMonitor( + GlobalDownloadCounter globalDownloadCounter, + int reportIntervalMs = 60000) + { + this.globalDownloadCounter = globalDownloadCounter; + this.reportIntervalMs = reportIntervalMs; + } + + public void Start() + { + if (monitorThread != null) + return; + + running = true; + monitorThread = new Thread(ReportAlive); + monitorThread.Start(); + } + + public void Stop() + { + if (!running) + return; + + running = false; + threadWait.Set(); + monitorThread?.Join(); + monitorThread = null; + } + + private void ReportAlive() + { + try + { + while (running) + { + ulong currentDownloadedBytes; + lock (lockObject) + { + currentDownloadedBytes = globalDownloadCounter.totalBytesUncompressed; + } + + if (currentDownloadedBytes > (ulong)Interlocked.Read(ref lastDownloadSize)) + { + Interlocked.Exchange(ref lastDownloadSize, (long)currentDownloadedBytes); + Console.WriteLine("Downloaded : {0}", Util.FormatSize((long)currentDownloadedBytes)); + } + threadWait.WaitOne(reportIntervalMs); + } + } + catch (Exception ex) + { + Console.WriteLine($"Error in download monitor: {ex.Message}"); + } + } + + public void Dispose() + { + Stop(); + threadWait.Dispose(); + } + } private static async Task DownloadSteam3Async(List depots) { Ansi.Progress(Ansi.ProgressState.Indeterminate); @@ -662,50 +736,70 @@ namespace DepotDownloader await cdnPool.UpdateServerList(); var cts = new CancellationTokenSource(); + var downloadCounter = new GlobalDownloadCounter(); + var downloadMonitor = new DownloadMonitor(downloadCounter); var depotsToDownload = new List(depots.Count); var allFileNamesAllDepots = new HashSet(); - // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup - foreach (var depot in depots) + try { - var depotFileData = await ProcessDepotManifestAndFiles(cts, depot, downloadCounter); - - if (depotFileData != null) + downloadMonitor.Start(); + // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup + foreach (var depot in depots) { - depotsToDownload.Add(depotFileData); - allFileNamesAllDepots.UnionWith(depotFileData.allFileNames); + var depotFileData = await ProcessDepotManifestAndFiles(cts, depot, downloadCounter); + + if (depotFileData != null) + { + depotsToDownload.Add(depotFileData); + allFileNamesAllDepots.UnionWith(depotFileData.allFileNames); + } + else + { + Console.WriteLine("Error: skipping depot {0} from app {1} with manifest {2}. could not get depot data", depot.DepotId, depot.AppId, depot.ManifestId); + continue; + } + cts.Token.ThrowIfCancellationRequested(); } - cts.Token.ThrowIfCancellationRequested(); - } + if (!Config.DepotLayout) + { - // If we're about to write all the files to the same directory, we will need to first de-duplicate any files by path - // This is in last-depot-wins order, from Steam or the list of depots supplied by the user - if (!string.IsNullOrWhiteSpace(Config.InstallDirectory) && depotsToDownload.Count > 0) - { - var claimedFileNames = new HashSet(); + if (!string.IsNullOrWhiteSpace(Config.InstallDirectory) && depotsToDownload.Count > 0) + { + var claimedFileNames = new HashSet(); - for (var i = depotsToDownload.Count - 1; i >= 0; i--) - { - // For each depot, remove all files from the list that have been claimed by a later depot - depotsToDownload[i].filteredFiles.RemoveAll(file => claimedFileNames.Contains(file.FileName)); + for (var i = depotsToDownload.Count - 1; i >= 0; i--) + { + // For each depot, remove all files from the list that have been claimed by a later depot + depotsToDownload[i].filteredFiles.RemoveAll(file => claimedFileNames.Contains(file.FileName)); - claimedFileNames.UnionWith(depotsToDownload[i].allFileNames); + claimedFileNames.UnionWith(depotsToDownload[i].allFileNames); + } + } } - } - foreach (var depotFileData in depotsToDownload) - { - await DownloadSteam3AsyncDepotFiles(cts, downloadCounter, depotFileData, allFileNamesAllDepots); - } - Ansi.Progress(Ansi.ProgressState.Hidden); + foreach (var depotFileData in depotsToDownload) + { + await DownloadSteam3AsyncDepotFiles(cts, downloadCounter, depotFileData, allFileNamesAllDepots); + } - Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", - downloadCounter.totalBytesCompressed, downloadCounter.totalBytesUncompressed, depots.Count); - } + Ansi.Progress(Ansi.ProgressState.Hidden); + Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", + Util.FormatSize((long)downloadCounter.totalBytesCompressed), Util.FormatSize((long)downloadCounter.totalBytesUncompressed), depots.Count); + } + catch (IOException ex) + { + Console.WriteLine("Failed to download steam app: {0}", ex.Message); + } + finally + { + downloadMonitor.Stop(); + } + } private static async Task ProcessDepotManifestAndFiles(CancellationTokenSource cts, DepotDownloadInfo depot, GlobalDownloadCounter downloadCounter) { var depotCounter = new DepotDownloadCounter(); @@ -785,6 +879,7 @@ namespace DepotDownloader if (manifestRequestCode == 0) { cts.Cancel(); + return null; } } @@ -972,7 +1067,7 @@ namespace DepotDownloader DepotConfigStore.Instance.InstalledManifestIDs[depot.DepotId] = depot.ManifestId; DepotConfigStore.Save(); - Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, depotCounter.depotBytesCompressed, depotCounter.depotBytesUncompressed); + Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, Util.FormatSize((long)depotCounter.depotBytesCompressed), Util.FormatSize((long)depotCounter.depotBytesUncompressed)); } private static void DownloadSteam3AsyncDepotFile( @@ -1004,6 +1099,20 @@ namespace DepotDownloader } List neededChunks; + // Check if existing file needs updating when leveraging bandwidth + // If outdated , delete to re-download + // Staging and updating blocks can be slower than fresh download on fast connections + if (Config.LeverageBandwidth && File.Exists(fileFinalPath)) + { + if ((oldManifestFile == null) || (!oldManifestFile.FileHash.SequenceEqual(file.FileHash))) + { + File.Delete(fileFinalPath); + oldProtoManifest = null; + Console.WriteLine("Re-downloading outdated file {0}", file.FileName); + } + + } + var fi = new FileInfo(fileFinalPath); var fileDidExist = fi.Exists; if (!fileDidExist) diff --git a/DepotDownloader/DepotDownloader.csproj b/DepotDownloader/DepotDownloader.csproj index e07aceca..1a959d65 100644 --- a/DepotDownloader/DepotDownloader.csproj +++ b/DepotDownloader/DepotDownloader.csproj @@ -11,7 +11,7 @@ ..\Icon\DepotDownloader.ico true true - true + true diff --git a/DepotDownloader/DownloadConfig.cs b/DepotDownloader/DownloadConfig.cs index 144853bf..51bfcc4f 100644 --- a/DepotDownloader/DownloadConfig.cs +++ b/DepotDownloader/DownloadConfig.cs @@ -32,5 +32,7 @@ namespace DepotDownloader public bool UseQrCode { get; set; } public bool SkipAppConfirmation { get; set; } + public bool DepotLayout { get; set; } + public bool LeverageBandwidth { get; set; } } } diff --git a/DepotDownloader/Program.cs b/DepotDownloader/Program.cs index 26b7f16f..b18191ac 100644 --- a/DepotDownloader/Program.cs +++ b/DepotDownloader/Program.cs @@ -135,6 +135,8 @@ namespace DepotDownloader } ContentDownloader.Config.InstallDirectory = GetParameter(args, "-dir"); + ContentDownloader.Config.DepotLayout = HasParameter(args, "-depot-layout"); + ContentDownloader.Config.LeverageBandwidth = HasParameter(args, "-leverage-bandwidth"); ContentDownloader.Config.VerifyAll = HasParameter(args, "-verify-all") || HasParameter(args, "-verify_all") || HasParameter(args, "-validate"); @@ -520,16 +522,19 @@ namespace DepotDownloader Console.WriteLine(" -no-mobile - prefer entering a 2FA code instead of prompting to accept in the Steam mobile app"); Console.WriteLine(); Console.WriteLine(" -dir - the directory in which to place downloaded files."); - Console.WriteLine(" -filelist - the name of a local file that contains a list of files to download (from the manifest)."); - Console.WriteLine(" prefix file path with `regex:` if you want to match with regex. each file path should be on their own line."); + Console.WriteLine(" -filelist - a list of files to download (from the manifest). Prefix file path with 'regex:' if you want to match with regex."); + Console.WriteLine(" -validate - Include checksum verification of files already downloaded"); Console.WriteLine(); - Console.WriteLine(" -validate - include checksum verification of files already downloaded"); Console.WriteLine(" -manifest-only - downloads a human readable manifest for any depots that would be downloaded."); Console.WriteLine(" -cellid <#> - the overridden CellID of the content server to download from."); + Console.WriteLine(" -max-servers <#> - maximum number of content servers to use. (default: 20)."); Console.WriteLine(" -max-downloads <#> - maximum number of chunks to download concurrently. (default: 8)."); Console.WriteLine(" -loginid <#> - a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently."); Console.WriteLine(" -use-lancache - forces downloads over the local network via a Lancache instance."); Console.WriteLine(); + Console.WriteLine(" -depot-layout - store each depot's files in separate folder named by depot ID.This can be useful for game development."); + Console.WriteLine(" -leverage-bandwidth - prefer network bandwidth over CPU usage by re-downloading changed files instead of patching them"); + Console.WriteLine(); Console.WriteLine(" -debug - enable verbose debug logging."); Console.WriteLine(" -V or --version - print version and runtime."); } diff --git a/DepotDownloader/Steam3Session.cs b/DepotDownloader/Steam3Session.cs index 9104f332..bb0ff363 100644 --- a/DepotDownloader/Steam3Session.cs +++ b/DepotDownloader/Steam3Session.cs @@ -242,19 +242,20 @@ namespace DepotDownloader if (DepotKeys.ContainsKey(depotId) || bAborted) return; - var depotKey = await steamApps.GetDepotDecryptionKey(depotId, appid); - - Console.WriteLine("Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result); - - if (depotKey.Result != EResult.OK) + await Util.ExecuteWithRetry(async () => { - return; - } + var depotKey = await steamApps.GetDepotDecryptionKey(depotId, appid); + Console.WriteLine($"Got depot key for {depotKey.DepotID} result: {depotKey.Result}"); - DepotKeys[depotKey.DepotID] = depotKey.DepotKey; - } + if (depotKey.Result != EResult.OK) + { + return; + } + DepotKeys[depotKey.DepotID] = depotKey.DepotKey; + }, 5); + } public async Task GetDepotManifestRequestCodeAsync(uint depotId, uint appId, ulong manifestId, string branch) { if (bAborted) diff --git a/DepotDownloader/Util.cs b/DepotDownloader/Util.cs index 2481b3d5..dcb1370c 100644 --- a/DepotDownloader/Util.cs +++ b/DepotDownloader/Util.cs @@ -15,6 +15,40 @@ namespace DepotDownloader { static class Util { + public static async Task ExecuteWithRetry(Func operation, int maxRetries = 3, int initialDelay = 1000) + { + for (var retryCount = 0; retryCount <= maxRetries; retryCount++) + { + try + { + await operation(); + return; + } + catch (Exception ex) when (retryCount < maxRetries) + { + var delay = initialDelay * (1 << retryCount); // Use bit shifting for more efficient power of 2 + Console.WriteLine($"Error: Attempt {retryCount + 1} of {maxRetries} failed: {ex.Message}"); + Console.WriteLine($"Retrying in {delay}ms..."); + await Task.Delay(delay); + } + } + throw new Exception($"Operation failed after {maxRetries} retries"); // Throw if all retries exhausted + } + + public static string FormatSize(long bytes) + { + string[] suffixes = { "B", "KB", "MB", "GB", "TB", "PB", "EB" }; + var counter = 0; + var number = (decimal)bytes; + const int kilo = 1024; + while (Math.Round(number / kilo) >= 1) + { + number /= kilo; + counter++; + } + return string.Format("{0:n1} {1}", number, suffixes[counter]); + } + public static string GetSteamOS() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -235,5 +269,6 @@ namespace DepotDownloader return output; } + } } diff --git a/README.md b/README.md index 810c9466..bc41b490 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,8 @@ Parameter | Description `-cellid <#>` | the overridden CellID of the content server to download from. `-max-downloads <#>` | maximum number of chunks to download concurrently. (default: 8). `-use-lancache` | forces downloads over the local network via a Lancache instance. +`-depot-layout` | download depots in a layout that is more suitable for game development. +`-leverage-bandwidth` | leverage bandwidth by re-downloading files that have changed. #### Other diff --git a/global.json b/global.json index 2bc13e80..f15a9592 100644 --- a/global.json +++ b/global.json @@ -3,4 +3,4 @@ "version": "9.0.100", "rollForward": "latestMinor" } -} +} \ No newline at end of file