diff --git a/.gitattributes b/.gitattributes
index ff739929..48b55300 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,4 +1,4 @@
*.cs text eol=lf
*.csproj text eol=lf
*.config eol=lf
-*.json eol=lf
+*.json eol=lf
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index ce528650..5d12f982 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,7 @@
# mstest test results
TestResults
-
+.history
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
@@ -118,4 +118,6 @@ cryptopp/
# misc
Thumbs.db
+.vscode
+DepotDownloader/Properties
launchSettings.json
diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader/ContentDownloader.cs
index a59a69fb..b73065de 100644
--- a/DepotDownloader/ContentDownloader.cs
+++ b/DepotDownloader/ContentDownloader.cs
@@ -79,9 +79,8 @@ namespace DepotDownloader
else
{
Directory.CreateDirectory(Config.InstallDirectory);
-
- installDir = Config.InstallDirectory;
-
+ installDir = Config.DepotLayout ? Path.Combine(Config.InstallDirectory, depotId.ToString()) : Config.InstallDirectory;
+ Directory.CreateDirectory(installDir);
Directory.CreateDirectory(Path.Combine(installDir, CONFIG_DIR));
Directory.CreateDirectory(Path.Combine(installDir, STAGING_DIR));
}
@@ -479,7 +478,9 @@ namespace DepotDownloader
else
{
var contentName = GetAppName(appId);
- throw new ContentDownloaderException(string.Format("App {0} ({1}) is not available from this account.", appId, contentName));
+ Console.WriteLine(string.Format("Error: App {0} ({1}) is not available from this account.", appId, contentName));
+ // Skip this app and continue processing other apps in the list since we don't have access to download it
+ return;
}
}
@@ -644,8 +645,7 @@ namespace DepotDownloader
Console.WriteLine("Error: Unable to create install directories!");
return null;
}
-
- // For depots that are proxied through depotfromapp, we still need to resolve the proxy app id, unless the app is freetodownload
+ // For depots that are proxied through depotfromapp, we still need to resolve the proxy app id
var containingAppId = appId;
var proxyAppId = GetSteam3DepotProxyAppId(depotId, appId);
if (proxyAppId != INVALID_APP_ID)
@@ -699,6 +699,80 @@ namespace DepotDownloader
public ulong depotBytesUncompressed;
}
+ ///
+ /// Monitors and reports download progress at regular intervals
+ ///
+ private sealed class DownloadMonitor : IDisposable
+ {
+ private readonly GlobalDownloadCounter globalDownloadCounter;
+ private readonly int reportIntervalMs;
+ private Thread monitorThread;
+ private volatile bool running = false;
+ private long lastDownloadSize = 0;
+ private readonly AutoResetEvent threadWait = new(false);
+ private readonly object lockObject = new();
+
+ public DownloadMonitor(
+ GlobalDownloadCounter globalDownloadCounter,
+ int reportIntervalMs = 60000)
+ {
+ this.globalDownloadCounter = globalDownloadCounter;
+ this.reportIntervalMs = reportIntervalMs;
+ }
+
+ public void Start()
+ {
+ if (monitorThread != null)
+ return;
+
+ running = true;
+ monitorThread = new Thread(ReportAlive);
+ monitorThread.Start();
+ }
+
+ public void Stop()
+ {
+ if (!running)
+ return;
+
+ running = false;
+ threadWait.Set();
+ monitorThread?.Join();
+ monitorThread = null;
+ }
+
+ private void ReportAlive()
+ {
+ try
+ {
+ while (running)
+ {
+ ulong currentDownloadedBytes;
+ lock (lockObject)
+ {
+ currentDownloadedBytes = globalDownloadCounter.totalBytesUncompressed;
+ }
+
+ if (currentDownloadedBytes > (ulong)Interlocked.Read(ref lastDownloadSize))
+ {
+ Interlocked.Exchange(ref lastDownloadSize, (long)currentDownloadedBytes);
+ Console.WriteLine("Downloaded : {0}", Util.FormatSize((long)currentDownloadedBytes));
+ }
+ threadWait.WaitOne(reportIntervalMs);
+ }
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine($"Error in download monitor: {ex.Message}");
+ }
+ }
+
+ public void Dispose()
+ {
+ Stop();
+ threadWait.Dispose();
+ }
+ }
private static async Task DownloadSteam3Async(List depots)
{
Ansi.Progress(Ansi.ProgressState.Indeterminate);
@@ -706,50 +780,70 @@ namespace DepotDownloader
await cdnPool.UpdateServerList();
var cts = new CancellationTokenSource();
+
var downloadCounter = new GlobalDownloadCounter();
+ var downloadMonitor = new DownloadMonitor(downloadCounter);
var depotsToDownload = new List(depots.Count);
var allFileNamesAllDepots = new HashSet();
- // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
- foreach (var depot in depots)
+ try
{
- var depotFileData = await ProcessDepotManifestAndFiles(cts, depot, downloadCounter);
-
- if (depotFileData != null)
+ downloadMonitor.Start();
+ // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
+ foreach (var depot in depots)
{
- depotsToDownload.Add(depotFileData);
- allFileNamesAllDepots.UnionWith(depotFileData.allFileNames);
+ var depotFileData = await ProcessDepotManifestAndFiles(cts, depot, downloadCounter);
+
+ if (depotFileData != null)
+ {
+ depotsToDownload.Add(depotFileData);
+ allFileNamesAllDepots.UnionWith(depotFileData.allFileNames);
+ }
+ else
+ {
+ Console.WriteLine("Error: skipping depot {0} from app {1} with manifest {2}. could not get depot data", depot.DepotId, depot.AppId, depot.ManifestId);
+ continue;
+ }
+ cts.Token.ThrowIfCancellationRequested();
}
- cts.Token.ThrowIfCancellationRequested();
- }
+ if (!Config.DepotLayout)
+ {
- // If we're about to write all the files to the same directory, we will need to first de-duplicate any files by path
- // This is in last-depot-wins order, from Steam or the list of depots supplied by the user
- if (!string.IsNullOrWhiteSpace(Config.InstallDirectory) && depotsToDownload.Count > 0)
- {
- var claimedFileNames = new HashSet();
+ if (!string.IsNullOrWhiteSpace(Config.InstallDirectory) && depotsToDownload.Count > 0)
+ {
+ var claimedFileNames = new HashSet();
- for (var i = depotsToDownload.Count - 1; i >= 0; i--)
- {
- // For each depot, remove all files from the list that have been claimed by a later depot
- depotsToDownload[i].filteredFiles.RemoveAll(file => claimedFileNames.Contains(file.FileName));
+ for (var i = depotsToDownload.Count - 1; i >= 0; i--)
+ {
+ // For each depot, remove all files from the list that have been claimed by a later depot
+ depotsToDownload[i].filteredFiles.RemoveAll(file => claimedFileNames.Contains(file.FileName));
- claimedFileNames.UnionWith(depotsToDownload[i].allFileNames);
+ claimedFileNames.UnionWith(depotsToDownload[i].allFileNames);
+ }
+ }
}
- }
- foreach (var depotFileData in depotsToDownload)
- {
- await DownloadSteam3AsyncDepotFiles(cts, downloadCounter, depotFileData, allFileNamesAllDepots);
- }
- Ansi.Progress(Ansi.ProgressState.Hidden);
+ foreach (var depotFileData in depotsToDownload)
+ {
+ await DownloadSteam3AsyncDepotFiles(cts, downloadCounter, depotFileData, allFileNamesAllDepots);
+ }
- Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots",
- downloadCounter.totalBytesCompressed, downloadCounter.totalBytesUncompressed, depots.Count);
- }
+ Ansi.Progress(Ansi.ProgressState.Hidden);
+ Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots",
+ Util.FormatSize((long)downloadCounter.totalBytesCompressed), Util.FormatSize((long)downloadCounter.totalBytesUncompressed), depots.Count);
+ }
+ catch (IOException ex)
+ {
+ Console.WriteLine("Failed to download steam app: {0}", ex.Message);
+ }
+ finally
+ {
+ downloadMonitor.Stop();
+ }
+ }
private static async Task ProcessDepotManifestAndFiles(CancellationTokenSource cts, DepotDownloadInfo depot, GlobalDownloadCounter downloadCounter)
{
var depotCounter = new DepotDownloadCounter();
@@ -829,6 +923,7 @@ namespace DepotDownloader
if (manifestRequestCode == 0)
{
cts.Cancel();
+ return null;
}
}
@@ -1016,7 +1111,7 @@ namespace DepotDownloader
DepotConfigStore.Instance.InstalledManifestIDs[depot.DepotId] = depot.ManifestId;
DepotConfigStore.Save();
- Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, depotCounter.depotBytesCompressed, depotCounter.depotBytesUncompressed);
+ Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, Util.FormatSize((long)depotCounter.depotBytesCompressed), Util.FormatSize((long)depotCounter.depotBytesUncompressed));
}
private static void DownloadSteam3AsyncDepotFile(
@@ -1048,6 +1143,20 @@ namespace DepotDownloader
}
List neededChunks;
+ // Check if existing file needs updating when leveraging bandwidth
+ // If outdated , delete to re-download
+ // Staging and updating blocks can be slower than fresh download on fast connections
+ if (Config.LeverageBandwidth && File.Exists(fileFinalPath))
+ {
+ if ((oldManifestFile == null) || (!oldManifestFile.FileHash.SequenceEqual(file.FileHash)))
+ {
+ File.Delete(fileFinalPath);
+ oldProtoManifest = null;
+ Console.WriteLine("Re-downloading outdated file {0}", file.FileName);
+ }
+
+ }
+
var fi = new FileInfo(fileFinalPath);
var fileDidExist = fi.Exists;
if (!fileDidExist)
diff --git a/DepotDownloader/DepotDownloader.csproj b/DepotDownloader/DepotDownloader.csproj
index 9cf5dde1..722ad75e 100644
--- a/DepotDownloader/DepotDownloader.csproj
+++ b/DepotDownloader/DepotDownloader.csproj
@@ -11,7 +11,7 @@
..\Icon\DepotDownloader.ico
true
true
- true
+ true
diff --git a/DepotDownloader/DownloadConfig.cs b/DepotDownloader/DownloadConfig.cs
index 144853bf..51bfcc4f 100644
--- a/DepotDownloader/DownloadConfig.cs
+++ b/DepotDownloader/DownloadConfig.cs
@@ -32,5 +32,7 @@ namespace DepotDownloader
public bool UseQrCode { get; set; }
public bool SkipAppConfirmation { get; set; }
+ public bool DepotLayout { get; set; }
+ public bool LeverageBandwidth { get; set; }
}
}
diff --git a/DepotDownloader/Program.cs b/DepotDownloader/Program.cs
index 26b7f16f..b18191ac 100644
--- a/DepotDownloader/Program.cs
+++ b/DepotDownloader/Program.cs
@@ -135,6 +135,8 @@ namespace DepotDownloader
}
ContentDownloader.Config.InstallDirectory = GetParameter(args, "-dir");
+ ContentDownloader.Config.DepotLayout = HasParameter(args, "-depot-layout");
+ ContentDownloader.Config.LeverageBandwidth = HasParameter(args, "-leverage-bandwidth");
ContentDownloader.Config.VerifyAll = HasParameter(args, "-verify-all") || HasParameter(args, "-verify_all") || HasParameter(args, "-validate");
@@ -520,16 +522,19 @@ namespace DepotDownloader
Console.WriteLine(" -no-mobile - prefer entering a 2FA code instead of prompting to accept in the Steam mobile app");
Console.WriteLine();
Console.WriteLine(" -dir - the directory in which to place downloaded files.");
- Console.WriteLine(" -filelist - the name of a local file that contains a list of files to download (from the manifest).");
- Console.WriteLine(" prefix file path with `regex:` if you want to match with regex. each file path should be on their own line.");
+ Console.WriteLine(" -filelist - a list of files to download (from the manifest). Prefix file path with 'regex:' if you want to match with regex.");
+ Console.WriteLine(" -validate - Include checksum verification of files already downloaded");
Console.WriteLine();
- Console.WriteLine(" -validate - include checksum verification of files already downloaded");
Console.WriteLine(" -manifest-only - downloads a human readable manifest for any depots that would be downloaded.");
Console.WriteLine(" -cellid <#> - the overridden CellID of the content server to download from.");
+ Console.WriteLine(" -max-servers <#> - maximum number of content servers to use. (default: 20).");
Console.WriteLine(" -max-downloads <#> - maximum number of chunks to download concurrently. (default: 8).");
Console.WriteLine(" -loginid <#> - a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently.");
Console.WriteLine(" -use-lancache - forces downloads over the local network via a Lancache instance.");
Console.WriteLine();
+ Console.WriteLine(" -depot-layout - store each depot's files in separate folder named by depot ID.This can be useful for game development.");
+ Console.WriteLine(" -leverage-bandwidth - prefer network bandwidth over CPU usage by re-downloading changed files instead of patching them");
+ Console.WriteLine();
Console.WriteLine(" -debug - enable verbose debug logging.");
Console.WriteLine(" -V or --version - print version and runtime.");
}
diff --git a/DepotDownloader/Steam3Session.cs b/DepotDownloader/Steam3Session.cs
index bff4f3ca..60c71da5 100644
--- a/DepotDownloader/Steam3Session.cs
+++ b/DepotDownloader/Steam3Session.cs
@@ -242,19 +242,20 @@ namespace DepotDownloader
if (DepotKeys.ContainsKey(depotId) || bAborted)
return;
- var depotKey = await steamApps.GetDepotDecryptionKey(depotId, appid);
-
- Console.WriteLine("Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result);
-
- if (depotKey.Result != EResult.OK)
+ await Util.ExecuteWithRetry(async () =>
{
- return;
- }
+ var depotKey = await steamApps.GetDepotDecryptionKey(depotId, appid);
+ Console.WriteLine($"Got depot key for {depotKey.DepotID} result: {depotKey.Result}");
- DepotKeys[depotKey.DepotID] = depotKey.DepotKey;
- }
+ if (depotKey.Result != EResult.OK)
+ {
+ return;
+ }
+ DepotKeys[depotKey.DepotID] = depotKey.DepotKey;
+ }, 5);
+ }
public async Task GetDepotManifestRequestCodeAsync(uint depotId, uint appId, ulong manifestId, string branch)
{
if (bAborted)
diff --git a/DepotDownloader/Util.cs b/DepotDownloader/Util.cs
index 2481b3d5..dcb1370c 100644
--- a/DepotDownloader/Util.cs
+++ b/DepotDownloader/Util.cs
@@ -15,6 +15,40 @@ namespace DepotDownloader
{
static class Util
{
+ public static async Task ExecuteWithRetry(Func operation, int maxRetries = 3, int initialDelay = 1000)
+ {
+ for (var retryCount = 0; retryCount <= maxRetries; retryCount++)
+ {
+ try
+ {
+ await operation();
+ return;
+ }
+ catch (Exception ex) when (retryCount < maxRetries)
+ {
+ var delay = initialDelay * (1 << retryCount); // Use bit shifting for more efficient power of 2
+ Console.WriteLine($"Error: Attempt {retryCount + 1} of {maxRetries} failed: {ex.Message}");
+ Console.WriteLine($"Retrying in {delay}ms...");
+ await Task.Delay(delay);
+ }
+ }
+ throw new Exception($"Operation failed after {maxRetries} retries"); // Throw if all retries exhausted
+ }
+
+ public static string FormatSize(long bytes)
+ {
+ string[] suffixes = { "B", "KB", "MB", "GB", "TB", "PB", "EB" };
+ var counter = 0;
+ var number = (decimal)bytes;
+ const int kilo = 1024;
+ while (Math.Round(number / kilo) >= 1)
+ {
+ number /= kilo;
+ counter++;
+ }
+ return string.Format("{0:n1} {1}", number, suffixes[counter]);
+ }
+
public static string GetSteamOS()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
@@ -235,5 +269,6 @@ namespace DepotDownloader
return output;
}
+
}
}
diff --git a/README.md b/README.md
index 810c9466..bc41b490 100644
--- a/README.md
+++ b/README.md
@@ -102,6 +102,8 @@ Parameter | Description
`-cellid <#>` | the overridden CellID of the content server to download from.
`-max-downloads <#>` | maximum number of chunks to download concurrently. (default: 8).
`-use-lancache` | forces downloads over the local network via a Lancache instance.
+`-depot-layout` | download depots in a layout that is more suitable for game development.
+`-leverage-bandwidth` | leverage bandwidth by re-downloading files that have changed.
#### Other
diff --git a/global.json b/global.json
index 2bc13e80..f15a9592 100644
--- a/global.json
+++ b/global.json
@@ -3,4 +3,4 @@
"version": "9.0.100",
"rollForward": "latestMinor"
}
-}
+}
\ No newline at end of file