Merge branch 'master' into master

pull/501/head
Giacomo Preciado 1 year ago committed by GitHub
commit 39e43a1bf9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -6,11 +6,16 @@ indent_style = space
charset = utf-8 charset = utf-8
end_of_line = lf end_of_line = lf
insert_final_newline = true insert_final_newline = true
trim_trailing_whitespace = true
# Code files # Code files
[*.{cs, csx, vb, vbx}] [*.{cs, csx, vb, vbx}]
indent_size = 4 indent_size = 4
# Github yaml files
[*.yml]
indent_size = 2
# XML project files # XML project files
[*.{csproj, vbproj, vcxproj, vcxproj.filters, proj, projitems, shproj}] [*.{csproj, vbproj, vcxproj, vcxproj.filters, proj, projitems, shproj}]
indent_size = 2 indent_size = 2
@ -95,7 +100,6 @@ dotnet_naming_rule.instance_fields_should_be_camel_case.style = instance_field_s
dotnet_naming_symbols.instance_fields.applicable_kinds = field dotnet_naming_symbols.instance_fields.applicable_kinds = field
dotnet_naming_style.instance_field_style.capitalization = camel_case dotnet_naming_style.instance_field_style.capitalization = camel_case
dotnet_naming_style.instance_field_style.required_prefix = _
# Locals and parameters are camelCase # Locals and parameters are camelCase
dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion
@ -124,26 +128,9 @@ dotnet_naming_symbols.all_members.applicable_kinds = *
dotnet_naming_style.pascal_case_style.capitalization = pascal_case dotnet_naming_style.pascal_case_style.capitalization = pascal_case
# Async methods should have "Async" suffix
dotnet_naming_rule.async_methods_end_in_async.symbols = any_async_methods
dotnet_naming_rule.async_methods_end_in_async.style = end_in_async
dotnet_naming_rule.async_methods_end_in_async.severity = warning
dotnet_naming_symbols.any_async_methods.applicable_kinds = method
dotnet_naming_symbols.any_async_methods.applicable_accessibilities = *
dotnet_naming_symbols.any_async_methods.required_modifiers = async
dotnet_naming_style.end_in_async.required_prefix =
dotnet_naming_style.end_in_async.required_suffix = Async
dotnet_naming_style.end_in_async.capitalization = pascal_case
dotnet_naming_style.end_in_async.word_separator =
# error RS2008: Enable analyzer release tracking for the analyzer project containing rule '{0}' # error RS2008: Enable analyzer release tracking for the analyzer project containing rule '{0}'
dotnet_diagnostic.RS2008.severity = none dotnet_diagnostic.RS2008.severity = none
# IDE0005: Remove unnecessary import
dotnet_diagnostic.IDE0005.severity = warning
# IDE0007: Use `var` instead of explicit type # IDE0007: Use `var` instead of explicit type
dotnet_diagnostic.IDE0007.severity = warning dotnet_diagnostic.IDE0007.severity = warning
@ -161,6 +148,11 @@ dotnet_diagnostic.IDE0044.severity = warning
# CSharp code style settings: # CSharp code style settings:
[*.cs] [*.cs]
# Require file header OR A source file contains a header that does not match the required text
file_header_template = This file is subject to the terms and conditions defined\nin file 'LICENSE', which is part of this source code package.
dotnet_diagnostic.IDE0073.severity = error
# Newline settings # Newline settings
csharp_new_line_before_open_brace = all csharp_new_line_before_open_brace = all
csharp_new_line_before_else = true csharp_new_line_before_else = true

2
.gitattributes vendored

@ -1,4 +1,4 @@
*.cs text eol=lf *.cs text eol=lf
*.csproj text eol=crlf *.csproj text eol=lf
*.config eol=lf *.config eol=lf
*.json eol=lf *.json eol=lf

@ -32,17 +32,24 @@ body:
validations: validations:
required: true required: true
- type: input - type: input
id: dotnet-version id: version
attributes: attributes:
label: Version label: Version
description: What version of DepotDownloader are you running on? description: What version of DepotDownloader are using?
validations:
required: true
- type: input
id: command
attributes:
label: Command
description: Specify the full command you used (except for username and password)
validations: validations:
required: true required: true
- type: textarea - type: textarea
id: logs id: logs
attributes: attributes:
label: Relevant log output label: Relevant log output
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. Run with `-debug` parameter to get additional output.
render: shell render: shell
- type: textarea - type: textarea
id: additional-info id: additional-info

@ -33,7 +33,7 @@ jobs:
dotnet-version: 8.0.x dotnet-version: 8.0.x
- name: Build - name: Build
run: dotnet publish DepotDownloader/DepotDownloader.csproj -c ${{ matrix.configuration }} -o artifacts run: dotnet publish DepotDownloader/DepotDownloader.csproj -c ${{ matrix.configuration }} -o artifacts /p:ContinuousIntegrationBuild=true
- name: Upload artifact - name: Upload artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
@ -126,3 +126,47 @@ jobs:
name: DepotDownloader-macos-arm64 name: DepotDownloader-macos-arm64
path: selfcontained-osx-arm64 path: selfcontained-osx-arm64
if-no-files-found: error if-no-files-found: error
release:
if: startsWith(github.ref, 'refs/tags/')
needs: build
runs-on: ubuntu-latest
steps:
- name: Download artifacts
uses: actions/download-artifact@v4
with:
path: artifacts
- name: Display artifacts folder structure
run: ls -Rl
working-directory: artifacts
- name: Create release files
run: |
set -eux
mkdir release
chmod +x artifacts/DepotDownloader-linux-x64/DepotDownloader
chmod +x artifacts/DepotDownloader-linux-arm/DepotDownloader
chmod +x artifacts/DepotDownloader-linux-arm64/DepotDownloader
chmod +x artifacts/DepotDownloader-macos-x64/DepotDownloader
chmod +x artifacts/DepotDownloader-macos-arm64/DepotDownloader
zip -9j release/DepotDownloader-framework.zip artifacts/DepotDownloader-framework/*
zip -9j release/DepotDownloader-windows-x64.zip artifacts/DepotDownloader-windows-x64/*
zip -9j release/DepotDownloader-windows-arm64.zip artifacts/DepotDownloader-windows-arm64/*
zip -9j release/DepotDownloader-linux-x64.zip artifacts/DepotDownloader-linux-x64/*
zip -9j release/DepotDownloader-linux-arm.zip artifacts/DepotDownloader-linux-arm/*
zip -9j release/DepotDownloader-linux-arm64.zip artifacts/DepotDownloader-linux-arm64/*
zip -9j release/DepotDownloader-macos-x64.zip artifacts/DepotDownloader-macos-x64/*
zip -9j release/DepotDownloader-macos-arm64.zip artifacts/DepotDownloader-macos-arm64/*
- name: Display structure of archived files
run: ls -Rl
working-directory: release
- name: Release
uses: softprops/action-gh-release@v2
with:
draft: true
files: release/*
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

@ -30,12 +30,12 @@ jobs:
dotnet add DepotDownloader/DepotDownloader.csproj package SteamKit2 --prerelease dotnet add DepotDownloader/DepotDownloader.csproj package SteamKit2 --prerelease
- name: Build - name: Build
run: dotnet publish DepotDownloader/DepotDownloader.csproj -c ${{ matrix.configuration }} -o artifacts run: dotnet publish DepotDownloader/DepotDownloader.csproj -c ${{ matrix.configuration }} -o artifacts /p:ContinuousIntegrationBuild=true
- name: Upload artifact - name: Upload artifact
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
if: matrix.configuration == 'Release' if: matrix.configuration == 'Release'
with: with:
name: DepotDownloader-${{ runner.os }} name: DepotDownloader-${{ matrix.runs-on }}
path: artifacts path: artifacts
if-no-files-found: error if-no-files-found: error

@ -0,0 +1,29 @@
name: WinGet submission on release
on:
workflow_dispatch:
release:
types: [published]
jobs:
winget:
name: Publish winget package
runs-on: windows-latest
steps:
- name: Submit package to Windows Package Manager Community Repository
run: |
$wingetPackage = "SteamRE.DepotDownloader"
$github = Invoke-RestMethod -uri "https://api.github.com/repos/SteamRE/DepotDownloader/releases"
$targetRelease = $github | Where-Object -Property name -match '^DepotDownloader' | Select -First 1
$assets = $targetRelease | Select -ExpandProperty assets -First 1
$zipX64Url = $assets | Where-Object -Property name -match 'DepotDownloader-windows-x64.zip' | Select -ExpandProperty browser_download_url
$zipArm64Url = $assets | Where-Object -Property name -match 'DepotDownloader-windows-arm64.zip' | Select -ExpandProperty browser_download_url
$ver = $targetRelease.tag_name -ireplace '^(DepotDownloader[ _])?v?'
# getting latest wingetcreate file
iwr https://aka.ms/wingetcreate/latest -OutFile wingetcreate.exe
# how to create a token: https://github.com/microsoft/winget-create?tab=readme-ov-file#github-personal-access-token-classic-permissions
.\wingetcreate.exe update $wingetPackage --submit --version $ver --urls "$zipX64Url" "$zipArm64Url" --token "${{ secrets.PT_WINGET }}"

@ -1,3 +1,6 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System; using System;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Collections.Generic; using System.Collections.Generic;

@ -0,0 +1,54 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System;
using Spectre.Console;
namespace DepotDownloader;
static class Ansi
{
// https://conemu.github.io/en/AnsiEscapeCodes.html#ConEmu_specific_OSC
// https://learn.microsoft.com/en-us/windows/terminal/tutorials/progress-bar-sequences
public enum ProgressState
{
Hidden = 0,
Default = 1,
Error = 2,
Indeterminate = 3,
Warning = 4,
}
const char ESC = (char)0x1B;
const char BEL = (char)0x07;
private static bool useProgress;
public static void Init()
{
if (Console.IsInputRedirected || Console.IsOutputRedirected)
{
return;
}
var (supportsAnsi, legacyConsole) = AnsiDetector.Detect(stdError: false, upgrade: true);
useProgress = supportsAnsi && !legacyConsole;
}
public static void Progress(ulong downloaded, ulong total)
{
var progress = (byte)MathF.Round(downloaded / (float)total * 100.0f);
Progress(ProgressState.Default, progress);
}
public static void Progress(ProgressState state, byte progress = 0)
{
if (!useProgress)
{
return;
}
Console.Write($"{ESC}]9;4;{(byte)state};{progress}{BEL}");
}
}

@ -0,0 +1,134 @@
// Copied from https://github.com/spectreconsole/spectre.console/blob/d79e6adc5f8e637fb35c88f987023ffda6707243/src/Spectre.Console/Internal/Backends/Ansi/AnsiDetector.cs
// MIT License - Copyright(c) 2020 Patrik Svensson, Phil Scott, Nils Andresen
// which is partially based on https://github.com/keqingrong/supports-ansi/blob/master/index.js
// <auto-generated/>
using System;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
namespace Spectre.Console;
internal static class AnsiDetector
{
private static readonly Regex[] _regexes =
[
new("^xterm"), // xterm, PuTTY, Mintty
new("^rxvt"), // RXVT
new("^eterm"), // Eterm
new("^screen"), // GNU screen, tmux
new("tmux"), // tmux
new("^vt100"), // DEC VT series
new("^vt102"), // DEC VT series
new("^vt220"), // DEC VT series
new("^vt320"), // DEC VT series
new("ansi"), // ANSI
new("scoansi"), // SCO ANSI
new("cygwin"), // Cygwin, MinGW
new("linux"), // Linux console
new("konsole"), // Konsole
new("bvterm"), // Bitvise SSH Client
new("^st-256color"), // Suckless Simple Terminal, st
new("alacritty"), // Alacritty
];
public static (bool SupportsAnsi, bool LegacyConsole) Detect(bool stdError, bool upgrade)
{
// Running on Windows?
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// Running under ConEmu?
var conEmu = Environment.GetEnvironmentVariable("ConEmuANSI");
if (!string.IsNullOrEmpty(conEmu) && conEmu.Equals("On", StringComparison.OrdinalIgnoreCase))
{
return (true, false);
}
var supportsAnsi = Windows.SupportsAnsi(upgrade, stdError, out var legacyConsole);
return (supportsAnsi, legacyConsole);
}
return DetectFromTerm();
}
private static (bool SupportsAnsi, bool LegacyConsole) DetectFromTerm()
{
// Check if the terminal is of type ANSI/VT100/xterm compatible.
var term = Environment.GetEnvironmentVariable("TERM");
if (!string.IsNullOrWhiteSpace(term))
{
if (_regexes.Any(regex => regex.IsMatch(term)))
{
return (true, false);
}
}
return (false, true);
}
private static class Windows
{
private const int STD_OUTPUT_HANDLE = -11;
private const int STD_ERROR_HANDLE = -12;
private const uint ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004;
private const uint DISABLE_NEWLINE_AUTO_RETURN = 0x0008;
[DllImport("kernel32.dll")]
private static extern bool GetConsoleMode(IntPtr hConsoleHandle, out uint lpMode);
[DllImport("kernel32.dll")]
private static extern bool SetConsoleMode(IntPtr hConsoleHandle, uint dwMode);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern IntPtr GetStdHandle(int nStdHandle);
[DllImport("kernel32.dll")]
public static extern uint GetLastError();
public static bool SupportsAnsi(bool upgrade, bool stdError, out bool isLegacy)
{
isLegacy = false;
try
{
var @out = GetStdHandle(stdError ? STD_ERROR_HANDLE : STD_OUTPUT_HANDLE);
if (!GetConsoleMode(@out, out var mode))
{
// Could not get console mode, try TERM (set in cygwin, WSL-Shell).
var (ansiFromTerm, legacyFromTerm) = DetectFromTerm();
isLegacy = ansiFromTerm ? legacyFromTerm : isLegacy;
return ansiFromTerm;
}
if ((mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) == 0)
{
isLegacy = true;
if (!upgrade)
{
return false;
}
// Try enable ANSI support.
mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN;
if (!SetConsoleMode(@out, mode))
{
// Enabling failed.
return false;
}
isLegacy = false;
}
return true;
}
catch
{
// All we know here is that we don't support ANSI.
return false;
}
}
}
}

@ -1,3 +1,6 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System; using System;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Collections.Generic; using System.Collections.Generic;

@ -1,4 +1,8 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System; using System;
using System.Buffers;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
@ -110,7 +114,7 @@ namespace DepotDownloader
IEnumerable<uint> licenseQuery; IEnumerable<uint> licenseQuery;
if (steam3.steamUser.SteamID.AccountType == EAccountType.AnonUser) if (steam3.steamUser.SteamID.AccountType == EAccountType.AnonUser)
{ {
licenseQuery = new List<uint> { 17906 }; licenseQuery = [17906];
} }
else else
{ {
@ -244,7 +248,7 @@ namespace DepotDownloader
byte[] manifest_bytes; byte[] manifest_bytes;
try try
{ {
manifest_bytes = CryptoHelper.SymmetricDecryptECB(input, appBetaPassword); manifest_bytes = Util.SymmetricDecryptECB(input, appBetaPassword);
} }
catch (Exception e) catch (Exception e)
{ {
@ -611,20 +615,23 @@ namespace DepotDownloader
private class GlobalDownloadCounter private class GlobalDownloadCounter
{ {
public ulong TotalBytesCompressed; public ulong completeDownloadSize;
public ulong TotalBytesUncompressed; public ulong totalBytesCompressed;
public ulong totalBytesUncompressed;
} }
private class DepotDownloadCounter private class DepotDownloadCounter
{ {
public ulong CompleteDownloadSize; public ulong completeDownloadSize;
public ulong SizeDownloaded; public ulong sizeDownloaded;
public ulong DepotBytesCompressed; public ulong depotBytesCompressed;
public ulong DepotBytesUncompressed; public ulong depotBytesUncompressed;
} }
private static async Task DownloadSteam3Async(List<DepotDownloadInfo> depots) private static async Task DownloadSteam3Async(List<DepotDownloadInfo> depots)
{ {
Ansi.Progress(Ansi.ProgressState.Indeterminate);
var cts = new CancellationTokenSource(); var cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts; cdnPool.ExhaustedToken = cts;
@ -635,7 +642,7 @@ namespace DepotDownloader
// First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
foreach (var depot in depots) foreach (var depot in depots)
{ {
var depotFileData = await ProcessDepotManifestAndFiles(cts, depot); var depotFileData = await ProcessDepotManifestAndFiles(cts, depot, downloadCounter);
if (depotFileData != null) if (depotFileData != null)
{ {
@ -666,11 +673,13 @@ namespace DepotDownloader
await DownloadSteam3AsyncDepotFiles(cts, downloadCounter, depotFileData, allFileNamesAllDepots); await DownloadSteam3AsyncDepotFiles(cts, downloadCounter, depotFileData, allFileNamesAllDepots);
} }
Ansi.Progress(Ansi.ProgressState.Hidden);
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots",
downloadCounter.TotalBytesCompressed, downloadCounter.TotalBytesUncompressed, depots.Count); downloadCounter.totalBytesCompressed, downloadCounter.totalBytesUncompressed, depots.Count);
} }
private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts, DepotDownloadInfo depot) private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts, DepotDownloadInfo depot, GlobalDownloadCounter downloadCounter)
{ {
var depotCounter = new DepotDownloadCounter(); var depotCounter = new DepotDownloadCounter();
@ -752,7 +761,7 @@ namespace DepotDownloader
} }
else else
{ {
Console.Write("Downloading depot manifest..."); Console.Write("Downloading depot manifest... ");
DepotManifest depotManifest = null; DepotManifest depotManifest = null;
ulong manifestRequestCode = 0; ulong manifestRequestCode = 0;
@ -768,6 +777,13 @@ namespace DepotDownloader
{ {
connection = cdnPool.GetConnection(cts.Token); connection = cdnPool.GetConnection(cts.Token);
string cdnToken = null;
if (steam3.CDNAuthTokens.TryGetValue((depot.DepotId, connection.Host), out var authTokenCallbackPromise))
{
var result = await authTokenCallbackPromise.Task;
cdnToken = result.Token;
}
var now = DateTime.Now; var now = DateTime.Now;
// In order to download this manifest, we need the current manifest request code // In order to download this manifest, we need the current manifest request code
@ -801,7 +817,8 @@ namespace DepotDownloader
manifestRequestCode, manifestRequestCode,
connection, connection,
depot.DepotKey, depot.DepotKey,
cdnPool.ProxyServer).ConfigureAwait(false); cdnPool.ProxyServer,
cdnToken).ConfigureAwait(false);
cdnPool.ReturnConnection(connection); cdnPool.ReturnConnection(connection);
} }
@ -811,11 +828,21 @@ namespace DepotDownloader
} }
catch (SteamKitWebRequestException e) catch (SteamKitWebRequestException e)
{ {
// If the CDN returned 403, attempt to get a cdn auth if we didn't yet
if (e.StatusCode == HttpStatusCode.Forbidden && !steam3.CDNAuthTokens.ContainsKey((depot.DepotId, connection.Host)))
{
await steam3.RequestCDNAuthToken(depot.AppId, depot.DepotId, connection);
cdnPool.ReturnConnection(connection);
continue;
}
cdnPool.ReturnBrokenConnection(connection); cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{ {
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.DepotId, depot.ManifestId); Console.WriteLine("Encountered {2} for depot manifest {0} {1}. Aborting.", depot.DepotId, depot.ManifestId, (int)e.StatusCode);
break; break;
} }
@ -890,7 +917,8 @@ namespace DepotDownloader
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
depotCounter.CompleteDownloadSize += file.TotalSize; downloadCounter.completeDownloadSize += file.TotalSize;
depotCounter.completeDownloadSize += file.TotalSize;
} }
}); });
@ -919,7 +947,7 @@ namespace DepotDownloader
await Util.InvokeAsync( await Util.InvokeAsync(
files.Select(file => new Func<Task>(async () => files.Select(file => new Func<Task>(async () =>
await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, depotFilesData, file, networkChunkQueue)))), await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, downloadCounter, depotFilesData, file, networkChunkQueue)))),
maxDegreeOfParallelism: Config.MaxDownloads maxDegreeOfParallelism: Config.MaxDownloads
); );
@ -962,11 +990,12 @@ namespace DepotDownloader
DepotConfigStore.Instance.InstalledManifestIDs[depot.DepotId] = depot.ManifestId; DepotConfigStore.Instance.InstalledManifestIDs[depot.DepotId] = depot.ManifestId;
DepotConfigStore.Save(); DepotConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed); Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, depotCounter.depotBytesCompressed, depotCounter.depotBytesUncompressed);
} }
private static void DownloadSteam3AsyncDepotFile( private static void DownloadSteam3AsyncDepotFile(
CancellationTokenSource cts, CancellationTokenSource cts,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData, DepotFilesData depotFilesData,
ProtoManifest.FileData file, ProtoManifest.FileData file,
ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue) ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue)
@ -1053,10 +1082,7 @@ namespace DepotDownloader
{ {
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
var tmp = new byte[match.OldChunk.UncompressedLength]; var adler = Util.AdlerHash(fsOld, (int)match.OldChunk.UncompressedLength);
fsOld.Read(tmp, 0, tmp.Length);
var adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum)) if (!adler.SequenceEqual(match.OldChunk.Checksum))
{ {
neededChunks.Add(match.NewChunk); neededChunks.Add(match.NewChunk);
@ -1125,8 +1151,13 @@ namespace DepotDownloader
{ {
lock (depotDownloadCounter) lock (depotDownloadCounter)
{ {
depotDownloadCounter.SizeDownloaded += file.TotalSize; depotDownloadCounter.sizeDownloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", (depotDownloadCounter.SizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath); Console.WriteLine("{0,6:#00.00}% {1}", (depotDownloadCounter.sizeDownloaded / (float)depotDownloadCounter.completeDownloadSize) * 100.0f, fileFinalPath);
}
lock (downloadCounter)
{
downloadCounter.completeDownloadSize -= file.TotalSize;
} }
return; return;
@ -1135,7 +1166,12 @@ namespace DepotDownloader
var sizeOnDisk = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum()); var sizeOnDisk = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum());
lock (depotDownloadCounter) lock (depotDownloadCounter)
{ {
depotDownloadCounter.SizeDownloaded += sizeOnDisk; depotDownloadCounter.sizeDownloaded += sizeOnDisk;
}
lock (downloadCounter)
{
downloadCounter.completeDownloadSize -= sizeOnDisk;
} }
} }
@ -1175,91 +1211,122 @@ namespace DepotDownloader
var depot = depotFilesData.depotDownloadInfo; var depot = depotFilesData.depotDownloadInfo;
var depotDownloadCounter = depotFilesData.depotCounter; var depotDownloadCounter = depotFilesData.depotCounter;
var chunkID = Util.EncodeHexString(chunk.ChunkID); var chunkID = Convert.ToHexString(chunk.ChunkID).ToLowerInvariant();
var data = new DepotManifest.ChunkData var data = new DepotManifest.ChunkData
{ {
ChunkID = chunk.ChunkID, ChunkID = chunk.ChunkID,
Checksum = chunk.Checksum, Checksum = BitConverter.ToUInt32(chunk.Checksum),
Offset = chunk.Offset, Offset = chunk.Offset,
CompressedLength = chunk.CompressedLength, CompressedLength = chunk.CompressedLength,
UncompressedLength = chunk.UncompressedLength UncompressedLength = chunk.UncompressedLength
}; };
DepotChunk chunkData = null; var written = 0;
var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)data.UncompressedLength);
do try
{ {
cts.Token.ThrowIfCancellationRequested(); do
{
cts.Token.ThrowIfCancellationRequested();
Server connection = null; Server connection = null;
try try
{ {
connection = cdnPool.GetConnection(cts.Token); connection = cdnPool.GetConnection(cts.Token);
DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); string cdnToken = null;
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync( if (steam3.CDNAuthTokens.TryGetValue((depot.DepotId, connection.Host), out var authTokenCallbackPromise))
depot.DepotId, {
data, var result = await authTokenCallbackPromise.Task;
connection, cdnToken = result.Token;
depot.DepotKey, }
cdnPool.ProxyServer).ConfigureAwait(false);
cdnPool.ReturnConnection(connection); DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
} written = await cdnPool.CDNClient.DownloadDepotChunkAsync(
catch (TaskCanceledException) depot.DepotId,
{ data,
Console.WriteLine("Connection timeout downloading chunk {0}", chunkID); connection,
} chunkBuffer,
catch (SteamKitWebRequestException e) depot.DepotKey,
{ cdnPool.ProxyServer,
cdnPool.ReturnBrokenConnection(connection); cdnToken).ConfigureAwait(false);
cdnPool.ReturnConnection(connection);
break;
}
catch (TaskCanceledException)
{
Console.WriteLine("Connection timeout downloading chunk {0}", chunkID);
}
catch (SteamKitWebRequestException e)
{
// If the CDN returned 403, attempt to get a cdn auth if we didn't yet,
// if auth task already exists, make sure it didn't complete yet, so that it gets awaited above
if (e.StatusCode == HttpStatusCode.Forbidden &&
(!steam3.CDNAuthTokens.TryGetValue((depot.DepotId, connection.Host), out var authTokenCallbackPromise) || !authTokenCallbackPromise.Task.IsCompleted))
{
await steam3.RequestCDNAuthToken(depot.AppId, depot.DepotId, connection);
cdnPool.ReturnConnection(connection);
continue;
}
cdnPool.ReturnBrokenConnection(connection);
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
{
Console.WriteLine("Encountered {1} for chunk {0}. Aborting.", chunkID, (int)e.StatusCode);
break;
}
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
}
catch (OperationCanceledException)
{ {
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
break; break;
} }
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
} while (written == 0);
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode); if (written == 0)
}
catch (OperationCanceledException)
{ {
break; Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.DepotId);
cts.Cancel();
} }
catch (Exception e)
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
}
} while (chunkData == null);
if (chunkData == null) // Throw the cancellation exception if requested so that this task is marked failed
{ cts.Token.ThrowIfCancellationRequested();
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.DepotId);
cts.Cancel();
}
// Throw the cancellation exception if requested so that this task is marked failed try
cts.Token.ThrowIfCancellationRequested(); {
await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false);
try if (fileStreamData.fileStream == null)
{ {
await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false); var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open);
}
if (fileStreamData.fileStream == null) fileStreamData.fileStream.Seek((long)data.Offset, SeekOrigin.Begin);
await fileStreamData.fileStream.WriteAsync(chunkBuffer.AsMemory(0, written), cts.Token);
}
finally
{ {
var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName); fileStreamData.fileLock.Release();
fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open);
} }
fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin);
await fileStreamData.fileStream.WriteAsync(chunkData.Data.AsMemory(0, chunkData.Data.Length), cts.Token);
} }
finally finally
{ {
fileStreamData.fileLock.Release(); ArrayPool<byte>.Shared.Return(chunkBuffer);
} }
var remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload); var remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload);
@ -1272,22 +1339,24 @@ namespace DepotDownloader
ulong sizeDownloaded = 0; ulong sizeDownloaded = 0;
lock (depotDownloadCounter) lock (depotDownloadCounter)
{ {
sizeDownloaded = depotDownloadCounter.SizeDownloaded + (ulong)chunkData.Data.Length; sizeDownloaded = depotDownloadCounter.sizeDownloaded + (ulong)written;
depotDownloadCounter.SizeDownloaded = sizeDownloaded; depotDownloadCounter.sizeDownloaded = sizeDownloaded;
depotDownloadCounter.DepotBytesCompressed += chunk.CompressedLength; depotDownloadCounter.depotBytesCompressed += chunk.CompressedLength;
depotDownloadCounter.DepotBytesUncompressed += chunk.UncompressedLength; depotDownloadCounter.depotBytesUncompressed += chunk.UncompressedLength;
} }
lock (downloadCounter) lock (downloadCounter)
{ {
downloadCounter.TotalBytesCompressed += chunk.CompressedLength; downloadCounter.totalBytesCompressed += chunk.CompressedLength;
downloadCounter.TotalBytesUncompressed += chunk.UncompressedLength; downloadCounter.totalBytesUncompressed += chunk.UncompressedLength;
Ansi.Progress(downloadCounter.totalBytesUncompressed, downloadCounter.completeDownloadSize);
} }
if (remainingChunks == 0) if (remainingChunks == 0)
{ {
var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName); var fileFinalPath = Path.Combine(depot.InstallDir, file.FileName);
Console.WriteLine("{0,6:#00.00}% {1}", (sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath); Console.WriteLine("{0,6:#00.00}% {1}", (sizeDownloaded / (float)depotDownloadCounter.completeDownloadSize) * 100.0f, fileFinalPath);
} }
} }

@ -1,3 +1,6 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;

@ -4,17 +4,28 @@
<TargetFramework>net8.0</TargetFramework> <TargetFramework>net8.0</TargetFramework>
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild> <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
<RollForward>LatestMajor</RollForward> <RollForward>LatestMajor</RollForward>
<Version>2.6.0</Version> <Version>2.7.1</Version>
<Description>Steam Downloading Utility</Description> <Description>Steam Downloading Utility</Description>
<Authors>SteamRE Team</Authors> <Authors>SteamRE Team</Authors>
<Copyright>Copyright © SteamRE Team 2024</Copyright> <Copyright>Copyright © SteamRE Team 2024</Copyright>
<ApplicationIcon>..\Icon\DepotDownloader.ico</ApplicationIcon> <ApplicationIcon>..\Icon\DepotDownloader.ico</ApplicationIcon>
<GenerateDocumentationFile>true</GenerateDocumentationFile> <Deterministic>true</Deterministic>
<TreatWarningsAsErrors Condition="'$(ContinuousIntegrationBuild)' == 'true'">true</TreatWarningsAsErrors>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<None Include="..\LICENSE" Link="LICENSE">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Windows.CsWin32" Version="0.3.106">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="protobuf-net" Version="3.2.30" /> <PackageReference Include="protobuf-net" Version="3.2.30" />
<PackageReference Include="QRCoder" Version="1.4.3" /> <PackageReference Include="QRCoder" Version="1.6.0" />
<PackageReference Include="SteamKit2" Version="3.0.0-Alpha.1" /> <PackageReference Include="SteamKit2" Version="3.0.0-Beta.2" />
</ItemGroup> </ItemGroup>
</Project> </Project>

@ -1,4 +1,7 @@
using System.Collections.Generic; // This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System.Collections.Generic;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
namespace DepotDownloader namespace DepotDownloader

@ -1,3 +1,6 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System.IO; using System.IO;
using System.Net.Http; using System.Net.Http;
using System.Net.Http.Headers; using System.Net.Http.Headers;

@ -1,3 +1,6 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System; using System;
using System.Diagnostics.Tracing; using System.Diagnostics.Tracing;
using System.Text; using System.Text;

@ -0,0 +1,2 @@
GetConsoleProcessList
MessageBox

@ -1,126 +1,52 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System; using System;
using System.IO;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using System.Runtime.Versioning;
namespace DepotDownloader namespace DepotDownloader
{ {
static class PlatformUtilities static class PlatformUtilities
{ {
private const int ModeExecuteOwner = 0x0040; public static void SetExecutable(string path, bool value)
private const int ModeExecuteGroup = 0x0008;
private const int ModeExecuteOther = 0x0001;
private const int ModeExecute = ModeExecuteOwner | ModeExecuteGroup | ModeExecuteOther;
[StructLayout(LayoutKind.Explicit, Size = 144)]
private readonly struct StatLinuxX64
{
[FieldOffset(24)] public readonly uint st_mode;
}
[StructLayout(LayoutKind.Explicit, Size = 104)]
private readonly struct StatLinuxArm32
{
[FieldOffset(16)] public readonly uint st_mode;
}
[StructLayout(LayoutKind.Explicit, Size = 128)]
private readonly struct StatLinuxArm64
{
[FieldOffset(16)] public readonly uint st_mode;
}
[StructLayout(LayoutKind.Explicit, Size = 144)]
private readonly struct StatOSX
{
[FieldOffset(4)] public readonly ushort st_mode;
}
[DllImport("libc", EntryPoint = "__xstat", SetLastError = true)]
private static extern int statLinuxX64(int version, string path, out StatLinuxX64 statLinux);
[DllImport("libc", EntryPoint = "__xstat", SetLastError = true)]
private static extern int statLinuxArm32(int version, string path, out StatLinuxArm32 statLinux);
[DllImport("libc", EntryPoint = "__xstat", SetLastError = true)]
private static extern int statLinuxArm64(int version, string path, out StatLinuxArm64 statLinux);
[DllImport("libc", EntryPoint = "stat", SetLastError = true)]
private static extern int statOSX(string path, out StatOSX stat);
[DllImport("libc", EntryPoint = "stat$INODE64", SetLastError = true)]
private static extern int statOSXCompat(string path, out StatOSX stat);
[DllImport("libc", SetLastError = true)]
private static extern int chmod(string path, uint mode);
[DllImport("libc", CallingConvention = CallingConvention.Cdecl, SetLastError = true)]
private static extern IntPtr strerror(int errno);
private static void ThrowIf(int i)
{ {
if (i == -1) if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{ {
var errno = Marshal.GetLastWin32Error(); return;
throw new Exception(Marshal.PtrToStringAnsi(strerror(errno)));
} }
}
private static uint GetFileMode(string path) const UnixFileMode ModeExecute = UnixFileMode.UserExecute | UnixFileMode.GroupExecute | UnixFileMode.OtherExecute;
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) var mode = File.GetUnixFileMode(path);
{ var hasExecuteMask = (mode & ModeExecute) == ModeExecute;
switch (RuntimeInformation.ProcessArchitecture) if (hasExecuteMask != value)
{
case Architecture.X64:
{
ThrowIf(statLinuxX64(1, path, out var stat));
return stat.st_mode;
}
case Architecture.Arm:
{
ThrowIf(statLinuxArm32(3, path, out var stat));
return stat.st_mode;
}
case Architecture.Arm64:
{
ThrowIf(statLinuxArm64(0, path, out var stat));
return stat.st_mode;
}
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{ {
switch (RuntimeInformation.ProcessArchitecture) File.SetUnixFileMode(path, value
{ ? mode | ModeExecute
case Architecture.X64: : mode & ~ModeExecute);
{
ThrowIf(statOSXCompat(path, out var stat));
return stat.st_mode;
}
case Architecture.Arm64:
{
ThrowIf(statOSX(path, out var stat));
return stat.st_mode;
}
}
} }
throw new PlatformNotSupportedException();
} }
public static void SetExecutable(string path, bool value) [SupportedOSPlatform("windows5.0")]
public static void VerifyConsoleLaunch()
{ {
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // Reference: https://devblogs.microsoft.com/oldnewthing/20160125-00/?p=92922
var processList = new uint[2];
var processCount = Windows.Win32.PInvoke.GetConsoleProcessList(processList);
if (processCount != 1)
{ {
return; return;
} }
var mode = GetFileMode(path); _ = Windows.Win32.PInvoke.MessageBox(
var hasExecuteMask = (mode & ModeExecute) == ModeExecute; Windows.Win32.Foundation.HWND.Null,
if (hasExecuteMask != value) "Depot Downloader is a console application; there is no GUI.\n\nIf you do not pass any command line parameters, it prints usage info and exits.\n\nYou must use this from a terminal/console.",
{ "Depot Downloader",
ThrowIf(chmod(path, (uint)(value Windows.Win32.UI.WindowsAndMessaging.MESSAGEBOX_STYLE.MB_OK | Windows.Win32.UI.WindowsAndMessaging.MESSAGEBOX_STYLE.MB_ICONWARNING
? mode | ModeExecute );
: mode & ~ModeExecute)));
}
} }
} }
} }

@ -1,3 +1,6 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
@ -13,27 +16,40 @@ namespace DepotDownloader
{ {
class Program class Program
{ {
static int Main(string[] args) static async Task<int> Main(string[] args)
=> MainAsync(args).GetAwaiter().GetResult();
internal static readonly char[] newLineCharacters = ['\n', '\r'];
static async Task<int> MainAsync(string[] args)
{ {
if (args.Length == 0) if (args.Length == 0)
{ {
PrintVersion();
PrintUsage(); PrintUsage();
return 1;
if (OperatingSystem.IsWindowsVersionAtLeast(5, 0))
{
PlatformUtilities.VerifyConsoleLaunch();
}
return 0;
} }
Ansi.Init();
DebugLog.Enabled = false; DebugLog.Enabled = false;
AccountSettingsStore.LoadFromFile("account.config"); AccountSettingsStore.LoadFromFile("account.config");
#region Common Options #region Common Options
// Not using HasParameter because it is case insensitive
if (args.Length == 1 && (args[0] == "-V" || args[0] == "--version"))
{
PrintVersion(true);
return 0;
}
if (HasParameter(args, "-debug")) if (HasParameter(args, "-debug"))
{ {
PrintVersion(true);
DebugLog.Enabled = true; DebugLog.Enabled = true;
DebugLog.AddListener((category, message) => DebugLog.AddListener((category, message) =>
{ {
@ -41,9 +57,6 @@ namespace DepotDownloader
}); });
var httpEventListener = new HttpDiagnosticEventListener(); var httpEventListener = new HttpDiagnosticEventListener();
DebugLog.WriteLine("DepotDownloader", "Version: {0}", Assembly.GetExecutingAssembly().GetName().Version);
DebugLog.WriteLine("DepotDownloader", "Runtime: {0}", RuntimeInformation.FrameworkDescription);
} }
var username = GetParameter<string>(args, "-username") ?? GetParameter<string>(args, "-user"); var username = GetParameter<string>(args, "-username") ?? GetParameter<string>(args, "-user");
@ -69,15 +82,19 @@ namespace DepotDownloader
try try
{ {
var fileListData = await File.ReadAllTextAsync(fileList);
var files = fileListData.Split(newLineCharacters, StringSplitOptions.RemoveEmptyEntries);
ContentDownloader.Config.UsingFileList = true; ContentDownloader.Config.UsingFileList = true;
ContentDownloader.Config.FilesToDownload = new HashSet<string>(StringComparer.OrdinalIgnoreCase); ContentDownloader.Config.FilesToDownload = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
ContentDownloader.Config.FilesToDownloadRegex = []; ContentDownloader.Config.FilesToDownloadRegex = [];
var files = await File.ReadAllLinesAsync(fileList);
foreach (var fileEntry in files) foreach (var fileEntry in files)
{ {
if (string.IsNullOrWhiteSpace(fileEntry))
{
continue;
}
if (fileEntry.StartsWith(RegexPrefix)) if (fileEntry.StartsWith(RegexPrefix))
{ {
var rgx = new Regex(fileEntry[RegexPrefix.Length..], RegexOptions.Compiled | RegexOptions.IgnoreCase); var rgx = new Regex(fileEntry[RegexPrefix.Length..], RegexOptions.Compiled | RegexOptions.IgnoreCase);
@ -375,46 +392,60 @@ namespace DepotDownloader
static void PrintUsage() static void PrintUsage()
{ {
// Do not use tabs to align parameters here because tab size may differ
Console.WriteLine(); Console.WriteLine();
Console.WriteLine("Usage - downloading one or all depots for an app:"); Console.WriteLine("Usage: downloading one or all depots for an app:");
Console.WriteLine("\tdepotdownloader -app <id> [-depot <id> [-manifest <id>]]"); Console.WriteLine(" depotdownloader -app <id> [-depot <id> [-manifest <id>]]");
Console.WriteLine("\t\t[-username <username> [-password <password>]] [other options]"); Console.WriteLine(" [-username <username> [-password <password>]] [other options]");
Console.WriteLine(); Console.WriteLine();
Console.WriteLine("Usage - downloading a workshop item using pubfile id"); Console.WriteLine("Usage: downloading a workshop item using pubfile id");
Console.WriteLine("\tdepotdownloader -app <id> -pubfile <id> [-username <username> [-password <password>]]"); Console.WriteLine(" depotdownloader -app <id> -pubfile <id> [-username <username> [-password <password>]]");
Console.WriteLine("Usage - downloading a workshop item using ugc id"); Console.WriteLine("Usage: downloading a workshop item using ugc id");
Console.WriteLine("\tdepotdownloader -app <id> -ugc <id> [-username <username> [-password <password>]]"); Console.WriteLine(" depotdownloader -app <id> -ugc <id> [-username <username> [-password <password>]]");
Console.WriteLine(); Console.WriteLine();
Console.WriteLine("Parameters:"); Console.WriteLine("Parameters:");
Console.WriteLine("\t-app <#>\t\t\t\t- the AppID to download."); Console.WriteLine(" -app <#> - the AppID to download.");
Console.WriteLine("\t-depot <#>\t\t\t\t- the DepotID to download."); Console.WriteLine(" -depot <#> - the DepotID to download.");
Console.WriteLine("\t-manifest <id>\t\t\t- manifest id of content to download (requires -depot, default: current for branch)."); Console.WriteLine(" -manifest <id> - manifest id of content to download (requires -depot, default: current for branch).");
Console.WriteLine($"\t-beta <branchname>\t\t\t- download from specified branch if available (default: {ContentDownloader.DEFAULT_BRANCH})."); Console.WriteLine($" -beta <branchname> - download from specified branch if available (default: {ContentDownloader.DEFAULT_BRANCH}).");
Console.WriteLine("\t-betapassword <pass>\t\t- branch password if applicable."); Console.WriteLine(" -betapassword <pass> - branch password if applicable.");
Console.WriteLine("\t-all-archs\t\t\t- download all architecture-specific depots when -app is used."); Console.WriteLine(" -all-platforms - downloads all platform-specific depots when -app is used.");
Console.WriteLine("\t-all-platforms\t\t\t- downloads all platform-specific depots when -app is used."); Console.WriteLine(" -all-archs - download all architecture-specific depots when -app is used.");
Console.WriteLine("\t-os <os>\t\t\t\t- the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)"); Console.WriteLine(" -os <os> - the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)");
Console.WriteLine("\t-osarch <arch>\t\t\t\t- the architecture for which to download the game (32 or 64, default: the host's architecture)"); Console.WriteLine(" -osarch <arch> - the architecture for which to download the game (32 or 64, default: the host's architecture)");
Console.WriteLine("\t-all-languages\t\t\t\t- download all language-specific depots when -app is used."); Console.WriteLine(" -all-languages - download all language-specific depots when -app is used.");
Console.WriteLine("\t-language <lang>\t\t\t\t- the language for which to download the game (default: english)"); Console.WriteLine(" -language <lang> - the language for which to download the game (default: english)");
Console.WriteLine("\t-lowviolence\t\t\t\t- download low violence depots when -app is used."); Console.WriteLine(" -lowviolence - download low violence depots when -app is used.");
Console.WriteLine(); Console.WriteLine();
Console.WriteLine("\t-ugc <#>\t\t\t\t- the UGC ID to download."); Console.WriteLine(" -ugc <#> - the UGC ID to download.");
Console.WriteLine("\t-pubfile <#>\t\t\t- the PublishedFileId to download. (Will automatically resolve to UGC id)"); Console.WriteLine(" -pubfile <#> - the PublishedFileId to download. (Will automatically resolve to UGC id)");
Console.WriteLine(); Console.WriteLine();
Console.WriteLine("\t-username <user>\t\t- the username of the account to login to for restricted content."); Console.WriteLine(" -username <user> - the username of the account to login to for restricted content.");
Console.WriteLine("\t-password <pass>\t\t- the password of the account to login to for restricted content."); Console.WriteLine(" -password <pass> - the password of the account to login to for restricted content.");
Console.WriteLine("\t-remember-password\t\t- if set, remember the password for subsequent logins of this user. (Use -username <username> -remember-password as login credentials)"); Console.WriteLine(" -remember-password - if set, remember the password for subsequent logins of this user. (Use -username <username> -remember-password as login credentials)");
Console.WriteLine(); Console.WriteLine();
Console.WriteLine("\t-dir <installdir>\t\t- the directory in which to place downloaded files."); Console.WriteLine(" -dir <installdir> - the directory in which to place downloaded files.");
Console.WriteLine("\t-filelist <file.txt>\t- a list of files to download (from the manifest). Prefix file path with 'regex:' if you want to match with regex."); Console.WriteLine(" -filelist <file.txt> - a list of files to download (from the manifest). Prefix file path with 'regex:' if you want to match with regex.");
Console.WriteLine("\t-validate\t\t\t\t- Include checksum verification of files already downloaded"); Console.WriteLine(" -validate - Include checksum verification of files already downloaded");
Console.WriteLine(); Console.WriteLine();
Console.WriteLine("\t-manifest-only\t\t\t- downloads a human readable manifest for any depots that would be downloaded."); Console.WriteLine(" -manifest-only - downloads a human readable manifest for any depots that would be downloaded.");
Console.WriteLine("\t-cellid <#>\t\t\t\t- the overridden CellID of the content server to download from."); Console.WriteLine(" -cellid <#> - the overridden CellID of the content server to download from.");
Console.WriteLine("\t-max-servers <#>\t\t- maximum number of content servers to use. (default: 20)."); Console.WriteLine(" -max-servers <#> - maximum number of content servers to use. (default: 20).");
Console.WriteLine("\t-max-downloads <#>\t\t- maximum number of chunks to download concurrently. (default: 8)."); Console.WriteLine(" -max-downloads <#> - maximum number of chunks to download concurrently. (default: 8).");
Console.WriteLine("\t-loginid <#>\t\t- a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently."); Console.WriteLine(" -loginid <#> - a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently.");
}
static void PrintVersion(bool printExtra = false)
{
var version = typeof(Program).Assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>().InformationalVersion;
Console.WriteLine($"DepotDownloader v{version}");
if (!printExtra)
{
return;
}
Console.WriteLine($"Runtime: {RuntimeInformation.FrameworkDescription} on {RuntimeInformation.OSDescription}");
} }
} }
} }

@ -1,3 +1,6 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
@ -76,7 +79,7 @@ namespace DepotDownloader
public ChunkData(DepotManifest.ChunkData sourceChunk) public ChunkData(DepotManifest.ChunkData sourceChunk)
{ {
ChunkID = sourceChunk.ChunkID; ChunkID = sourceChunk.ChunkID;
Checksum = sourceChunk.Checksum; Checksum = BitConverter.GetBytes(sourceChunk.Checksum);
Offset = sourceChunk.Offset; Offset = sourceChunk.Offset;
CompressedLength = sourceChunk.CompressedLength; CompressedLength = sourceChunk.CompressedLength;
UncompressedLength = sourceChunk.UncompressedLength; UncompressedLength = sourceChunk.UncompressedLength;

@ -1,4 +1,8 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System; using System;
using System.Collections.Concurrent;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.ObjectModel; using System.Collections.ObjectModel;
using System.Linq; using System.Linq;
@ -7,6 +11,7 @@ using System.Threading.Tasks;
using QRCoder; using QRCoder;
using SteamKit2; using SteamKit2;
using SteamKit2.Authentication; using SteamKit2.Authentication;
using SteamKit2.CDN;
using SteamKit2.Internal; using SteamKit2.Internal;
namespace DepotDownloader namespace DepotDownloader
@ -24,6 +29,7 @@ namespace DepotDownloader
public Dictionary<uint, ulong> AppTokens { get; } = []; public Dictionary<uint, ulong> AppTokens { get; } = [];
public Dictionary<uint, ulong> PackageTokens { get; } = []; public Dictionary<uint, ulong> PackageTokens { get; } = [];
public Dictionary<uint, byte[]> DepotKeys { get; } = []; public Dictionary<uint, byte[]> DepotKeys { get; } = [];
public ConcurrentDictionary<(uint, string), TaskCompletionSource<SteamApps.CDNAuthTokenCallback>> CDNAuthTokens { get; } = [];
public Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo> AppInfo { get; } = []; public Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo> AppInfo { get; } = [];
public Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo> PackageInfo { get; } = []; public Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo> PackageInfo { get; } = [];
public Dictionary<string, byte[]> AppBetaPasswords { get; } = []; public Dictionary<string, byte[]> AppBetaPasswords { get; } = [];
@ -282,6 +288,30 @@ namespace DepotDownloader
return requestCode; return requestCode;
} }
public async Task RequestCDNAuthToken(uint appid, uint depotid, Server server)
{
var cdnKey = (depotid, server.Host);
var completion = new TaskCompletionSource<SteamApps.CDNAuthTokenCallback>();
if (bAborted || !CDNAuthTokens.TryAdd(cdnKey, completion))
{
return;
}
DebugLog.WriteLine(nameof(Steam3Session), $"Requesting CDN auth token for {server.Host}");
var cdnAuth = await steamApps.GetCDNAuthToken(appid, depotid, server.Host);
Console.WriteLine($"Got CDN auth token for {server.Host} result: {cdnAuth.Result} (expires {cdnAuth.Expiration})");
if (cdnAuth.Result != EResult.OK)
{
return;
}
completion.TrySetResult(cdnAuth);
}
public void CheckAppBetaPassword(uint appid, string password) public void CheckAppBetaPassword(uint appid, string password)
{ {
var completed = false; var completed = false;
@ -403,6 +433,8 @@ namespace DepotDownloader
bIsConnectionRecovery = false; bIsConnectionRecovery = false;
steamClient.Disconnect(); steamClient.Disconnect();
Ansi.Progress(Ansi.ProgressState.Hidden);
// flush callbacks until our disconnected event // flush callbacks until our disconnected event
while (!bDidDisconnect) while (!bDidDisconnect)
{ {

@ -1,8 +1,12 @@
// This file is subject to the terms and conditions defined
// in file 'LICENSE', which is part of this source code package.
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -27,6 +31,12 @@ namespace DepotDownloader
return "linux"; return "linux";
} }
if (RuntimeInformation.IsOSPlatform(OSPlatform.FreeBSD))
{
// Return linux as freebsd steam client doesn't exist yet
return "linux";
}
return "unknown"; return "unknown";
} }
@ -71,26 +81,12 @@ namespace DepotDownloader
public static List<ProtoManifest.ChunkData> ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata) public static List<ProtoManifest.ChunkData> ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata)
{ {
var neededChunks = new List<ProtoManifest.ChunkData>(); var neededChunks = new List<ProtoManifest.ChunkData>();
int read;
foreach (var data in chunkdata) foreach (var data in chunkdata)
{ {
var chunk = new byte[data.UncompressedLength];
fs.Seek((long)data.Offset, SeekOrigin.Begin); fs.Seek((long)data.Offset, SeekOrigin.Begin);
read = fs.Read(chunk, 0, (int)data.UncompressedLength);
byte[] tempchunk; var adler = AdlerHash(fs, (int)data.UncompressedLength);
if (read < data.UncompressedLength)
{
tempchunk = new byte[read];
Array.Copy(chunk, 0, tempchunk, 0, read);
}
else
{
tempchunk = chunk;
}
var adler = AdlerHash(tempchunk);
if (!adler.SequenceEqual(data.Checksum)) if (!adler.SequenceEqual(data.Checksum))
{ {
neededChunks.Add(data); neededChunks.Add(data);
@ -100,12 +96,14 @@ namespace DepotDownloader
return neededChunks; return neededChunks;
} }
public static byte[] AdlerHash(byte[] input) public static byte[] AdlerHash(Stream stream, int length)
{ {
uint a = 0, b = 0; uint a = 0, b = 0;
for (var i = 0; i < input.Length; i++) for (var i = 0; i < length; i++)
{ {
a = (a + input[i]) % 65521; var c = (uint)stream.ReadByte();
a = (a + c) % 65521;
b = (b + a) % 65521; b = (b + a) % 65521;
} }
@ -126,11 +124,21 @@ namespace DepotDownloader
return bytes; return bytes;
} }
public static string EncodeHexString(byte[] input) /// <summary>
/// Decrypts using AES/ECB/PKCS7
/// </summary>
public static byte[] SymmetricDecryptECB(byte[] input, byte[] key)
{ {
return input.Aggregate(new StringBuilder(), using var aes = Aes.Create();
(sb, v) => sb.Append(v.ToString("x2")) aes.BlockSize = 128;
).ToString(); aes.KeySize = 256;
aes.Mode = CipherMode.ECB;
aes.Padding = PaddingMode.PKCS7;
using var aesTransform = aes.CreateDecryptor(key, null);
var output = aesTransform.TransformFinalBlock(input, 0, input.Length);
return output;
} }
public static async Task InvokeAsync(IEnumerable<Func<Task>> taskFactories, int maxDegreeOfParallelism) public static async Task InvokeAsync(IEnumerable<Func<Task>> taskFactories, int maxDegreeOfParallelism)

@ -3,58 +3,92 @@ DepotDownloader
Steam depot downloader utilizing the SteamKit2 library. Supports .NET 8.0 Steam depot downloader utilizing the SteamKit2 library. Supports .NET 8.0
### Downloading one or all depots for an app This program must be run from a console, it has no GUI.
## Installation
### Directly from GitHub
Download a binary from [the releases page](https://github.com/SteamRE/DepotDownloader/releases/latest).
### via Windows Package Manager CLI (aka winget)
On Windows, [winget](https://github.com/microsoft/winget-cli) users can download and install
the latest Terminal release by installing the `SteamRE.DepotDownloader`
package:
```powershell
winget install --exact --id SteamRE.DepotDownloader
``` ```
dotnet DepotDownloader.dll -app <id> [-depot <id> [-manifest <id>]]
[-username <username> [-password <password>]] [other options] ### via Homebrew
On macOS, [Homebrew](https://brew.sh) users can download and install that latest release by running the following commands:
```shell
brew tap steamre/tools
brew install depotdownloader
``` ```
For example: `dotnet DepotDownloader.dll -app 730 -depot 731 -manifest 7617088375292372759` ## Usage
### Downloading a workshop item using pubfile id ### Downloading one or all depots for an app
```powershell
./DepotDownloader -app <id> [-depot <id> [-manifest <id>]]
[-username <username> [-password <password>]] [other options]
``` ```
dotnet DepotDownloader.dll -app <id> -pubfile <id> [-username <username> [-password <password>]]
For example: `./DepotDownloader -app 730 -depot 731 -manifest 7617088375292372759`
By default it will use anonymous account ([view which apps are available on it here](https://steamdb.info/sub/17906/)).
To use your account, specify the `-username <username>` parameter. Password will be asked interactively if you do
not use specify the `-password` parameter.
### Downloading a workshop item using pubfile id
```powershell
./DepotDownloader -app <id> -pubfile <id> [-username <username> [-password <password>]]
``` ```
For example: `dotnet DepotDownloader.dll -app 730 -pubfile 1885082371` For example: `./DepotDownloader -app 730 -pubfile 1885082371`
### Downloading a workshop item using ugc id ### Downloading a workshop item using ugc id
``` ```powershell
dotnet DepotDownloader.dll -app <id> -ugc <id> [-username <username> [-password <password>]] ./DepotDownloader -app <id> -ugc <id> [-username <username> [-password <password>]]
``` ```
For example: `dotnet DepotDownloader.dll -app 730 -ugc 770604181014286929` For example: `./DepotDownloader -app 730 -ugc 770604181014286929`
## Parameters ## Parameters
Parameter | Description Parameter | Description
--------- | ----------- ----------------------- | -----------
-app \<#> | the AppID to download. `-app <#>` | the AppID to download.
-depot \<#> | the DepotID to download. `-depot <#>` | the DepotID to download.
-manifest \<id> | manifest id of content to download (requires -depot, default: current for branch). `-manifest <id>` | manifest id of content to download (requires `-depot`, default: current for branch).
-ugc \<#> | the UGC ID to download. `-ugc <#>` | the UGC ID to download.
-beta \<branchname> | download from specified branch if available (default: Public). `-beta <branchname>` | download from specified branch if available (default: Public).
-betapassword \<pass> | branch password if applicable. `-betapassword <pass>` | branch password if applicable.
-all-platforms | downloads all platform-specific depots when -app is used. `-all-platforms` | downloads all platform-specific depots when `-app` is used.
-os \<os> | the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on) `-os <os>` | the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)
-osarch \<arch> | the architecture for which to download the game (32 or 64, default: the host's architecture) `-osarch <arch>` | the architecture for which to download the game (32 or 64, default: the host's architecture)
-all-archs | download all architecture-specific depots when -app is used. `-all-archs` | download all architecture-specific depots when `-app` is used.
-all-languages | download all language-specific depots when -app is used. `-all-languages` | download all language-specific depots when `-app` is used.
-language \<lang> | the language for which to download the game (default: english) `-language <lang>` | the language for which to download the game (default: english)
-lowviolence | download low violence depots when -app is used. `-lowviolence` | download low violence depots when `-app` is used.
-pubfile \<#> | the PublishedFileId to download. (Will automatically resolve to UGC id) `-pubfile <#>` | the PublishedFileId to download. (Will automatically resolve to UGC id)
-username \<user> | the username of the account to login to for restricted content. `-username <user>` | the username of the account to login to for restricted content.
-password \<pass> | the password of the account to login to for restricted content. `-password <pass>` | the password of the account to login to for restricted content.
-remember-password | if set, remember the password for subsequent logins of this user. (Use -username <username> -remember-password as login credentials) `-remember-password` | if set, remember the password for subsequent logins of this user. (Use `-username <username> -remember-password` as login credentials)
-dir \<installdir> | the directory in which to place downloaded files. `-dir <installdir>` | the directory in which to place downloaded files.
-filelist \<file.txt> | a list of files to download (from the manifest). Prefix file path with `regex:` if you want to match with regex. `-filelist <file.txt>` | a list of files to download (from the manifest). Prefix file path with `regex:` if you want to match with regex.
-validate | Include checksum verification of files already downloaded `-validate` | Include checksum verification of files already downloaded
-manifest-only | downloads a human readable manifest for any depots that would be downloaded. `-manifest-only` | downloads a human readable manifest for any depots that would be downloaded.
-cellid \<#> | the overridden CellID of the content server to download from. `-cellid <#>` | the overridden CellID of the content server to download from.
-max-servers \<#> | maximum number of content servers to use. (default: 20). `-max-servers <#>` | maximum number of content servers to use. (default: 20).
-max-downloads \<#> | maximum number of chunks to download concurrently. (default: 8). `-max-downloads <#>` | maximum number of chunks to download concurrently. (default: 8).
-loginid \<#> | a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently. `-loginid <#>` | a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently.
`-V` or `--version` | print version and runtime
## Frequently Asked Questions ## Frequently Asked Questions

Loading…
Cancel
Save