diff --git a/.editorconfig b/.editorconfig index f202f057..d05495c2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,7 +1,230 @@ -; EditorConfig: http://EditorConfig.org - +# top-most EditorConfig file root = true [*] indent_style = space -indent_size = 4 \ No newline at end of file +charset = utf-8 +end_of_line = lf +insert_final_newline = true + +# Code files +[*.{cs, csx, vb, vbx}] +indent_size = 4 + +# XML project files +[*.{csproj, vbproj, vcxproj, vcxproj.filters, proj, projitems, shproj}] +indent_size = 2 + +# XML config files +[*.{props, targets, ruleset, config, nuspec, resx, vsixmanifest, vsct}] +indent_size = 2 + +# Dotnet code style settings: +[*.{cs, vb}] + +# IDE0055: Fix formatting +dotnet_diagnostic.IDE0055.severity = warning + +# Sort using and Import directives with System.* appearing first +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = false +# Avoid "this." and "Me." if not necessary +dotnet_style_qualification_for_field = false:refactoring +dotnet_style_qualification_for_property = false:refactoring +dotnet_style_qualification_for_method = false:refactoring +dotnet_style_qualification_for_event = false:refactoring + +# Use language keywords instead of framework type names for type references +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion + +# Suggest more modern language features when available +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion + +# Non-private static fields are PascalCase +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.symbols = non_private_static_fields +dotnet_naming_rule.non_private_static_fields_should_be_pascal_case.style = non_private_static_field_style + +dotnet_naming_symbols.non_private_static_fields.applicable_kinds = field +dotnet_naming_symbols.non_private_static_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected +dotnet_naming_symbols.non_private_static_fields.required_modifiers = static + +dotnet_naming_style.non_private_static_field_style.capitalization = pascal_case + +# Non-private readonly fields are PascalCase +dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.symbols = non_private_readonly_fields +dotnet_naming_rule.non_private_readonly_fields_should_be_pascal_case.style = non_private_readonly_field_style + +dotnet_naming_symbols.non_private_readonly_fields.applicable_kinds = field +dotnet_naming_symbols.non_private_readonly_fields.applicable_accessibilities = public, protected, internal, protected_internal, private_protected +dotnet_naming_symbols.non_private_readonly_fields.required_modifiers = readonly + +dotnet_naming_style.non_private_readonly_field_style.capitalization = pascal_case + +# Constants are PascalCase +dotnet_naming_rule.constants_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constants_should_be_pascal_case.symbols = constants +dotnet_naming_rule.constants_should_be_pascal_case.style = constant_style + +dotnet_naming_symbols.constants.applicable_kinds = field, local +dotnet_naming_symbols.constants.required_modifiers = const + +dotnet_naming_style.constant_style.capitalization = pascal_case + +# Static readonly fields are PascalCase +dotnet_naming_rule.static_fields_should_be_camel_case.severity = suggestion +dotnet_naming_rule.static_fields_should_be_camel_case.symbols = static_fields +dotnet_naming_rule.static_fields_should_be_camel_case.style = static_field_style + +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.required_modifiers = static, readonly + +dotnet_naming_style.static_field_style.capitalization = pascal_case + +# Instance fields are camelCase and start with _ +dotnet_naming_rule.instance_fields_should_be_camel_case.severity = suggestion +dotnet_naming_rule.instance_fields_should_be_camel_case.symbols = instance_fields +dotnet_naming_rule.instance_fields_should_be_camel_case.style = instance_field_style + +dotnet_naming_symbols.instance_fields.applicable_kinds = field + +dotnet_naming_style.instance_field_style.capitalization = camel_case +dotnet_naming_style.instance_field_style.required_prefix = _ + +# Locals and parameters are camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style + +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local + +dotnet_naming_style.camel_case_style.capitalization = camel_case + +# Local functions are PascalCase +dotnet_naming_rule.local_functions_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.local_functions_should_be_pascal_case.symbols = local_functions +dotnet_naming_rule.local_functions_should_be_pascal_case.style = local_function_style + +dotnet_naming_symbols.local_functions.applicable_kinds = local_function + +dotnet_naming_style.local_function_style.capitalization = pascal_case + +# By default, name items with PascalCase +dotnet_naming_rule.members_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.members_should_be_pascal_case.symbols = all_members +dotnet_naming_rule.members_should_be_pascal_case.style = pascal_case_style + +dotnet_naming_symbols.all_members.applicable_kinds = * + +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# Async methods should have "Async" suffix +dotnet_naming_rule.async_methods_end_in_async.symbols = any_async_methods +dotnet_naming_rule.async_methods_end_in_async.style = end_in_async +dotnet_naming_rule.async_methods_end_in_async.severity = warning + +dotnet_naming_symbols.any_async_methods.applicable_kinds = method +dotnet_naming_symbols.any_async_methods.applicable_accessibilities = * +dotnet_naming_symbols.any_async_methods.required_modifiers = async + +dotnet_naming_style.end_in_async.required_prefix = +dotnet_naming_style.end_in_async.required_suffix = Async +dotnet_naming_style.end_in_async.capitalization = pascal_case +dotnet_naming_style.end_in_async.word_separator = + +# error RS2008: Enable analyzer release tracking for the analyzer project containing rule '{0}' +dotnet_diagnostic.RS2008.severity = none + +# IDE0005: Remove unnecessary import +dotnet_diagnostic.IDE0005.severity = warning + +# IDE0007: Use `var` instead of explicit type +dotnet_diagnostic.IDE0007.severity = warning + +# IDE0035: Remove unreachable code +dotnet_diagnostic.IDE0035.severity = warning + +# IDE0036: Order modifiers +dotnet_diagnostic.IDE0036.severity = warning + +# IDE0043: Format string contains invalid placeholder +dotnet_diagnostic.IDE0043.severity = warning + +# IDE0044: Make field readonly +dotnet_diagnostic.IDE0044.severity = warning + +# CSharp code style settings: +[*.cs] +# Newline settings +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +# csharp_new_line_before_members_in_object_initializers = true TODO seems like Rider/ReSharper has the value inverted, uncomment when its fixed +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = false +csharp_indent_switch_labels = true +csharp_indent_labels = flush_left + +# Prefer "var" everywhere +csharp_style_var_for_built_in_types = true:suggestion +csharp_style_var_when_type_is_apparent = true:suggestion +csharp_style_var_elsewhere = true:suggestion + +# Prefer method-like constructs to have a block body +csharp_style_expression_bodied_methods = false:none +csharp_style_expression_bodied_constructors = false:none +csharp_style_expression_bodied_operators = false:none + +# Prefer property-like constructs to have an expression-body +csharp_style_expression_bodied_properties = true:none +csharp_style_expression_bodied_indexers = true:none +csharp_style_expression_bodied_accessors = true:none + +# Suggest more modern language features when available +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Space preferences +csharp_space_after_cast = false +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_after_semicolon_in_for_statement = true +csharp_space_around_binary_operators = before_and_after +csharp_space_around_declaration_statements = do_not_ignore +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_between_square_brackets = false + +# Blocks are allowed +csharp_prefer_braces = true:silent +csharp_preserve_single_line_blocks = true +csharp_preserve_single_line_statements = true diff --git a/DepotDownloader/AccountSettingsStore.cs b/DepotDownloader/AccountSettingsStore.cs index 2a8c9836..c6c9a251 100644 --- a/DepotDownloader/AccountSettingsStore.cs +++ b/DepotDownloader/AccountSettingsStore.cs @@ -1,33 +1,31 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; -using ProtoBuf; using System.IO; using System.IO.Compression; using System.IO.IsolatedStorage; -using System.Linq; -using SteamKit2; -using SteamKit2.Discovery; +using ProtoBuf; namespace DepotDownloader { [ProtoContract] class AccountSettingsStore { - [ProtoMember(1, IsRequired=false)] + [ProtoMember(1, IsRequired = false)] public Dictionary SentryData { get; private set; } [ProtoMember(2, IsRequired = false)] - public System.Collections.Concurrent.ConcurrentDictionary ContentServerPenalty { get; private set; } + public ConcurrentDictionary ContentServerPenalty { get; private set; } [ProtoMember(3, IsRequired = false)] public Dictionary LoginKeys { get; private set; } - string FileName = null; + string FileName; AccountSettingsStore() { SentryData = new Dictionary(); - ContentServerPenalty = new System.Collections.Concurrent.ConcurrentDictionary(); + ContentServerPenalty = new ConcurrentDictionary(); LoginKeys = new Dictionary(); } @@ -36,7 +34,7 @@ namespace DepotDownloader get { return Instance != null; } } - public static AccountSettingsStore Instance = null; + public static AccountSettingsStore Instance; static readonly IsolatedStorageFile IsolatedStorage = IsolatedStorageFile.GetUserStoreForAssembly(); public static void LoadFromFile(string filename) @@ -49,9 +47,9 @@ namespace DepotDownloader try { using (var fs = IsolatedStorage.OpenFile(filename, FileMode.Open, FileAccess.Read)) - using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress)) + using (var ds = new DeflateStream(fs, CompressionMode.Decompress)) { - Instance = ProtoBuf.Serializer.Deserialize(ds); + Instance = Serializer.Deserialize(ds); } } catch (IOException ex) @@ -76,9 +74,9 @@ namespace DepotDownloader try { using (var fs = IsolatedStorage.OpenFile(Instance.FileName, FileMode.Create, FileAccess.Write)) - using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Compress)) + using (var ds = new DeflateStream(fs, CompressionMode.Compress)) { - ProtoBuf.Serializer.Serialize(ds, Instance); + Serializer.Serialize(ds, Instance); } } catch (IOException ex) diff --git a/DepotDownloader/CDNClientPool.cs b/DepotDownloader/CDNClientPool.cs index 42056f2f..485bbcd2 100644 --- a/DepotDownloader/CDNClientPool.cs +++ b/DepotDownloader/CDNClientPool.cs @@ -1,11 +1,11 @@ -using SteamKit2; -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; +using SteamKit2; namespace DepotDownloader { @@ -82,7 +82,7 @@ namespace DepotDownloader private async Task ConnectionPoolMonitorAsync() { - bool didPopulate = false; + var didPopulate = false; while (!shutdownToken.IsCancellationRequested) { @@ -165,10 +165,8 @@ namespace DepotDownloader var result = await authTokenCallbackPromise.Task; return result.Token; } - else - { - throw new Exception($"Failed to retrieve CDN token for server {server.Host} depot {depotId}"); - } + + throw new Exception($"Failed to retrieve CDN token for server {server.Host} depot {depotId}"); } public void ReturnConnection(CDNClient.Server server) diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader/ContentDownloader.cs index 15f670ec..ba1ced91 100644 --- a/DepotDownloader/ContentDownloader.cs +++ b/DepotDownloader/ContentDownloader.cs @@ -1,1374 +1,1362 @@ -using SteamKit2; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; - -namespace DepotDownloader -{ - public class ContentDownloaderException : System.Exception - { - public ContentDownloaderException( String value ) : base( value ) {} - } - - static class ContentDownloader - { - public const uint INVALID_APP_ID = uint.MaxValue; - public const uint INVALID_DEPOT_ID = uint.MaxValue; - public const ulong INVALID_MANIFEST_ID = ulong.MaxValue; - public const string DEFAULT_BRANCH = "Public"; - - public static DownloadConfig Config = new DownloadConfig(); - - private static Steam3Session steam3; - private static Steam3Session.Credentials steam3Credentials; - private static CDNClientPool cdnPool; - - private const string DEFAULT_DOWNLOAD_DIR = "depots"; - private const string CONFIG_DIR = ".DepotDownloader"; - private static readonly string STAGING_DIR = Path.Combine( CONFIG_DIR, "staging" ); - - private sealed class DepotDownloadInfo - { - public uint id { get; private set; } - public string installDir { get; private set; } - public string contentName { get; private set; } - - public ulong manifestId { get; private set; } - public byte[] depotKey; - - public DepotDownloadInfo( uint depotid, ulong manifestId, string installDir, string contentName ) - { - this.id = depotid; - this.manifestId = manifestId; - this.installDir = installDir; - this.contentName = contentName; - } - } - - static bool CreateDirectories( uint depotId, uint depotVersion, out string installDir ) - { - installDir = null; - try - { - if ( string.IsNullOrWhiteSpace( ContentDownloader.Config.InstallDirectory ) ) - { - Directory.CreateDirectory( DEFAULT_DOWNLOAD_DIR ); - - string depotPath = Path.Combine( DEFAULT_DOWNLOAD_DIR, depotId.ToString() ); - Directory.CreateDirectory( depotPath ); - - installDir = Path.Combine( depotPath, depotVersion.ToString() ); - Directory.CreateDirectory( installDir ); - - Directory.CreateDirectory( Path.Combine( installDir, CONFIG_DIR ) ); - Directory.CreateDirectory( Path.Combine( installDir, STAGING_DIR ) ); - } - else - { - Directory.CreateDirectory( ContentDownloader.Config.InstallDirectory ); - - installDir = ContentDownloader.Config.InstallDirectory; - - Directory.CreateDirectory( Path.Combine( installDir, CONFIG_DIR ) ); - Directory.CreateDirectory( Path.Combine( installDir, STAGING_DIR ) ); - } - } - catch - { - return false; - } - - return true; - } - - static bool TestIsFileIncluded( string filename ) - { - if ( !Config.UsingFileList ) - return true; - - filename = filename.Replace( '\\', '/' ); - - if ( Config.FilesToDownload.Contains( filename ) ) - { - return true; - } - - foreach ( Regex rgx in Config.FilesToDownloadRegex ) - { - Match m = rgx.Match( filename ); - - if ( m.Success ) - return true; - } - - return false; - } - - static bool AccountHasAccess( uint depotId ) - { - if ( steam3 == null || steam3.steamUser.SteamID == null || ( steam3.Licenses == null && steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser ) ) - return false; - - IEnumerable licenseQuery; - if ( steam3.steamUser.SteamID.AccountType == EAccountType.AnonUser ) - { - licenseQuery = new List() { 17906 }; - } - else - { - licenseQuery = steam3.Licenses.Select( x => x.PackageID ).Distinct(); - } - - steam3.RequestPackageInfo( licenseQuery ); - - foreach ( var license in licenseQuery ) - { - SteamApps.PICSProductInfoCallback.PICSProductInfo package; - if ( steam3.PackageInfo.TryGetValue( license, out package ) && package != null ) - { - if ( package.KeyValues[ "appids" ].Children.Any( child => child.AsUnsignedInteger() == depotId ) ) - return true; - - if ( package.KeyValues[ "depotids" ].Children.Any( child => child.AsUnsignedInteger() == depotId ) ) - return true; - } - } - - return false; - } - - internal static KeyValue GetSteam3AppSection( uint appId, EAppInfoSection section ) - { - if ( steam3 == null || steam3.AppInfo == null ) - { - return null; - } - - SteamApps.PICSProductInfoCallback.PICSProductInfo app; - if ( !steam3.AppInfo.TryGetValue( appId, out app ) || app == null ) - { - return null; - } - - KeyValue appinfo = app.KeyValues; - string section_key; - - switch ( section ) - { - case EAppInfoSection.Common: - section_key = "common"; - break; - case EAppInfoSection.Extended: - section_key = "extended"; - break; - case EAppInfoSection.Config: - section_key = "config"; - break; - case EAppInfoSection.Depots: - section_key = "depots"; - break; - default: - throw new NotImplementedException(); - } - - KeyValue section_kv = appinfo.Children.Where( c => c.Name == section_key ).FirstOrDefault(); - return section_kv; - } - - static uint GetSteam3AppBuildNumber( uint appId, string branch ) - { - if ( appId == INVALID_APP_ID ) - return 0; - - - KeyValue depots = ContentDownloader.GetSteam3AppSection( appId, EAppInfoSection.Depots ); - KeyValue branches = depots[ "branches" ]; - KeyValue node = branches[ branch ]; - - if ( node == KeyValue.Invalid ) - return 0; - - KeyValue buildid = node[ "buildid" ]; - - if ( buildid == KeyValue.Invalid ) - return 0; - - return uint.Parse( buildid.Value ); - } - - static ulong GetSteam3DepotManifest( uint depotId, uint appId, string branch ) - { - KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); - KeyValue depotChild = depots[ depotId.ToString() ]; - - if ( depotChild == KeyValue.Invalid ) - return INVALID_MANIFEST_ID; - - // Shared depots can either provide manifests, or leave you relying on their parent app. - // It seems that with the latter, "sharedinstall" will exist (and equals 2 in the one existance I know of). - // Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680. - if ( depotChild[ "manifests" ] == KeyValue.Invalid && depotChild[ "depotfromapp" ] != KeyValue.Invalid ) - { - uint otherAppId = depotChild["depotfromapp"].AsUnsignedInteger(); - if ( otherAppId == appId ) - { - // This shouldn't ever happen, but ya never know with Valve. Don't infinite loop. - Console.WriteLine( "App {0}, Depot {1} has depotfromapp of {2}!", - appId, depotId, otherAppId ); - return INVALID_MANIFEST_ID; - } - - steam3.RequestAppInfo( otherAppId ); - - return GetSteam3DepotManifest( depotId, otherAppId, branch ); - } - - var manifests = depotChild[ "manifests" ]; - var manifests_encrypted = depotChild[ "encryptedmanifests" ]; - - if ( manifests.Children.Count == 0 && manifests_encrypted.Children.Count == 0 ) - return INVALID_MANIFEST_ID; - - var node = manifests[ branch ]; - - if ( branch != "Public" && node == KeyValue.Invalid ) - { - var node_encrypted = manifests_encrypted[ branch ]; - if ( node_encrypted != KeyValue.Invalid ) - { - string password = Config.BetaPassword; - if ( password == null ) - { - Console.Write( "Please enter the password for branch {0}: ", branch ); - Config.BetaPassword = password = Console.ReadLine(); - } - - var encrypted_v1 = node_encrypted[ "encrypted_gid" ]; - var encrypted_v2 = node_encrypted[ "encrypted_gid_2" ]; - - if ( encrypted_v1 != KeyValue.Invalid ) - { - byte[] input = Util.DecodeHexString( encrypted_v1.Value ); - byte[] manifest_bytes = CryptoHelper.VerifyAndDecryptPassword( input, password ); - - if ( manifest_bytes == null ) - { - Console.WriteLine( "Password was invalid for branch {0}", branch ); - return INVALID_MANIFEST_ID; - } - - return BitConverter.ToUInt64( manifest_bytes, 0 ); - } - else if ( encrypted_v2 != KeyValue.Invalid ) - { - // Submit the password to Steam now to get encryption keys - steam3.CheckAppBetaPassword( appId, Config.BetaPassword ); - - if ( !steam3.AppBetaPasswords.ContainsKey( branch ) ) - { - Console.WriteLine( "Password was invalid for branch {0}", branch ); - return INVALID_MANIFEST_ID; - } - - byte[] input = Util.DecodeHexString( encrypted_v2.Value ); - byte[] manifest_bytes; - try - { - manifest_bytes = CryptoHelper.SymmetricDecryptECB( input, steam3.AppBetaPasswords[ branch ] ); - } - catch ( Exception e ) - { - Console.WriteLine( "Failed to decrypt branch {0}: {1}", branch, e.Message ); - return INVALID_MANIFEST_ID; - } - - return BitConverter.ToUInt64( manifest_bytes, 0 ); - } - else - { - Console.WriteLine( "Unhandled depot encryption for depotId {0}", depotId ); - return INVALID_MANIFEST_ID; - } - - } - - return INVALID_MANIFEST_ID; - } - - if ( node.Value == null ) - return INVALID_MANIFEST_ID; - - return UInt64.Parse( node.Value ); - } - - static string GetAppOrDepotName( uint depotId, uint appId ) - { - if ( depotId == INVALID_DEPOT_ID ) - { - KeyValue info = GetSteam3AppSection( appId, EAppInfoSection.Common ); - - if ( info == null ) - return String.Empty; - - return info[ "name" ].AsString(); - } - else - { - KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); - - if ( depots == null ) - return String.Empty; - - KeyValue depotChild = depots[ depotId.ToString() ]; - - if ( depotChild == null ) - return String.Empty; - - return depotChild[ "name" ].AsString(); - } - } - - public static bool InitializeSteam3( string username, string password ) - { - string loginKey = null; - - if ( username != null && Config.RememberPassword ) - { - _ = AccountSettingsStore.Instance.LoginKeys.TryGetValue( username, out loginKey ); - } - - steam3 = new Steam3Session( - new SteamUser.LogOnDetails() - { - Username = username, - Password = loginKey == null ? password : null, - ShouldRememberPassword = Config.RememberPassword, - LoginKey = loginKey, - LoginID = Config.LoginID ?? 0x534B32, // "SK2" - } - ); - - steam3Credentials = steam3.WaitForCredentials(); - - if ( !steam3Credentials.IsValid ) - { - Console.WriteLine( "Unable to get steam3 credentials." ); - return false; - } - - return true; - } - - public static void ShutdownSteam3() - { - if (cdnPool != null) - { - cdnPool.Shutdown(); - cdnPool = null; - } - - if ( steam3 == null ) - return; - - steam3.TryWaitForLoginKey(); - steam3.Disconnect(); - } - - public static async Task DownloadPubfileAsync( uint appId, ulong publishedFileId ) - { - var details = steam3.GetPublishedFileDetails( appId, publishedFileId ); - - if ( !string.IsNullOrEmpty( details?.file_url ) ) - { - await DownloadWebFile( appId, details.filename, details.file_url ); - } - else if ( details?.hcontent_file > 0 ) - { - await DownloadAppAsync( appId, new List<(uint, ulong)>() { ( appId, details.hcontent_file ) }, DEFAULT_BRANCH, null, null, null, false, true ); - } - else - { - Console.WriteLine( "Unable to locate manifest ID for published file {0}", publishedFileId ); - } - } - - public static async Task DownloadUGCAsync( uint appId, ulong ugcId ) - { - SteamCloud.UGCDetailsCallback details = null; - - if ( steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser ) - { - details = steam3.GetUGCDetails( ugcId ); - } - else - { - Console.WriteLine( $"Unable to query UGC details for {ugcId} from an anonymous account" ); - } - - if ( !string.IsNullOrEmpty( details?.URL ) ) - { - await DownloadWebFile( appId, details.FileName, details.URL ); - } - else - { - await DownloadAppAsync( appId, new List<(uint, ulong)>() { ( appId, ugcId ) }, DEFAULT_BRANCH, null, null, null, false, true ); - } - } - - private static async Task DownloadWebFile( uint appId, string fileName, string url ) - { - string installDir; - if ( !CreateDirectories( appId, 0, out installDir ) ) - { - Console.WriteLine( "Error: Unable to create install directories!" ); - return; - } - - var stagingDir = Path.Combine( installDir, STAGING_DIR ); - var fileStagingPath = Path.Combine( stagingDir, fileName ); - var fileFinalPath = Path.Combine( installDir, fileName ); - - Directory.CreateDirectory( Path.GetDirectoryName( fileFinalPath ) ); - Directory.CreateDirectory( Path.GetDirectoryName( fileStagingPath ) ); - - using ( var file = File.OpenWrite( fileStagingPath ) ) - using ( var client = HttpClientFactory.CreateHttpClient() ) - { - Console.WriteLine( "Downloading {0}", fileName ); - var responseStream = await client.GetStreamAsync( url ); - await responseStream.CopyToAsync( file ); - } - - if ( File.Exists( fileFinalPath ) ) - { - File.Delete( fileFinalPath ); - } - - File.Move( fileStagingPath, fileFinalPath ); - } - - public static async Task DownloadAppAsync( uint appId, List<(uint depotId, ulong manifestId)> depotManifestIds, string branch, string os, string arch, string language, bool lv, bool isUgc ) - { - cdnPool = new CDNClientPool(steam3, appId); - - // Load our configuration data containing the depots currently installed - string configPath = ContentDownloader.Config.InstallDirectory; - if (string.IsNullOrWhiteSpace(configPath)) - { - configPath = DEFAULT_DOWNLOAD_DIR; - } - - Directory.CreateDirectory(Path.Combine(configPath, CONFIG_DIR)); - DepotConfigStore.LoadFromFile(Path.Combine(configPath, CONFIG_DIR, "depot.config")); - - if ( steam3 != null ) - steam3.RequestAppInfo( appId ); - - if ( !AccountHasAccess( appId ) ) - { - if ( steam3.RequestFreeAppLicense( appId ) ) - { - Console.WriteLine( "Obtained FreeOnDemand license for app {0}", appId ); - - // Fetch app info again in case we didn't get it fully without a license. - steam3.RequestAppInfo( appId, true ); - } - else - { - string contentName = GetAppOrDepotName( INVALID_DEPOT_ID, appId ); - throw new ContentDownloaderException( String.Format( "App {0} ({1}) is not available from this account.", appId, contentName ) ); - } - } - - var hasSpecificDepots = depotManifestIds.Count > 0; - var depotIdsFound = new List(); - var depotIdsExpected = depotManifestIds.Select( x => x.Item1 ).ToList(); - KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); - - if ( isUgc ) - { - var workshopDepot = depots["workshopdepot"].AsUnsignedInteger(); - if ( workshopDepot != 0 && !depotIdsExpected.Contains( workshopDepot ) ) - { - depotIdsExpected.Add( workshopDepot ); - depotManifestIds = depotManifestIds.Select( pair => ( workshopDepot, pair.manifestId ) ).ToList(); - } - - depotIdsFound.AddRange( depotIdsExpected ); - } - else - { - Console.WriteLine( "Using app branch: '{0}'.", branch ); - - if ( depots != null ) - { - foreach ( var depotSection in depots.Children ) - { - uint id = INVALID_DEPOT_ID; - if ( depotSection.Children.Count == 0 ) - continue; - - if ( !uint.TryParse( depotSection.Name, out id ) ) - continue; - - if ( hasSpecificDepots && !depotIdsExpected.Contains( id ) ) - continue; - - if ( !hasSpecificDepots ) - { - var depotConfig = depotSection[ "config" ]; - if ( depotConfig != KeyValue.Invalid ) - { - if ( !Config.DownloadAllPlatforms && - depotConfig["oslist"] != KeyValue.Invalid && - !string.IsNullOrWhiteSpace( depotConfig["oslist"].Value ) ) - { - var oslist = depotConfig["oslist"].Value.Split( ',' ); - if ( Array.IndexOf( oslist, os ?? Util.GetSteamOS() ) == -1 ) - continue; - } - - if ( depotConfig["osarch"] != KeyValue.Invalid && - !string.IsNullOrWhiteSpace( depotConfig["osarch"].Value ) ) - { - var depotArch = depotConfig["osarch"].Value; - if ( depotArch != ( arch ?? Util.GetSteamArch() ) ) - continue; - } - - if ( !Config.DownloadAllLanguages && - depotConfig["language"] != KeyValue.Invalid && - !string.IsNullOrWhiteSpace( depotConfig["language"].Value ) ) - { - var depotLang = depotConfig["language"].Value; - if ( depotLang != ( language ?? "english" ) ) - continue; - } - - if ( !lv && - depotConfig["lowviolence"] != KeyValue.Invalid && - depotConfig["lowviolence"].AsBoolean() ) - continue; - } - } - - depotIdsFound.Add( id ); - - if ( !hasSpecificDepots ) - depotManifestIds.Add( ( id, ContentDownloader.INVALID_MANIFEST_ID ) ); - } - } - if ( depotManifestIds.Count == 0 && !hasSpecificDepots ) - { - throw new ContentDownloaderException( String.Format( "Couldn't find any depots to download for app {0}", appId ) ); - } - else if ( depotIdsFound.Count < depotIdsExpected.Count ) - { - var remainingDepotIds = depotIdsExpected.Except( depotIdsFound ); - throw new ContentDownloaderException( String.Format( "Depot {0} not listed for app {1}", string.Join(", ", remainingDepotIds), appId ) ); - } - } - - var infos = new List(); - - foreach ( var depotManifest in depotManifestIds ) - { - var info = GetDepotInfo( depotManifest.Item1, appId, depotManifest.Item2, branch ); - if ( info != null ) - { - infos.Add( info ); - } - } - - try - { - await DownloadSteam3Async( appId, infos ).ConfigureAwait( false ); - } - catch ( OperationCanceledException ) - { - Console.WriteLine( "App {0} was not completely downloaded.", appId ); - throw; - } - } - - static DepotDownloadInfo GetDepotInfo( uint depotId, uint appId, ulong manifestId, string branch ) - { - if ( steam3 != null && appId != INVALID_APP_ID ) - steam3.RequestAppInfo( ( uint )appId ); - - string contentName = GetAppOrDepotName( depotId, appId ); - - if ( !AccountHasAccess( depotId ) ) - { - Console.WriteLine( "Depot {0} ({1}) is not available from this account.", depotId, contentName ); - - return null; - } - - if (manifestId == INVALID_MANIFEST_ID) - { - manifestId = GetSteam3DepotManifest(depotId, appId, branch); - if (manifestId == INVALID_MANIFEST_ID && branch != "public") - { - Console.WriteLine("Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch); - branch = "public"; - manifestId = GetSteam3DepotManifest(depotId, appId, branch); - } - - if (manifestId == INVALID_MANIFEST_ID) - { - Console.WriteLine("Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName); - return null; - } - } - - uint uVersion = GetSteam3AppBuildNumber( appId, branch ); - - string installDir; - if ( !CreateDirectories( depotId, uVersion, out installDir ) ) - { - Console.WriteLine( "Error: Unable to create install directories!" ); - return null; - } - - steam3.RequestDepotKey( depotId, appId ); - if ( !steam3.DepotKeys.ContainsKey( depotId ) ) - { - Console.WriteLine( "No valid depot key for {0}, unable to download.", depotId ); - return null; - } - - byte[] depotKey = steam3.DepotKeys[ depotId ]; - - var info = new DepotDownloadInfo( depotId, manifestId, installDir, contentName ); - info.depotKey = depotKey; - return info; - } - - private class ChunkMatch - { - public ChunkMatch( ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk ) - { - OldChunk = oldChunk; - NewChunk = newChunk; - } - public ProtoManifest.ChunkData OldChunk { get; private set; } - public ProtoManifest.ChunkData NewChunk { get; private set; } - } - - private class DepotFilesData - { - public DepotDownloadInfo depotDownloadInfo; - public DepotDownloadCounter depotCounter; - public string stagingDir; - public ProtoManifest manifest; - public ProtoManifest previousManifest; - public List filteredFiles; - public HashSet allFileNames; - } - - private class FileStreamData - { - public FileStream fileStream; - public SemaphoreSlim fileLock; - public int chunksToDownload; - } - - private class GlobalDownloadCounter - { - public ulong TotalBytesCompressed; - public ulong TotalBytesUncompressed; - } - - private class DepotDownloadCounter - { - public ulong CompleteDownloadSize; - public ulong SizeDownloaded; - public ulong DepotBytesCompressed; - public ulong DepotBytesUncompressed; - - } - - private static async Task DownloadSteam3Async(uint appId, List depots) - { - CancellationTokenSource cts = new CancellationTokenSource(); - cdnPool.ExhaustedToken = cts; - - GlobalDownloadCounter downloadCounter = new GlobalDownloadCounter(); - var depotsToDownload = new List(depots.Count); - var allFileNamesAllDepots = new HashSet(); - - // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup - foreach (var depot in depots) - { - var depotFileData = await ProcessDepotManifestAndFiles(cts, appId, depot); - - if (depotFileData != null) - { - depotsToDownload.Add(depotFileData); - allFileNamesAllDepots.UnionWith(depotFileData.allFileNames); - } - - cts.Token.ThrowIfCancellationRequested(); - } - - // If we're about to write all the files to the same directory, we will need to first de-duplicate any files by path - // This is in last-depot-wins order, from Steam or the list of depots supplied by the user - if (!string.IsNullOrWhiteSpace(ContentDownloader.Config.InstallDirectory) && depotsToDownload.Count > 0) - { - var claimedFileNames = new HashSet(); - - for (var i = depotsToDownload.Count - 1; i >= 0; i--) - { - // For each depot, remove all files from the list that have been claimed by a later depot - depotsToDownload[i].filteredFiles.RemoveAll(file => claimedFileNames.Contains(file.FileName)); - - claimedFileNames.UnionWith(depotsToDownload[i].allFileNames); - } - } - - foreach (var depotFileData in depotsToDownload) - { - await DownloadSteam3AsyncDepotFiles(cts, appId, downloadCounter, depotFileData, allFileNamesAllDepots); - } - - Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", - downloadCounter.TotalBytesCompressed, downloadCounter.TotalBytesUncompressed, depots.Count); - } - - private static async Task ProcessDepotManifestAndFiles(CancellationTokenSource cts, - uint appId, DepotDownloadInfo depot) - { - DepotDownloadCounter depotCounter = new DepotDownloadCounter(); - - Console.WriteLine("Processing depot {0} - {1}", depot.id, depot.contentName); - - ProtoManifest oldProtoManifest = null; - ProtoManifest newProtoManifest = null; - string configDir = Path.Combine(depot.installDir, CONFIG_DIR); - - ulong lastManifestId = INVALID_MANIFEST_ID; - DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId); - - // In case we have an early exit, this will force equiv of verifyall next run. - DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID; - DepotConfigStore.Save(); - - if (lastManifestId != INVALID_MANIFEST_ID) - { - var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId)); - - if (File.Exists(oldManifestFileName)) - { - byte[] expectedChecksum, currentChecksum; - - try - { - expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); - } - catch (IOException) - { - expectedChecksum = null; - } - - oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum); - - if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) - { - // We only have to show this warning if the old manifest ID was different - if (lastManifestId != depot.manifestId) - Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); - oldProtoManifest = null; - } - } - } - - if (lastManifestId == depot.manifestId && oldProtoManifest != null) - { - newProtoManifest = oldProtoManifest; - Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); - } - else - { - var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId)); - if (newManifestFileName != null) - { - byte[] expectedChecksum, currentChecksum; - - try - { - expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); - } - catch (IOException) - { - expectedChecksum = null; - } - - newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum); - - if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) - { - Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId); - newProtoManifest = null; - } - } - - if (newProtoManifest != null) - { - Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); - } - else - { - Console.Write("Downloading depot manifest..."); - - DepotManifest depotManifest = null; - - do - { - cts.Token.ThrowIfCancellationRequested(); - - CDNClient.Server connection = null; - - try - { - connection = cdnPool.GetConnection(cts.Token); - - DebugLog.WriteLine("ContentDownloader", "Authenticating connection to {0}", connection); - var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection); - - DebugLog.WriteLine("ContentDownloader", "Downloading manifest {0} from {1} with {2}", depot.manifestId, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); - depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId, - connection, cdnToken, depot.depotKey, proxyServer: cdnPool.ProxyServer).ConfigureAwait(false); - - cdnPool.ReturnConnection(connection); - } - catch (TaskCanceledException) - { - Console.WriteLine("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId); - } - catch (SteamKitWebRequestException e) - { - cdnPool.ReturnBrokenConnection(connection); - - if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) - { - Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); - break; - } - else if (e.StatusCode == HttpStatusCode.NotFound) - { - Console.WriteLine("Encountered 404 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); - break; - } - else - { - Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode); - } - } - catch (OperationCanceledException) - { - break; - } - catch (Exception e) - { - cdnPool.ReturnBrokenConnection(connection); - Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message); - } - } - while (depotManifest == null); - - if (depotManifest == null) - { - Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); - cts.Cancel(); - } - - // Throw the cancellation exception if requested so that this task is marked failed - cts.Token.ThrowIfCancellationRequested(); - - byte[] checksum; - - newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); - newProtoManifest.SaveToFile(newManifestFileName, out checksum); - File.WriteAllBytes(newManifestFileName + ".sha", checksum); - - Console.WriteLine(" Done!"); - } - } - - newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal)); - - Console.WriteLine("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime); - - if (Config.DownloadManifestOnly) - { - DumpManifestToTextFile(depot, newProtoManifest); - return null; - } - - string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); - - var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); - var allFileNames = new HashSet(filesAfterExclusions.Count); - - // Pre-process - filesAfterExclusions.ForEach(file => - { - allFileNames.Add(file.FileName); - - var fileFinalPath = Path.Combine(depot.installDir, file.FileName); - var fileStagingPath = Path.Combine(stagingDir, file.FileName); - - if (file.Flags.HasFlag(EDepotFileFlag.Directory)) - { - Directory.CreateDirectory(fileFinalPath); - Directory.CreateDirectory(fileStagingPath); - } - else - { - // Some manifests don't explicitly include all necessary directories - Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); - Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); - - depotCounter.CompleteDownloadSize += file.TotalSize; - } - }); - - return new DepotFilesData - { - depotDownloadInfo = depot, - depotCounter = depotCounter, - stagingDir = stagingDir, - manifest = newProtoManifest, - previousManifest = oldProtoManifest, - filteredFiles = filesAfterExclusions, - allFileNames = allFileNames - }; - } - - private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource cts, uint appId, - GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, HashSet allFileNamesAllDepots) - { - var depot = depotFilesData.depotDownloadInfo; - var depotCounter = depotFilesData.depotCounter; - - Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName); - - var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); - var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>(); - - await Util.InvokeAsync( - files.Select(file => new Func(async () => - await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, depotFilesData, file, networkChunkQueue)))), - maxDegreeOfParallelism: Config.MaxDownloads - ); - - await Util.InvokeAsync( - networkChunkQueue.Select(q => new Func(async () => - await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, appId, downloadCounter, depotFilesData, - q.fileData, q.fileStreamData, q.chunk)))), - maxDegreeOfParallelism: Config.MaxDownloads - ); - - // Check for deleted files if updating the depot. - if (depotFilesData.previousManifest != null) - { - var previousFilteredFiles = depotFilesData.previousManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).Select(f => f.FileName).ToHashSet(); - - // Check if we are writing to a single output directory. If not, each depot folder is managed independently - if (string.IsNullOrWhiteSpace(ContentDownloader.Config.InstallDirectory)) - { - // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names - previousFilteredFiles.ExceptWith(depotFilesData.allFileNames); - } - else - { - // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names across all depots being downloaded - previousFilteredFiles.ExceptWith(allFileNamesAllDepots); - } - - foreach(var existingFileName in previousFilteredFiles) - { - string fileFinalPath = Path.Combine(depot.installDir, existingFileName); - - if (!File.Exists(fileFinalPath)) - continue; - - File.Delete(fileFinalPath); - Console.WriteLine("Deleted {0}", fileFinalPath); - } - } - - DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId; - DepotConfigStore.Save(); - - Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed); - } - - private static void DownloadSteam3AsyncDepotFile( - CancellationTokenSource cts, - DepotFilesData depotFilesData, - ProtoManifest.FileData file, - ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue) - { - cts.Token.ThrowIfCancellationRequested(); - - var depot = depotFilesData.depotDownloadInfo; - var stagingDir = depotFilesData.stagingDir; - var depotDownloadCounter = depotFilesData.depotCounter; - var oldProtoManifest = depotFilesData.previousManifest; - - string fileFinalPath = Path.Combine(depot.installDir, file.FileName); - string fileStagingPath = Path.Combine(stagingDir, file.FileName); - - // This may still exist if the previous run exited before cleanup - if (File.Exists(fileStagingPath)) - { - File.Delete(fileStagingPath); - } - - FileStream fs = null; - List neededChunks; - FileInfo fi = new FileInfo(fileFinalPath); - if (!fi.Exists) - { - Console.WriteLine("Pre-allocating {0}", fileFinalPath); - - // create new file. need all chunks - fs = File.Create(fileFinalPath); - try - { - fs.SetLength((long)file.TotalSize); - } - catch (IOException ex) - { - throw new ContentDownloaderException(String.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message)); - } - neededChunks = new List(file.Chunks); - } - else - { - // open existing - ProtoManifest.FileData oldManifestFile = null; - if (oldProtoManifest != null) - { - oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); - } - - if (oldManifestFile != null) - { - neededChunks = new List(); - - var hashMatches = oldManifestFile.FileHash.SequenceEqual(file.FileHash); - if (Config.VerifyAll || !hashMatches) - { - // we have a version of this file, but it doesn't fully match what we want - if (Config.VerifyAll) - { - Console.WriteLine("Validating {0}", fileFinalPath); - } - - var matchingChunks = new List(); - - foreach (var chunk in file.Chunks) - { - var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); - if (oldChunk != null) - { - matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); - } - else - { - neededChunks.Add(chunk); - } - } - - var orderedChunks = matchingChunks.OrderBy(x => x.OldChunk.Offset); - - var copyChunks = new List(); - - using (var fsOld = File.Open(fileFinalPath, FileMode.Open)) - { - foreach (var match in orderedChunks) - { - fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); - - byte[] tmp = new byte[match.OldChunk.UncompressedLength]; - fsOld.Read(tmp, 0, tmp.Length); - - byte[] adler = Util.AdlerHash(tmp); - if (!adler.SequenceEqual(match.OldChunk.Checksum)) - { - neededChunks.Add(match.NewChunk); - } - else - { - copyChunks.Add(match); - } - } - } - - if (!hashMatches || neededChunks.Count > 0) - { - File.Move(fileFinalPath, fileStagingPath); - - using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) - { - fs = File.Open(fileFinalPath, FileMode.Create); - try - { - fs.SetLength((long)file.TotalSize); - } - catch (IOException ex) - { - throw new ContentDownloaderException(String.Format("Failed to resize file to expected size {0}: {1}", fileFinalPath, ex.Message)); - } - - foreach (var match in copyChunks) - { - fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); - - byte[] tmp = new byte[match.OldChunk.UncompressedLength]; - fsOld.Read(tmp, 0, tmp.Length); - - fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin); - fs.Write(tmp, 0, tmp.Length); - } - } - - File.Delete(fileStagingPath); - } - } - } - else - { - // No old manifest or file not in old manifest. We must validate. - - fs = File.Open(fileFinalPath, FileMode.Open); - if ((ulong)fi.Length != file.TotalSize) - { - try - { - fs.SetLength((long)file.TotalSize); - } - catch (IOException ex) - { - throw new ContentDownloaderException(String.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message)); - } - } - - Console.WriteLine("Validating {0}", fileFinalPath); - neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); - } - - if (neededChunks.Count() == 0) - { - lock (depotDownloadCounter) - { - depotDownloadCounter.SizeDownloaded += (ulong)file.TotalSize; - Console.WriteLine("{0,6:#00.00}% {1}", ((float)depotDownloadCounter.SizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath); - } - - if (fs != null) - fs.Dispose(); - return; - } - else - { - var sizeOnDisk = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum()); - lock (depotDownloadCounter) - { - depotDownloadCounter.SizeDownloaded += sizeOnDisk; - } - } - } - - FileStreamData fileStreamData = new FileStreamData - { - fileStream = fs, - fileLock = new SemaphoreSlim(1), - chunksToDownload = neededChunks.Count - }; - - foreach (var chunk in neededChunks) - { - networkChunkQueue.Enqueue((fileStreamData, file, chunk)); - } - } - - private static async Task DownloadSteam3AsyncDepotFileChunk( - CancellationTokenSource cts, uint appId, - GlobalDownloadCounter downloadCounter, - DepotFilesData depotFilesData, - ProtoManifest.FileData file, - FileStreamData fileStreamData, - ProtoManifest.ChunkData chunk) - { - cts.Token.ThrowIfCancellationRequested(); - - var depot = depotFilesData.depotDownloadInfo; - var depotDownloadCounter = depotFilesData.depotCounter; - - string chunkID = Util.EncodeHexString(chunk.ChunkID); - - DepotManifest.ChunkData data = new DepotManifest.ChunkData(); - data.ChunkID = chunk.ChunkID; - data.Checksum = chunk.Checksum; - data.Offset = chunk.Offset; - data.CompressedLength = chunk.CompressedLength; - data.UncompressedLength = chunk.UncompressedLength; - - CDNClient.DepotChunk chunkData = null; - - do - { - cts.Token.ThrowIfCancellationRequested(); - - CDNClient.Server connection = null; - - try - { - connection = cdnPool.GetConnection(cts.Token); - - DebugLog.WriteLine("ContentDownloader", "Authenticating connection to {0}", connection); - var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection); - - DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); - chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data, - connection, cdnToken, depot.depotKey, proxyServer: cdnPool.ProxyServer).ConfigureAwait(false); - - cdnPool.ReturnConnection(connection); - } - catch (TaskCanceledException) - { - Console.WriteLine("Connection timeout downloading chunk {0}", chunkID); - } - catch (SteamKitWebRequestException e) - { - cdnPool.ReturnBrokenConnection(connection); - - if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) - { - Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); - break; - } - else - { - Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode); - } - } - catch (OperationCanceledException) - { - break; - } - catch (Exception e) - { - cdnPool.ReturnBrokenConnection(connection); - Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); - } - } - while (chunkData == null); - - if (chunkData == null) - { - Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id); - cts.Cancel(); - } - - // Throw the cancellation exception if requested so that this task is marked failed - cts.Token.ThrowIfCancellationRequested(); - - try - { - await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false); - - fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin); - await fileStreamData.fileStream.WriteAsync(chunkData.Data, 0, chunkData.Data.Length); - } - finally - { - fileStreamData.fileLock.Release(); - } - - int remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload); - if (remainingChunks == 0) - { - fileStreamData.fileStream.Dispose(); - fileStreamData.fileLock.Dispose(); - } - - ulong sizeDownloaded = 0; - lock (depotDownloadCounter) - { - sizeDownloaded = depotDownloadCounter.SizeDownloaded + (ulong)chunkData.Data.Length; - depotDownloadCounter.SizeDownloaded = sizeDownloaded; - depotDownloadCounter.DepotBytesCompressed += chunk.CompressedLength; - depotDownloadCounter.DepotBytesUncompressed += chunk.UncompressedLength; - } - - lock (downloadCounter) - { - downloadCounter.TotalBytesCompressed += chunk.CompressedLength; - downloadCounter.TotalBytesUncompressed += chunk.UncompressedLength; - } - - if (remainingChunks == 0) - { - var fileFinalPath = Path.Combine(depot.installDir, file.FileName); - Console.WriteLine("{0,6:#00.00}% {1}", ((float)sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath); - } - - } - - static void DumpManifestToTextFile( DepotDownloadInfo depot, ProtoManifest manifest ) - { - var txtManifest = Path.Combine( depot.installDir, $"manifest_{depot.id}_{depot.manifestId}.txt" ); - - using ( var sw = new StreamWriter( txtManifest ) ) - { - sw.WriteLine( $"Content Manifest for Depot {depot.id}" ); - sw.WriteLine(); - sw.WriteLine( $"Manifest ID / date : {depot.manifestId} / {manifest.CreationTime}" ); - - int numFiles = 0, numChunks = 0; - ulong uncompressedSize = 0, compressedSize = 0; - - foreach ( var file in manifest.Files ) - { - if ( file.Flags.HasFlag( EDepotFileFlag.Directory ) ) - continue; - - numFiles++; - numChunks += file.Chunks.Count; - - foreach ( var chunk in file.Chunks ) - { - uncompressedSize += chunk.UncompressedLength; - compressedSize += chunk.CompressedLength; - } - } - - sw.WriteLine( $"Total number of files : {numFiles}" ); - sw.WriteLine( $"Total number of chunks : {numChunks}" ); - sw.WriteLine( $"Total bytes on disk : {uncompressedSize}" ); - sw.WriteLine( $"Total bytes compressed : {compressedSize}" ); - sw.WriteLine(); - sw.WriteLine( " Size Chunks File SHA Flags Name" ); - - foreach ( var file in manifest.Files ) - { - var sha1Hash = BitConverter.ToString( file.FileHash ).Replace( "-", "" ); - sw.WriteLine( $"{file.TotalSize,14} {file.Chunks.Count,6} {sha1Hash} {file.Flags,5:D} {file.FileName}" ); - } - } - } - } -} +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using SteamKit2; + +namespace DepotDownloader +{ + public class ContentDownloaderException : Exception + { + public ContentDownloaderException(String value) : base(value) { } + } + + static class ContentDownloader + { + public const uint INVALID_APP_ID = uint.MaxValue; + public const uint INVALID_DEPOT_ID = uint.MaxValue; + public const ulong INVALID_MANIFEST_ID = ulong.MaxValue; + public const string DEFAULT_BRANCH = "Public"; + + public static DownloadConfig Config = new DownloadConfig(); + + private static Steam3Session steam3; + private static Steam3Session.Credentials steam3Credentials; + private static CDNClientPool cdnPool; + + private const string DEFAULT_DOWNLOAD_DIR = "depots"; + private const string CONFIG_DIR = ".DepotDownloader"; + private static readonly string STAGING_DIR = Path.Combine(CONFIG_DIR, "staging"); + + private sealed class DepotDownloadInfo + { + public uint id { get; private set; } + public string installDir { get; private set; } + public string contentName { get; private set; } + + public ulong manifestId { get; private set; } + public byte[] depotKey; + + public DepotDownloadInfo(uint depotid, ulong manifestId, string installDir, string contentName) + { + this.id = depotid; + this.manifestId = manifestId; + this.installDir = installDir; + this.contentName = contentName; + } + } + + static bool CreateDirectories(uint depotId, uint depotVersion, out string installDir) + { + installDir = null; + try + { + if (string.IsNullOrWhiteSpace(Config.InstallDirectory)) + { + Directory.CreateDirectory(DEFAULT_DOWNLOAD_DIR); + + var depotPath = Path.Combine(DEFAULT_DOWNLOAD_DIR, depotId.ToString()); + Directory.CreateDirectory(depotPath); + + installDir = Path.Combine(depotPath, depotVersion.ToString()); + Directory.CreateDirectory(installDir); + + Directory.CreateDirectory(Path.Combine(installDir, CONFIG_DIR)); + Directory.CreateDirectory(Path.Combine(installDir, STAGING_DIR)); + } + else + { + Directory.CreateDirectory(Config.InstallDirectory); + + installDir = Config.InstallDirectory; + + Directory.CreateDirectory(Path.Combine(installDir, CONFIG_DIR)); + Directory.CreateDirectory(Path.Combine(installDir, STAGING_DIR)); + } + } + catch + { + return false; + } + + return true; + } + + static bool TestIsFileIncluded(string filename) + { + if (!Config.UsingFileList) + return true; + + filename = filename.Replace('\\', '/'); + + if (Config.FilesToDownload.Contains(filename)) + { + return true; + } + + foreach (var rgx in Config.FilesToDownloadRegex) + { + var m = rgx.Match(filename); + + if (m.Success) + return true; + } + + return false; + } + + static bool AccountHasAccess(uint depotId) + { + if (steam3 == null || steam3.steamUser.SteamID == null || (steam3.Licenses == null && steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser)) + return false; + + IEnumerable licenseQuery; + if (steam3.steamUser.SteamID.AccountType == EAccountType.AnonUser) + { + licenseQuery = new List { 17906 }; + } + else + { + licenseQuery = steam3.Licenses.Select(x => x.PackageID).Distinct(); + } + + steam3.RequestPackageInfo(licenseQuery); + + foreach (var license in licenseQuery) + { + SteamApps.PICSProductInfoCallback.PICSProductInfo package; + if (steam3.PackageInfo.TryGetValue(license, out package) && package != null) + { + if (package.KeyValues["appids"].Children.Any(child => child.AsUnsignedInteger() == depotId)) + return true; + + if (package.KeyValues["depotids"].Children.Any(child => child.AsUnsignedInteger() == depotId)) + return true; + } + } + + return false; + } + + internal static KeyValue GetSteam3AppSection(uint appId, EAppInfoSection section) + { + if (steam3 == null || steam3.AppInfo == null) + { + return null; + } + + SteamApps.PICSProductInfoCallback.PICSProductInfo app; + if (!steam3.AppInfo.TryGetValue(appId, out app) || app == null) + { + return null; + } + + var appinfo = app.KeyValues; + string section_key; + + switch (section) + { + case EAppInfoSection.Common: + section_key = "common"; + break; + case EAppInfoSection.Extended: + section_key = "extended"; + break; + case EAppInfoSection.Config: + section_key = "config"; + break; + case EAppInfoSection.Depots: + section_key = "depots"; + break; + default: + throw new NotImplementedException(); + } + + var section_kv = appinfo.Children.Where(c => c.Name == section_key).FirstOrDefault(); + return section_kv; + } + + static uint GetSteam3AppBuildNumber(uint appId, string branch) + { + if (appId == INVALID_APP_ID) + return 0; + + + var depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); + var branches = depots["branches"]; + var node = branches[branch]; + + if (node == KeyValue.Invalid) + return 0; + + var buildid = node["buildid"]; + + if (buildid == KeyValue.Invalid) + return 0; + + return uint.Parse(buildid.Value); + } + + static ulong GetSteam3DepotManifest(uint depotId, uint appId, string branch) + { + var depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); + var depotChild = depots[depotId.ToString()]; + + if (depotChild == KeyValue.Invalid) + return INVALID_MANIFEST_ID; + + // Shared depots can either provide manifests, or leave you relying on their parent app. + // It seems that with the latter, "sharedinstall" will exist (and equals 2 in the one existance I know of). + // Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680. + if (depotChild["manifests"] == KeyValue.Invalid && depotChild["depotfromapp"] != KeyValue.Invalid) + { + var otherAppId = depotChild["depotfromapp"].AsUnsignedInteger(); + if (otherAppId == appId) + { + // This shouldn't ever happen, but ya never know with Valve. Don't infinite loop. + Console.WriteLine("App {0}, Depot {1} has depotfromapp of {2}!", + appId, depotId, otherAppId); + return INVALID_MANIFEST_ID; + } + + steam3.RequestAppInfo(otherAppId); + + return GetSteam3DepotManifest(depotId, otherAppId, branch); + } + + var manifests = depotChild["manifests"]; + var manifests_encrypted = depotChild["encryptedmanifests"]; + + if (manifests.Children.Count == 0 && manifests_encrypted.Children.Count == 0) + return INVALID_MANIFEST_ID; + + var node = manifests[branch]; + + if (branch != "Public" && node == KeyValue.Invalid) + { + var node_encrypted = manifests_encrypted[branch]; + if (node_encrypted != KeyValue.Invalid) + { + var password = Config.BetaPassword; + if (password == null) + { + Console.Write("Please enter the password for branch {0}: ", branch); + Config.BetaPassword = password = Console.ReadLine(); + } + + var encrypted_v1 = node_encrypted["encrypted_gid"]; + var encrypted_v2 = node_encrypted["encrypted_gid_2"]; + + if (encrypted_v1 != KeyValue.Invalid) + { + var input = Util.DecodeHexString(encrypted_v1.Value); + var manifest_bytes = CryptoHelper.VerifyAndDecryptPassword(input, password); + + if (manifest_bytes == null) + { + Console.WriteLine("Password was invalid for branch {0}", branch); + return INVALID_MANIFEST_ID; + } + + return BitConverter.ToUInt64(manifest_bytes, 0); + } + + if (encrypted_v2 != KeyValue.Invalid) + { + // Submit the password to Steam now to get encryption keys + steam3.CheckAppBetaPassword(appId, Config.BetaPassword); + + if (!steam3.AppBetaPasswords.ContainsKey(branch)) + { + Console.WriteLine("Password was invalid for branch {0}", branch); + return INVALID_MANIFEST_ID; + } + + var input = Util.DecodeHexString(encrypted_v2.Value); + byte[] manifest_bytes; + try + { + manifest_bytes = CryptoHelper.SymmetricDecryptECB(input, steam3.AppBetaPasswords[branch]); + } + catch (Exception e) + { + Console.WriteLine("Failed to decrypt branch {0}: {1}", branch, e.Message); + return INVALID_MANIFEST_ID; + } + + return BitConverter.ToUInt64(manifest_bytes, 0); + } + + Console.WriteLine("Unhandled depot encryption for depotId {0}", depotId); + return INVALID_MANIFEST_ID; + } + + return INVALID_MANIFEST_ID; + } + + if (node.Value == null) + return INVALID_MANIFEST_ID; + + return UInt64.Parse(node.Value); + } + + static string GetAppOrDepotName(uint depotId, uint appId) + { + if (depotId == INVALID_DEPOT_ID) + { + var info = GetSteam3AppSection(appId, EAppInfoSection.Common); + + if (info == null) + return String.Empty; + + return info["name"].AsString(); + } + + var depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); + + if (depots == null) + return String.Empty; + + var depotChild = depots[depotId.ToString()]; + + if (depotChild == null) + return String.Empty; + + return depotChild["name"].AsString(); + } + + public static bool InitializeSteam3(string username, string password) + { + string loginKey = null; + + if (username != null && Config.RememberPassword) + { + _ = AccountSettingsStore.Instance.LoginKeys.TryGetValue(username, out loginKey); + } + + steam3 = new Steam3Session( + new SteamUser.LogOnDetails + { + Username = username, + Password = loginKey == null ? password : null, + ShouldRememberPassword = Config.RememberPassword, + LoginKey = loginKey, + LoginID = Config.LoginID ?? 0x534B32, // "SK2" + } + ); + + steam3Credentials = steam3.WaitForCredentials(); + + if (!steam3Credentials.IsValid) + { + Console.WriteLine("Unable to get steam3 credentials."); + return false; + } + + return true; + } + + public static void ShutdownSteam3() + { + if (cdnPool != null) + { + cdnPool.Shutdown(); + cdnPool = null; + } + + if (steam3 == null) + return; + + steam3.TryWaitForLoginKey(); + steam3.Disconnect(); + } + + public static async Task DownloadPubfileAsync(uint appId, ulong publishedFileId) + { + var details = steam3.GetPublishedFileDetails(appId, publishedFileId); + + if (!string.IsNullOrEmpty(details?.file_url)) + { + await DownloadWebFile(appId, details.filename, details.file_url); + } + else if (details?.hcontent_file > 0) + { + await DownloadAppAsync(appId, new List<(uint, ulong)> { (appId, details.hcontent_file) }, DEFAULT_BRANCH, null, null, null, false, true); + } + else + { + Console.WriteLine("Unable to locate manifest ID for published file {0}", publishedFileId); + } + } + + public static async Task DownloadUGCAsync(uint appId, ulong ugcId) + { + SteamCloud.UGCDetailsCallback details = null; + + if (steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser) + { + details = steam3.GetUGCDetails(ugcId); + } + else + { + Console.WriteLine($"Unable to query UGC details for {ugcId} from an anonymous account"); + } + + if (!string.IsNullOrEmpty(details?.URL)) + { + await DownloadWebFile(appId, details.FileName, details.URL); + } + else + { + await DownloadAppAsync(appId, new List<(uint, ulong)> { (appId, ugcId) }, DEFAULT_BRANCH, null, null, null, false, true); + } + } + + private static async Task DownloadWebFile(uint appId, string fileName, string url) + { + string installDir; + if (!CreateDirectories(appId, 0, out installDir)) + { + Console.WriteLine("Error: Unable to create install directories!"); + return; + } + + var stagingDir = Path.Combine(installDir, STAGING_DIR); + var fileStagingPath = Path.Combine(stagingDir, fileName); + var fileFinalPath = Path.Combine(installDir, fileName); + + Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); + Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); + + using (var file = File.OpenWrite(fileStagingPath)) + using (var client = HttpClientFactory.CreateHttpClient()) + { + Console.WriteLine("Downloading {0}", fileName); + var responseStream = await client.GetStreamAsync(url); + await responseStream.CopyToAsync(file); + } + + if (File.Exists(fileFinalPath)) + { + File.Delete(fileFinalPath); + } + + File.Move(fileStagingPath, fileFinalPath); + } + + public static async Task DownloadAppAsync(uint appId, List<(uint depotId, ulong manifestId)> depotManifestIds, string branch, string os, string arch, string language, bool lv, bool isUgc) + { + cdnPool = new CDNClientPool(steam3, appId); + + // Load our configuration data containing the depots currently installed + var configPath = Config.InstallDirectory; + if (string.IsNullOrWhiteSpace(configPath)) + { + configPath = DEFAULT_DOWNLOAD_DIR; + } + + Directory.CreateDirectory(Path.Combine(configPath, CONFIG_DIR)); + DepotConfigStore.LoadFromFile(Path.Combine(configPath, CONFIG_DIR, "depot.config")); + + if (steam3 != null) + steam3.RequestAppInfo(appId); + + if (!AccountHasAccess(appId)) + { + if (steam3.RequestFreeAppLicense(appId)) + { + Console.WriteLine("Obtained FreeOnDemand license for app {0}", appId); + + // Fetch app info again in case we didn't get it fully without a license. + steam3.RequestAppInfo(appId, true); + } + else + { + var contentName = GetAppOrDepotName(INVALID_DEPOT_ID, appId); + throw new ContentDownloaderException(String.Format("App {0} ({1}) is not available from this account.", appId, contentName)); + } + } + + var hasSpecificDepots = depotManifestIds.Count > 0; + var depotIdsFound = new List(); + var depotIdsExpected = depotManifestIds.Select(x => x.Item1).ToList(); + var depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); + + if (isUgc) + { + var workshopDepot = depots["workshopdepot"].AsUnsignedInteger(); + if (workshopDepot != 0 && !depotIdsExpected.Contains(workshopDepot)) + { + depotIdsExpected.Add(workshopDepot); + depotManifestIds = depotManifestIds.Select(pair => (workshopDepot, pair.manifestId)).ToList(); + } + + depotIdsFound.AddRange(depotIdsExpected); + } + else + { + Console.WriteLine("Using app branch: '{0}'.", branch); + + if (depots != null) + { + foreach (var depotSection in depots.Children) + { + var id = INVALID_DEPOT_ID; + if (depotSection.Children.Count == 0) + continue; + + if (!uint.TryParse(depotSection.Name, out id)) + continue; + + if (hasSpecificDepots && !depotIdsExpected.Contains(id)) + continue; + + if (!hasSpecificDepots) + { + var depotConfig = depotSection["config"]; + if (depotConfig != KeyValue.Invalid) + { + if (!Config.DownloadAllPlatforms && + depotConfig["oslist"] != KeyValue.Invalid && + !string.IsNullOrWhiteSpace(depotConfig["oslist"].Value)) + { + var oslist = depotConfig["oslist"].Value.Split(','); + if (Array.IndexOf(oslist, os ?? Util.GetSteamOS()) == -1) + continue; + } + + if (depotConfig["osarch"] != KeyValue.Invalid && + !string.IsNullOrWhiteSpace(depotConfig["osarch"].Value)) + { + var depotArch = depotConfig["osarch"].Value; + if (depotArch != (arch ?? Util.GetSteamArch())) + continue; + } + + if (!Config.DownloadAllLanguages && + depotConfig["language"] != KeyValue.Invalid && + !string.IsNullOrWhiteSpace(depotConfig["language"].Value)) + { + var depotLang = depotConfig["language"].Value; + if (depotLang != (language ?? "english")) + continue; + } + + if (!lv && + depotConfig["lowviolence"] != KeyValue.Invalid && + depotConfig["lowviolence"].AsBoolean()) + continue; + } + } + + depotIdsFound.Add(id); + + if (!hasSpecificDepots) + depotManifestIds.Add((id, INVALID_MANIFEST_ID)); + } + } + + if (depotManifestIds.Count == 0 && !hasSpecificDepots) + { + throw new ContentDownloaderException(String.Format("Couldn't find any depots to download for app {0}", appId)); + } + + if (depotIdsFound.Count < depotIdsExpected.Count) + { + var remainingDepotIds = depotIdsExpected.Except(depotIdsFound); + throw new ContentDownloaderException(String.Format("Depot {0} not listed for app {1}", string.Join(", ", remainingDepotIds), appId)); + } + } + + var infos = new List(); + + foreach (var depotManifest in depotManifestIds) + { + var info = GetDepotInfo(depotManifest.Item1, appId, depotManifest.Item2, branch); + if (info != null) + { + infos.Add(info); + } + } + + try + { + await DownloadSteam3Async(appId, infos).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + Console.WriteLine("App {0} was not completely downloaded.", appId); + throw; + } + } + + static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, ulong manifestId, string branch) + { + if (steam3 != null && appId != INVALID_APP_ID) + steam3.RequestAppInfo(appId); + + var contentName = GetAppOrDepotName(depotId, appId); + + if (!AccountHasAccess(depotId)) + { + Console.WriteLine("Depot {0} ({1}) is not available from this account.", depotId, contentName); + + return null; + } + + if (manifestId == INVALID_MANIFEST_ID) + { + manifestId = GetSteam3DepotManifest(depotId, appId, branch); + if (manifestId == INVALID_MANIFEST_ID && branch != "public") + { + Console.WriteLine("Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch); + branch = "public"; + manifestId = GetSteam3DepotManifest(depotId, appId, branch); + } + + if (manifestId == INVALID_MANIFEST_ID) + { + Console.WriteLine("Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName); + return null; + } + } + + var uVersion = GetSteam3AppBuildNumber(appId, branch); + + string installDir; + if (!CreateDirectories(depotId, uVersion, out installDir)) + { + Console.WriteLine("Error: Unable to create install directories!"); + return null; + } + + steam3.RequestDepotKey(depotId, appId); + if (!steam3.DepotKeys.ContainsKey(depotId)) + { + Console.WriteLine("No valid depot key for {0}, unable to download.", depotId); + return null; + } + + var depotKey = steam3.DepotKeys[depotId]; + + var info = new DepotDownloadInfo(depotId, manifestId, installDir, contentName); + info.depotKey = depotKey; + return info; + } + + private class ChunkMatch + { + public ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk) + { + OldChunk = oldChunk; + NewChunk = newChunk; + } + + public ProtoManifest.ChunkData OldChunk { get; private set; } + public ProtoManifest.ChunkData NewChunk { get; private set; } + } + + private class DepotFilesData + { + public DepotDownloadInfo depotDownloadInfo; + public DepotDownloadCounter depotCounter; + public string stagingDir; + public ProtoManifest manifest; + public ProtoManifest previousManifest; + public List filteredFiles; + public HashSet allFileNames; + } + + private class FileStreamData + { + public FileStream fileStream; + public SemaphoreSlim fileLock; + public int chunksToDownload; + } + + private class GlobalDownloadCounter + { + public ulong TotalBytesCompressed; + public ulong TotalBytesUncompressed; + } + + private class DepotDownloadCounter + { + public ulong CompleteDownloadSize; + public ulong SizeDownloaded; + public ulong DepotBytesCompressed; + public ulong DepotBytesUncompressed; + } + + private static async Task DownloadSteam3Async(uint appId, List depots) + { + var cts = new CancellationTokenSource(); + cdnPool.ExhaustedToken = cts; + + var downloadCounter = new GlobalDownloadCounter(); + var depotsToDownload = new List(depots.Count); + var allFileNamesAllDepots = new HashSet(); + + // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup + foreach (var depot in depots) + { + var depotFileData = await ProcessDepotManifestAndFiles(cts, appId, depot); + + if (depotFileData != null) + { + depotsToDownload.Add(depotFileData); + allFileNamesAllDepots.UnionWith(depotFileData.allFileNames); + } + + cts.Token.ThrowIfCancellationRequested(); + } + + // If we're about to write all the files to the same directory, we will need to first de-duplicate any files by path + // This is in last-depot-wins order, from Steam or the list of depots supplied by the user + if (!string.IsNullOrWhiteSpace(Config.InstallDirectory) && depotsToDownload.Count > 0) + { + var claimedFileNames = new HashSet(); + + for (var i = depotsToDownload.Count - 1; i >= 0; i--) + { + // For each depot, remove all files from the list that have been claimed by a later depot + depotsToDownload[i].filteredFiles.RemoveAll(file => claimedFileNames.Contains(file.FileName)); + + claimedFileNames.UnionWith(depotsToDownload[i].allFileNames); + } + } + + foreach (var depotFileData in depotsToDownload) + { + await DownloadSteam3AsyncDepotFiles(cts, appId, downloadCounter, depotFileData, allFileNamesAllDepots); + } + + Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", + downloadCounter.TotalBytesCompressed, downloadCounter.TotalBytesUncompressed, depots.Count); + } + + private static async Task ProcessDepotManifestAndFiles(CancellationTokenSource cts, + uint appId, DepotDownloadInfo depot) + { + var depotCounter = new DepotDownloadCounter(); + + Console.WriteLine("Processing depot {0} - {1}", depot.id, depot.contentName); + + ProtoManifest oldProtoManifest = null; + ProtoManifest newProtoManifest = null; + var configDir = Path.Combine(depot.installDir, CONFIG_DIR); + + var lastManifestId = INVALID_MANIFEST_ID; + DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId); + + // In case we have an early exit, this will force equiv of verifyall next run. + DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID; + DepotConfigStore.Save(); + + if (lastManifestId != INVALID_MANIFEST_ID) + { + var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId)); + + if (File.Exists(oldManifestFileName)) + { + byte[] expectedChecksum, currentChecksum; + + try + { + expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); + } + catch (IOException) + { + expectedChecksum = null; + } + + oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum); + + if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) + { + // We only have to show this warning if the old manifest ID was different + if (lastManifestId != depot.manifestId) + Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); + oldProtoManifest = null; + } + } + } + + if (lastManifestId == depot.manifestId && oldProtoManifest != null) + { + newProtoManifest = oldProtoManifest; + Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); + } + else + { + var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId)); + if (newManifestFileName != null) + { + byte[] expectedChecksum, currentChecksum; + + try + { + expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); + } + catch (IOException) + { + expectedChecksum = null; + } + + newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum); + + if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) + { + Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId); + newProtoManifest = null; + } + } + + if (newProtoManifest != null) + { + Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); + } + else + { + Console.Write("Downloading depot manifest..."); + + DepotManifest depotManifest = null; + + do + { + cts.Token.ThrowIfCancellationRequested(); + + CDNClient.Server connection = null; + + try + { + connection = cdnPool.GetConnection(cts.Token); + + DebugLog.WriteLine("ContentDownloader", "Authenticating connection to {0}", connection); + var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection); + + DebugLog.WriteLine("ContentDownloader", "Downloading manifest {0} from {1} with {2}", depot.manifestId, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); + depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId, + connection, cdnToken, depot.depotKey, proxyServer: cdnPool.ProxyServer).ConfigureAwait(false); + + cdnPool.ReturnConnection(connection); + } + catch (TaskCanceledException) + { + Console.WriteLine("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId); + } + catch (SteamKitWebRequestException e) + { + cdnPool.ReturnBrokenConnection(connection); + + if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) + { + Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); + break; + } + + if (e.StatusCode == HttpStatusCode.NotFound) + { + Console.WriteLine("Encountered 404 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); + break; + } + + Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception e) + { + cdnPool.ReturnBrokenConnection(connection); + Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message); + } + } while (depotManifest == null); + + if (depotManifest == null) + { + Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); + cts.Cancel(); + } + + // Throw the cancellation exception if requested so that this task is marked failed + cts.Token.ThrowIfCancellationRequested(); + + byte[] checksum; + + newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); + newProtoManifest.SaveToFile(newManifestFileName, out checksum); + File.WriteAllBytes(newManifestFileName + ".sha", checksum); + + Console.WriteLine(" Done!"); + } + } + + newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal)); + + Console.WriteLine("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime); + + if (Config.DownloadManifestOnly) + { + DumpManifestToTextFile(depot, newProtoManifest); + return null; + } + + var stagingDir = Path.Combine(depot.installDir, STAGING_DIR); + + var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); + var allFileNames = new HashSet(filesAfterExclusions.Count); + + // Pre-process + filesAfterExclusions.ForEach(file => + { + allFileNames.Add(file.FileName); + + var fileFinalPath = Path.Combine(depot.installDir, file.FileName); + var fileStagingPath = Path.Combine(stagingDir, file.FileName); + + if (file.Flags.HasFlag(EDepotFileFlag.Directory)) + { + Directory.CreateDirectory(fileFinalPath); + Directory.CreateDirectory(fileStagingPath); + } + else + { + // Some manifests don't explicitly include all necessary directories + Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); + Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); + + depotCounter.CompleteDownloadSize += file.TotalSize; + } + }); + + return new DepotFilesData + { + depotDownloadInfo = depot, + depotCounter = depotCounter, + stagingDir = stagingDir, + manifest = newProtoManifest, + previousManifest = oldProtoManifest, + filteredFiles = filesAfterExclusions, + allFileNames = allFileNames + }; + } + + private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource cts, uint appId, + GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, HashSet allFileNamesAllDepots) + { + var depot = depotFilesData.depotDownloadInfo; + var depotCounter = depotFilesData.depotCounter; + + Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName); + + var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); + var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>(); + + await Util.InvokeAsync( + files.Select(file => new Func(async () => + await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, depotFilesData, file, networkChunkQueue)))), + maxDegreeOfParallelism: Config.MaxDownloads + ); + + await Util.InvokeAsync( + networkChunkQueue.Select(q => new Func(async () => + await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, appId, downloadCounter, depotFilesData, + q.fileData, q.fileStreamData, q.chunk)))), + maxDegreeOfParallelism: Config.MaxDownloads + ); + + // Check for deleted files if updating the depot. + if (depotFilesData.previousManifest != null) + { + var previousFilteredFiles = depotFilesData.previousManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).Select(f => f.FileName).ToHashSet(); + + // Check if we are writing to a single output directory. If not, each depot folder is managed independently + if (string.IsNullOrWhiteSpace(Config.InstallDirectory)) + { + // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names + previousFilteredFiles.ExceptWith(depotFilesData.allFileNames); + } + else + { + // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names across all depots being downloaded + previousFilteredFiles.ExceptWith(allFileNamesAllDepots); + } + + foreach (var existingFileName in previousFilteredFiles) + { + var fileFinalPath = Path.Combine(depot.installDir, existingFileName); + + if (!File.Exists(fileFinalPath)) + continue; + + File.Delete(fileFinalPath); + Console.WriteLine("Deleted {0}", fileFinalPath); + } + } + + DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId; + DepotConfigStore.Save(); + + Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed); + } + + private static void DownloadSteam3AsyncDepotFile( + CancellationTokenSource cts, + DepotFilesData depotFilesData, + ProtoManifest.FileData file, + ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue) + { + cts.Token.ThrowIfCancellationRequested(); + + var depot = depotFilesData.depotDownloadInfo; + var stagingDir = depotFilesData.stagingDir; + var depotDownloadCounter = depotFilesData.depotCounter; + var oldProtoManifest = depotFilesData.previousManifest; + + var fileFinalPath = Path.Combine(depot.installDir, file.FileName); + var fileStagingPath = Path.Combine(stagingDir, file.FileName); + + // This may still exist if the previous run exited before cleanup + if (File.Exists(fileStagingPath)) + { + File.Delete(fileStagingPath); + } + + FileStream fs = null; + List neededChunks; + var fi = new FileInfo(fileFinalPath); + if (!fi.Exists) + { + Console.WriteLine("Pre-allocating {0}", fileFinalPath); + + // create new file. need all chunks + fs = File.Create(fileFinalPath); + try + { + fs.SetLength((long)file.TotalSize); + } + catch (IOException ex) + { + throw new ContentDownloaderException(String.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message)); + } + + neededChunks = new List(file.Chunks); + } + else + { + // open existing + ProtoManifest.FileData oldManifestFile = null; + if (oldProtoManifest != null) + { + oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); + } + + if (oldManifestFile != null) + { + neededChunks = new List(); + + var hashMatches = oldManifestFile.FileHash.SequenceEqual(file.FileHash); + if (Config.VerifyAll || !hashMatches) + { + // we have a version of this file, but it doesn't fully match what we want + if (Config.VerifyAll) + { + Console.WriteLine("Validating {0}", fileFinalPath); + } + + var matchingChunks = new List(); + + foreach (var chunk in file.Chunks) + { + var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); + if (oldChunk != null) + { + matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); + } + else + { + neededChunks.Add(chunk); + } + } + + var orderedChunks = matchingChunks.OrderBy(x => x.OldChunk.Offset); + + var copyChunks = new List(); + + using (var fsOld = File.Open(fileFinalPath, FileMode.Open)) + { + foreach (var match in orderedChunks) + { + fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); + + var tmp = new byte[match.OldChunk.UncompressedLength]; + fsOld.Read(tmp, 0, tmp.Length); + + var adler = Util.AdlerHash(tmp); + if (!adler.SequenceEqual(match.OldChunk.Checksum)) + { + neededChunks.Add(match.NewChunk); + } + else + { + copyChunks.Add(match); + } + } + } + + if (!hashMatches || neededChunks.Count > 0) + { + File.Move(fileFinalPath, fileStagingPath); + + using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) + { + fs = File.Open(fileFinalPath, FileMode.Create); + try + { + fs.SetLength((long)file.TotalSize); + } + catch (IOException ex) + { + throw new ContentDownloaderException(String.Format("Failed to resize file to expected size {0}: {1}", fileFinalPath, ex.Message)); + } + + foreach (var match in copyChunks) + { + fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); + + var tmp = new byte[match.OldChunk.UncompressedLength]; + fsOld.Read(tmp, 0, tmp.Length); + + fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin); + fs.Write(tmp, 0, tmp.Length); + } + } + + File.Delete(fileStagingPath); + } + } + } + else + { + // No old manifest or file not in old manifest. We must validate. + + fs = File.Open(fileFinalPath, FileMode.Open); + if ((ulong)fi.Length != file.TotalSize) + { + try + { + fs.SetLength((long)file.TotalSize); + } + catch (IOException ex) + { + throw new ContentDownloaderException(String.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message)); + } + } + + Console.WriteLine("Validating {0}", fileFinalPath); + neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); + } + + if (neededChunks.Count() == 0) + { + lock (depotDownloadCounter) + { + depotDownloadCounter.SizeDownloaded += file.TotalSize; + Console.WriteLine("{0,6:#00.00}% {1}", (depotDownloadCounter.SizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath); + } + + if (fs != null) + fs.Dispose(); + return; + } + + var sizeOnDisk = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum()); + lock (depotDownloadCounter) + { + depotDownloadCounter.SizeDownloaded += sizeOnDisk; + } + } + + var fileStreamData = new FileStreamData + { + fileStream = fs, + fileLock = new SemaphoreSlim(1), + chunksToDownload = neededChunks.Count + }; + + foreach (var chunk in neededChunks) + { + networkChunkQueue.Enqueue((fileStreamData, file, chunk)); + } + } + + private static async Task DownloadSteam3AsyncDepotFileChunk( + CancellationTokenSource cts, uint appId, + GlobalDownloadCounter downloadCounter, + DepotFilesData depotFilesData, + ProtoManifest.FileData file, + FileStreamData fileStreamData, + ProtoManifest.ChunkData chunk) + { + cts.Token.ThrowIfCancellationRequested(); + + var depot = depotFilesData.depotDownloadInfo; + var depotDownloadCounter = depotFilesData.depotCounter; + + var chunkID = Util.EncodeHexString(chunk.ChunkID); + + var data = new DepotManifest.ChunkData(); + data.ChunkID = chunk.ChunkID; + data.Checksum = chunk.Checksum; + data.Offset = chunk.Offset; + data.CompressedLength = chunk.CompressedLength; + data.UncompressedLength = chunk.UncompressedLength; + + CDNClient.DepotChunk chunkData = null; + + do + { + cts.Token.ThrowIfCancellationRequested(); + + CDNClient.Server connection = null; + + try + { + connection = cdnPool.GetConnection(cts.Token); + + DebugLog.WriteLine("ContentDownloader", "Authenticating connection to {0}", connection); + var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection); + + DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy"); + chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data, + connection, cdnToken, depot.depotKey, proxyServer: cdnPool.ProxyServer).ConfigureAwait(false); + + cdnPool.ReturnConnection(connection); + } + catch (TaskCanceledException) + { + Console.WriteLine("Connection timeout downloading chunk {0}", chunkID); + } + catch (SteamKitWebRequestException e) + { + cdnPool.ReturnBrokenConnection(connection); + + if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden) + { + Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); + break; + } + + Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception e) + { + cdnPool.ReturnBrokenConnection(connection); + Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); + } + } while (chunkData == null); + + if (chunkData == null) + { + Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id); + cts.Cancel(); + } + + // Throw the cancellation exception if requested so that this task is marked failed + cts.Token.ThrowIfCancellationRequested(); + + try + { + await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false); + + fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin); + await fileStreamData.fileStream.WriteAsync(chunkData.Data, 0, chunkData.Data.Length); + } + finally + { + fileStreamData.fileLock.Release(); + } + + var remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload); + if (remainingChunks == 0) + { + fileStreamData.fileStream.Dispose(); + fileStreamData.fileLock.Dispose(); + } + + ulong sizeDownloaded = 0; + lock (depotDownloadCounter) + { + sizeDownloaded = depotDownloadCounter.SizeDownloaded + (ulong)chunkData.Data.Length; + depotDownloadCounter.SizeDownloaded = sizeDownloaded; + depotDownloadCounter.DepotBytesCompressed += chunk.CompressedLength; + depotDownloadCounter.DepotBytesUncompressed += chunk.UncompressedLength; + } + + lock (downloadCounter) + { + downloadCounter.TotalBytesCompressed += chunk.CompressedLength; + downloadCounter.TotalBytesUncompressed += chunk.UncompressedLength; + } + + if (remainingChunks == 0) + { + var fileFinalPath = Path.Combine(depot.installDir, file.FileName); + Console.WriteLine("{0,6:#00.00}% {1}", (sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath); + } + } + + static void DumpManifestToTextFile(DepotDownloadInfo depot, ProtoManifest manifest) + { + var txtManifest = Path.Combine(depot.installDir, $"manifest_{depot.id}_{depot.manifestId}.txt"); + + using (var sw = new StreamWriter(txtManifest)) + { + sw.WriteLine($"Content Manifest for Depot {depot.id}"); + sw.WriteLine(); + sw.WriteLine($"Manifest ID / date : {depot.manifestId} / {manifest.CreationTime}"); + + int numFiles = 0, numChunks = 0; + ulong uncompressedSize = 0, compressedSize = 0; + + foreach (var file in manifest.Files) + { + if (file.Flags.HasFlag(EDepotFileFlag.Directory)) + continue; + + numFiles++; + numChunks += file.Chunks.Count; + + foreach (var chunk in file.Chunks) + { + uncompressedSize += chunk.UncompressedLength; + compressedSize += chunk.CompressedLength; + } + } + + sw.WriteLine($"Total number of files : {numFiles}"); + sw.WriteLine($"Total number of chunks : {numChunks}"); + sw.WriteLine($"Total bytes on disk : {uncompressedSize}"); + sw.WriteLine($"Total bytes compressed : {compressedSize}"); + sw.WriteLine(); + sw.WriteLine(" Size Chunks File SHA Flags Name"); + + foreach (var file in manifest.Files) + { + var sha1Hash = BitConverter.ToString(file.FileHash).Replace("-", ""); + sw.WriteLine($"{file.TotalSize,14} {file.Chunks.Count,6} {sha1Hash} {file.Flags,5:D} {file.FileName}"); + } + } + } + } +} diff --git a/DepotDownloader/DepotConfigStore.cs b/DepotDownloader/DepotConfigStore.cs index 79a414ab..2bd2bb59 100644 --- a/DepotDownloader/DepotConfigStore.cs +++ b/DepotDownloader/DepotConfigStore.cs @@ -1,8 +1,8 @@ using System; using System.Collections.Generic; -using ProtoBuf; using System.IO; using System.IO.Compression; +using ProtoBuf; namespace DepotDownloader { @@ -12,7 +12,7 @@ namespace DepotDownloader [ProtoMember(1)] public Dictionary InstalledManifestIDs { get; private set; } - string FileName = null; + string FileName; DepotConfigStore() { @@ -24,7 +24,7 @@ namespace DepotDownloader get { return Instance != null; } } - public static DepotConfigStore Instance = null; + public static DepotConfigStore Instance; public static void LoadFromFile(string filename) { @@ -33,9 +33,9 @@ namespace DepotDownloader if (File.Exists(filename)) { - using (FileStream fs = File.Open(filename, FileMode.Open)) - using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress)) - Instance = ProtoBuf.Serializer.Deserialize(ds); + using (var fs = File.Open(filename, FileMode.Open)) + using (var ds = new DeflateStream(fs, CompressionMode.Decompress)) + Instance = Serializer.Deserialize(ds); } else { @@ -50,9 +50,9 @@ namespace DepotDownloader if (!Loaded) throw new Exception("Saved config before loading"); - using (FileStream fs = File.Open(Instance.FileName, FileMode.Create)) - using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Compress)) - ProtoBuf.Serializer.Serialize(ds, Instance); + using (var fs = File.Open(Instance.FileName, FileMode.Create)) + using (var ds = new DeflateStream(fs, CompressionMode.Compress)) + Serializer.Serialize(ds, Instance); } } } diff --git a/DepotDownloader/DepotDownloader.csproj b/DepotDownloader/DepotDownloader.csproj index 8843126f..b4933488 100644 --- a/DepotDownloader/DepotDownloader.csproj +++ b/DepotDownloader/DepotDownloader.csproj @@ -2,6 +2,7 @@ Exe net5.0 + true 2.4.3 Steam Downloading Utility @@ -13,4 +14,4 @@ - \ No newline at end of file + diff --git a/DepotDownloader/DownloadConfig.cs b/DepotDownloader/DownloadConfig.cs index 14d1638c..64ec241e 100644 --- a/DepotDownloader/DownloadConfig.cs +++ b/DepotDownloader/DownloadConfig.cs @@ -26,6 +26,6 @@ namespace DepotDownloader public bool RememberPassword { get; set; } // A Steam LoginID to allow multiple concurrent connections - public uint? LoginID {get; set; } + public uint? LoginID { get; set; } } } diff --git a/DepotDownloader/HttpClientFactory.cs b/DepotDownloader/HttpClientFactory.cs index fc28888c..8067826b 100644 --- a/DepotDownloader/HttpClientFactory.cs +++ b/DepotDownloader/HttpClientFactory.cs @@ -1,11 +1,7 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; +using System.IO; using System.Net.Http; using System.Net.Http.Headers; using System.Net.Sockets; -using System.Text; using System.Threading; using System.Threading.Tasks; @@ -33,7 +29,7 @@ namespace DepotDownloader // By default, we create dual-mode sockets: // Socket socket = new Socket(SocketType.Stream, ProtocolType.Tcp); - Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); socket.NoDelay = true; try diff --git a/DepotDownloader/HttpDiagnosticEventListener.cs b/DepotDownloader/HttpDiagnosticEventListener.cs index eb883b05..31546b97 100644 --- a/DepotDownloader/HttpDiagnosticEventListener.cs +++ b/DepotDownloader/HttpDiagnosticEventListener.cs @@ -26,7 +26,7 @@ namespace DepotDownloader protected override void OnEventWritten(EventWrittenEventArgs eventData) { var sb = new StringBuilder().Append($"{eventData.TimeStamp:HH:mm:ss.fffffff} {eventData.EventSource.Name}.{eventData.EventName}("); - for (int i = 0; i < eventData.Payload?.Count; i++) + for (var i = 0; i < eventData.Payload?.Count; i++) { sb.Append(eventData.PayloadNames?[i]).Append(": ").Append(eventData.Payload[i]); if (i < eventData.Payload?.Count - 1) diff --git a/DepotDownloader/Program.cs b/DepotDownloader/Program.cs index aa3944e3..748ee5dc 100644 --- a/DepotDownloader/Program.cs +++ b/DepotDownloader/Program.cs @@ -1,397 +1,400 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text.RegularExpressions; -using SteamKit2; -using System.ComponentModel; -using System.Threading.Tasks; -using System.Linq; - -namespace DepotDownloader -{ - class Program - { - static int Main( string[] args ) - => MainAsync( args ).GetAwaiter().GetResult(); - - static async Task MainAsync( string[] args ) - { - if ( args.Length == 0 ) - { - PrintUsage(); - return 1; - } - - DebugLog.Enabled = false; - - AccountSettingsStore.LoadFromFile( "account.config" ); - - #region Common Options - - if ( HasParameter( args, "-debug" ) ) - { - DebugLog.Enabled = true; - DebugLog.AddListener( ( category, message ) => - { - Console.WriteLine( "[{0}] {1}", category, message ); - }); - - var httpEventListener = new HttpDiagnosticEventListener(); - } - - string username = GetParameter( args, "-username" ) ?? GetParameter( args, "-user" ); - string password = GetParameter( args, "-password" ) ?? GetParameter( args, "-pass" ); - ContentDownloader.Config.RememberPassword = HasParameter( args, "-remember-password" ); - - ContentDownloader.Config.DownloadManifestOnly = HasParameter( args, "-manifest-only" ); - - int cellId = GetParameter( args, "-cellid", -1 ); - if ( cellId == -1 ) - { - cellId = 0; - } - - ContentDownloader.Config.CellID = cellId; - - string fileList = GetParameter( args, "-filelist" ); - - if ( fileList != null ) - { - try - { - string fileListData = await File.ReadAllTextAsync( fileList ); - var files = fileListData.Split( new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries ); - - ContentDownloader.Config.UsingFileList = true; - ContentDownloader.Config.FilesToDownload = new HashSet( StringComparer.OrdinalIgnoreCase ); - ContentDownloader.Config.FilesToDownloadRegex = new List(); - - foreach ( var fileEntry in files ) - { - if ( fileEntry.StartsWith( "regex:" ) ) - { - Regex rgx = new Regex( fileEntry.Substring( 6 ), RegexOptions.Compiled | RegexOptions.IgnoreCase ); - ContentDownloader.Config.FilesToDownloadRegex.Add( rgx ); - } - else - { - ContentDownloader.Config.FilesToDownload.Add( fileEntry.Replace( '\\', '/' ) ); - } - } - - Console.WriteLine( "Using filelist: '{0}'.", fileList ); - } - catch (Exception ex) - { - Console.WriteLine( "Warning: Unable to load filelist: {0}", ex.ToString() ); - } - } - - ContentDownloader.Config.InstallDirectory = GetParameter( args, "-dir" ); - - ContentDownloader.Config.VerifyAll = HasParameter( args, "-verify-all" ) || HasParameter( args, "-verify_all" ) || HasParameter( args, "-validate" ); - ContentDownloader.Config.MaxServers = GetParameter( args, "-max-servers", 20 ); - ContentDownloader.Config.MaxDownloads = GetParameter( args, "-max-downloads", 8 ); - ContentDownloader.Config.MaxServers = Math.Max( ContentDownloader.Config.MaxServers, ContentDownloader.Config.MaxDownloads ); - ContentDownloader.Config.LoginID = HasParameter( args, "-loginid" ) ? (uint?)GetParameter( args, "-loginid" ) : null; - - #endregion - - uint appId = GetParameter( args, "-app", ContentDownloader.INVALID_APP_ID ); - if ( appId == ContentDownloader.INVALID_APP_ID ) - { - Console.WriteLine( "Error: -app not specified!" ); - return 1; - } - - ulong pubFile = GetParameter( args, "-pubfile", ContentDownloader.INVALID_MANIFEST_ID ); - ulong ugcId = GetParameter( args, "-ugc", ContentDownloader.INVALID_MANIFEST_ID ); - if ( pubFile != ContentDownloader.INVALID_MANIFEST_ID ) - { - #region Pubfile Downloading - - if ( InitializeSteam( username, password ) ) - { - try - { - await ContentDownloader.DownloadPubfileAsync( appId, pubFile ).ConfigureAwait( false ); - } - catch ( Exception ex ) when ( - ex is ContentDownloaderException - || ex is OperationCanceledException ) - { - Console.WriteLine( ex.Message ); - return 1; - } - catch ( Exception e ) - { - Console.WriteLine( "Download failed to due to an unhandled exception: {0}", e.Message ); - throw; - } - finally - { - ContentDownloader.ShutdownSteam3(); - } - } - else - { - Console.WriteLine( "Error: InitializeSteam failed" ); - return 1; - } - - #endregion - } - else if ( ugcId != ContentDownloader.INVALID_MANIFEST_ID ) - { - #region UGC Downloading - - if ( InitializeSteam( username, password ) ) - { - try - { - await ContentDownloader.DownloadUGCAsync( appId, ugcId ).ConfigureAwait( false ); - } - catch ( Exception ex ) when ( - ex is ContentDownloaderException - || ex is OperationCanceledException ) - { - Console.WriteLine( ex.Message ); - return 1; - } - catch ( Exception e ) - { - Console.WriteLine( "Download failed to due to an unhandled exception: {0}", e.Message ); - throw; - } - finally - { - ContentDownloader.ShutdownSteam3(); - } - } - else - { - Console.WriteLine( "Error: InitializeSteam failed" ); - return 1; - } - - #endregion - } - else - { - #region App downloading - - string branch = GetParameter( args, "-branch" ) ?? GetParameter( args, "-beta" ) ?? ContentDownloader.DEFAULT_BRANCH; - ContentDownloader.Config.BetaPassword = GetParameter( args, "-betapassword" ); - - ContentDownloader.Config.DownloadAllPlatforms = HasParameter( args, "-all-platforms" ); - string os = GetParameter( args, "-os", null ); - - if ( ContentDownloader.Config.DownloadAllPlatforms && !String.IsNullOrEmpty( os ) ) - { - Console.WriteLine("Error: Cannot specify -os when -all-platforms is specified."); - return 1; - } - - string arch = GetParameter( args, "-osarch", null ); - - ContentDownloader.Config.DownloadAllLanguages = HasParameter( args, "-all-languages" ); - string language = GetParameter( args, "-language", null ); - - if ( ContentDownloader.Config.DownloadAllLanguages && !String.IsNullOrEmpty( language ) ) - { - Console.WriteLine( "Error: Cannot specify -language when -all-languages is specified." ); - return 1; - } - - bool lv = HasParameter( args, "-lowviolence" ); - - List<(uint, ulong)> depotManifestIds = new List<(uint, ulong)>(); - bool isUGC = false; - - List depotIdList = GetParameterList( args, "-depot" ); - List manifestIdList = GetParameterList( args, "-manifest" ); - if ( manifestIdList.Count > 0 ) - { - if ( depotIdList.Count != manifestIdList.Count ) - { - Console.WriteLine( "Error: -manifest requires one id for every -depot specified" ); - return 1; - } - - var zippedDepotManifest = depotIdList.Zip( manifestIdList, ( depotId, manifestId ) => ( depotId, manifestId ) ); - depotManifestIds.AddRange( zippedDepotManifest ); - } - else - { - depotManifestIds.AddRange( depotIdList.Select( depotId => ( depotId, ContentDownloader.INVALID_MANIFEST_ID ) ) ); - } - - if ( InitializeSteam( username, password ) ) - { - try - { - await ContentDownloader.DownloadAppAsync( appId, depotManifestIds, branch, os, arch, language, lv, isUGC ).ConfigureAwait( false ); - } - catch ( Exception ex ) when ( - ex is ContentDownloaderException - || ex is OperationCanceledException ) - { - Console.WriteLine( ex.Message ); - return 1; - } - catch ( Exception e ) - { - Console.WriteLine( "Download failed to due to an unhandled exception: {0}", e.Message ); - throw; - } - finally - { - ContentDownloader.ShutdownSteam3(); - } - } - else - { - Console.WriteLine( "Error: InitializeSteam failed" ); - return 1; - } - - #endregion - } - - return 0; - } - - static bool InitializeSteam( string username, string password ) - { - if ( username != null && password == null && ( !ContentDownloader.Config.RememberPassword || !AccountSettingsStore.Instance.LoginKeys.ContainsKey( username ) ) ) - { - do - { - Console.Write( "Enter account password for \"{0}\": ", username ); - if ( Console.IsInputRedirected ) - { - password = Console.ReadLine(); - } - else - { - // Avoid console echoing of password - password = Util.ReadPassword(); - } - Console.WriteLine(); - } while ( String.Empty == password ); - } - else if ( username == null ) - { - Console.WriteLine( "No username given. Using anonymous account with dedicated server subscription." ); - } - - // capture the supplied password in case we need to re-use it after checking the login key - ContentDownloader.Config.SuppliedPassword = password; - - return ContentDownloader.InitializeSteam3( username, password ); - } - - static int IndexOfParam( string[] args, string param ) - { - for ( int x = 0; x < args.Length; ++x ) - { - if ( args[ x ].Equals( param, StringComparison.OrdinalIgnoreCase ) ) - return x; - } - return -1; - } - static bool HasParameter( string[] args, string param ) - { - return IndexOfParam( args, param ) > -1; - } - - static T GetParameter( string[] args, string param, T defaultValue = default( T ) ) - { - int index = IndexOfParam( args, param ); - - if ( index == -1 || index == ( args.Length - 1 ) ) - return defaultValue; - - string strParam = args[ index + 1 ]; - - var converter = TypeDescriptor.GetConverter( typeof( T ) ); - if ( converter != null ) - { - return ( T )converter.ConvertFromString( strParam ); - } - - return default( T ); - } - - static List GetParameterList(string[] args, string param) - { - List list = new List(); - int index = IndexOfParam(args, param); - - if (index == -1 || index == (args.Length - 1)) - return list; - - index++; - - while (index < args.Length) - { - string strParam = args[index]; - - if (strParam[0] == '-') break; - - var converter = TypeDescriptor.GetConverter(typeof(T)); - if (converter != null) - { - list.Add((T)converter.ConvertFromString(strParam)); - } - - index++; - } - - return list; - } - - static void PrintUsage() - { - Console.WriteLine(); - Console.WriteLine( "Usage - downloading one or all depots for an app:" ); - Console.WriteLine( "\tdepotdownloader -app [-depot [-manifest ]]" ); - Console.WriteLine( "\t\t[-username [-password ]] [other options]" ); - Console.WriteLine(); - Console.WriteLine("Usage - downloading a workshop item using pubfile id"); - Console.WriteLine( "\tdepotdownloader -app -pubfile [-username [-password ]]" ); - Console.WriteLine("Usage - downloading a workshop item using ugc id"); - Console.WriteLine("\tdepotdownloader -app -ugc [-username [-password ]]"); - Console.WriteLine(); - Console.WriteLine( "Parameters:" ); - Console.WriteLine( "\t-app <#>\t\t\t\t- the AppID to download." ); - Console.WriteLine( "\t-depot <#>\t\t\t\t- the DepotID to download." ); - Console.WriteLine( "\t-manifest \t\t\t- manifest id of content to download (requires -depot, default: current for branch)." ); - Console.WriteLine( "\t-beta \t\t\t- download from specified branch if available (default: Public)." ); - Console.WriteLine( "\t-betapassword \t\t- branch password if applicable." ); - Console.WriteLine( "\t-all-platforms\t\t\t- downloads all platform-specific depots when -app is used." ); - Console.WriteLine( "\t-os \t\t\t\t- the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)" ); - Console.WriteLine( "\t-osarch \t\t\t\t- the architecture for which to download the game (32 or 64, default: the host's architecture)" ); - Console.WriteLine( "\t-all-languages\t\t\t\t- download all language-specific depots when -app is used." ); - Console.WriteLine( "\t-language \t\t\t\t- the language for which to download the game (default: english)" ); - Console.WriteLine( "\t-lowviolence\t\t\t\t- download low violence depots when -app is used." ); - Console.WriteLine(); - Console.WriteLine( "\t-ugc <#>\t\t\t\t- the UGC ID to download." ); - Console.WriteLine( "\t-pubfile <#>\t\t\t- the PublishedFileId to download. (Will automatically resolve to UGC id)" ); - Console.WriteLine(); - Console.WriteLine( "\t-username \t\t- the username of the account to login to for restricted content."); - Console.WriteLine( "\t-password \t\t- the password of the account to login to for restricted content." ); - Console.WriteLine( "\t-remember-password\t\t- if set, remember the password for subsequent logins of this user." ); - Console.WriteLine(); - Console.WriteLine( "\t-dir \t\t- the directory in which to place downloaded files." ); - Console.WriteLine( "\t-filelist \t- a list of files to download (from the manifest). Prefix file path with 'regex:' if you want to match with regex." ); - Console.WriteLine( "\t-validate\t\t\t\t- Include checksum verification of files already downloaded" ); - Console.WriteLine(); - Console.WriteLine( "\t-manifest-only\t\t\t- downloads a human readable manifest for any depots that would be downloaded." ); - Console.WriteLine( "\t-cellid <#>\t\t\t\t- the overridden CellID of the content server to download from." ); - Console.WriteLine( "\t-max-servers <#>\t\t- maximum number of content servers to use. (default: 20)." ); - Console.WriteLine( "\t-max-downloads <#>\t\t- maximum number of chunks to download concurrently. (default: 8)." ); - Console.WriteLine( "\t-loginid <#>\t\t- a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently." ); - } - } -} +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using SteamKit2; + +namespace DepotDownloader +{ + class Program + { + static int Main(string[] args) + => MainAsync(args).GetAwaiter().GetResult(); + + static async Task MainAsync(string[] args) + { + if (args.Length == 0) + { + PrintUsage(); + return 1; + } + + DebugLog.Enabled = false; + + AccountSettingsStore.LoadFromFile("account.config"); + + #region Common Options + + if (HasParameter(args, "-debug")) + { + DebugLog.Enabled = true; + DebugLog.AddListener((category, message) => + { + Console.WriteLine("[{0}] {1}", category, message); + }); + + var httpEventListener = new HttpDiagnosticEventListener(); + } + + var username = GetParameter(args, "-username") ?? GetParameter(args, "-user"); + var password = GetParameter(args, "-password") ?? GetParameter(args, "-pass"); + ContentDownloader.Config.RememberPassword = HasParameter(args, "-remember-password"); + + ContentDownloader.Config.DownloadManifestOnly = HasParameter(args, "-manifest-only"); + + var cellId = GetParameter(args, "-cellid", -1); + if (cellId == -1) + { + cellId = 0; + } + + ContentDownloader.Config.CellID = cellId; + + var fileList = GetParameter(args, "-filelist"); + + if (fileList != null) + { + try + { + var fileListData = await File.ReadAllTextAsync(fileList); + var files = fileListData.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries); + + ContentDownloader.Config.UsingFileList = true; + ContentDownloader.Config.FilesToDownload = new HashSet(StringComparer.OrdinalIgnoreCase); + ContentDownloader.Config.FilesToDownloadRegex = new List(); + + foreach (var fileEntry in files) + { + if (fileEntry.StartsWith("regex:")) + { + var rgx = new Regex(fileEntry.Substring(6), RegexOptions.Compiled | RegexOptions.IgnoreCase); + ContentDownloader.Config.FilesToDownloadRegex.Add(rgx); + } + else + { + ContentDownloader.Config.FilesToDownload.Add(fileEntry.Replace('\\', '/')); + } + } + + Console.WriteLine("Using filelist: '{0}'.", fileList); + } + catch (Exception ex) + { + Console.WriteLine("Warning: Unable to load filelist: {0}", ex); + } + } + + ContentDownloader.Config.InstallDirectory = GetParameter(args, "-dir"); + + ContentDownloader.Config.VerifyAll = HasParameter(args, "-verify-all") || HasParameter(args, "-verify_all") || HasParameter(args, "-validate"); + ContentDownloader.Config.MaxServers = GetParameter(args, "-max-servers", 20); + ContentDownloader.Config.MaxDownloads = GetParameter(args, "-max-downloads", 8); + ContentDownloader.Config.MaxServers = Math.Max(ContentDownloader.Config.MaxServers, ContentDownloader.Config.MaxDownloads); + ContentDownloader.Config.LoginID = HasParameter(args, "-loginid") ? GetParameter(args, "-loginid") : null; + + #endregion + + var appId = GetParameter(args, "-app", ContentDownloader.INVALID_APP_ID); + if (appId == ContentDownloader.INVALID_APP_ID) + { + Console.WriteLine("Error: -app not specified!"); + return 1; + } + + var pubFile = GetParameter(args, "-pubfile", ContentDownloader.INVALID_MANIFEST_ID); + var ugcId = GetParameter(args, "-ugc", ContentDownloader.INVALID_MANIFEST_ID); + if (pubFile != ContentDownloader.INVALID_MANIFEST_ID) + { + #region Pubfile Downloading + + if (InitializeSteam(username, password)) + { + try + { + await ContentDownloader.DownloadPubfileAsync(appId, pubFile).ConfigureAwait(false); + } + catch (Exception ex) when ( + ex is ContentDownloaderException + || ex is OperationCanceledException) + { + Console.WriteLine(ex.Message); + return 1; + } + catch (Exception e) + { + Console.WriteLine("Download failed to due to an unhandled exception: {0}", e.Message); + throw; + } + finally + { + ContentDownloader.ShutdownSteam3(); + } + } + else + { + Console.WriteLine("Error: InitializeSteam failed"); + return 1; + } + + #endregion + } + else if (ugcId != ContentDownloader.INVALID_MANIFEST_ID) + { + #region UGC Downloading + + if (InitializeSteam(username, password)) + { + try + { + await ContentDownloader.DownloadUGCAsync(appId, ugcId).ConfigureAwait(false); + } + catch (Exception ex) when ( + ex is ContentDownloaderException + || ex is OperationCanceledException) + { + Console.WriteLine(ex.Message); + return 1; + } + catch (Exception e) + { + Console.WriteLine("Download failed to due to an unhandled exception: {0}", e.Message); + throw; + } + finally + { + ContentDownloader.ShutdownSteam3(); + } + } + else + { + Console.WriteLine("Error: InitializeSteam failed"); + return 1; + } + + #endregion + } + else + { + #region App downloading + + var branch = GetParameter(args, "-branch") ?? GetParameter(args, "-beta") ?? ContentDownloader.DEFAULT_BRANCH; + ContentDownloader.Config.BetaPassword = GetParameter(args, "-betapassword"); + + ContentDownloader.Config.DownloadAllPlatforms = HasParameter(args, "-all-platforms"); + var os = GetParameter(args, "-os"); + + if (ContentDownloader.Config.DownloadAllPlatforms && !String.IsNullOrEmpty(os)) + { + Console.WriteLine("Error: Cannot specify -os when -all-platforms is specified."); + return 1; + } + + var arch = GetParameter(args, "-osarch"); + + ContentDownloader.Config.DownloadAllLanguages = HasParameter(args, "-all-languages"); + var language = GetParameter(args, "-language"); + + if (ContentDownloader.Config.DownloadAllLanguages && !String.IsNullOrEmpty(language)) + { + Console.WriteLine("Error: Cannot specify -language when -all-languages is specified."); + return 1; + } + + var lv = HasParameter(args, "-lowviolence"); + + var depotManifestIds = new List<(uint, ulong)>(); + var isUGC = false; + + var depotIdList = GetParameterList(args, "-depot"); + var manifestIdList = GetParameterList(args, "-manifest"); + if (manifestIdList.Count > 0) + { + if (depotIdList.Count != manifestIdList.Count) + { + Console.WriteLine("Error: -manifest requires one id for every -depot specified"); + return 1; + } + + var zippedDepotManifest = depotIdList.Zip(manifestIdList, (depotId, manifestId) => (depotId, manifestId)); + depotManifestIds.AddRange(zippedDepotManifest); + } + else + { + depotManifestIds.AddRange(depotIdList.Select(depotId => (depotId, ContentDownloader.INVALID_MANIFEST_ID))); + } + + if (InitializeSteam(username, password)) + { + try + { + await ContentDownloader.DownloadAppAsync(appId, depotManifestIds, branch, os, arch, language, lv, isUGC).ConfigureAwait(false); + } + catch (Exception ex) when ( + ex is ContentDownloaderException + || ex is OperationCanceledException) + { + Console.WriteLine(ex.Message); + return 1; + } + catch (Exception e) + { + Console.WriteLine("Download failed to due to an unhandled exception: {0}", e.Message); + throw; + } + finally + { + ContentDownloader.ShutdownSteam3(); + } + } + else + { + Console.WriteLine("Error: InitializeSteam failed"); + return 1; + } + + #endregion + } + + return 0; + } + + static bool InitializeSteam(string username, string password) + { + if (username != null && password == null && (!ContentDownloader.Config.RememberPassword || !AccountSettingsStore.Instance.LoginKeys.ContainsKey(username))) + { + do + { + Console.Write("Enter account password for \"{0}\": ", username); + if (Console.IsInputRedirected) + { + password = Console.ReadLine(); + } + else + { + // Avoid console echoing of password + password = Util.ReadPassword(); + } + + Console.WriteLine(); + } while (String.Empty == password); + } + else if (username == null) + { + Console.WriteLine("No username given. Using anonymous account with dedicated server subscription."); + } + + // capture the supplied password in case we need to re-use it after checking the login key + ContentDownloader.Config.SuppliedPassword = password; + + return ContentDownloader.InitializeSteam3(username, password); + } + + static int IndexOfParam(string[] args, string param) + { + for (var x = 0; x < args.Length; ++x) + { + if (args[x].Equals(param, StringComparison.OrdinalIgnoreCase)) + return x; + } + + return -1; + } + + static bool HasParameter(string[] args, string param) + { + return IndexOfParam(args, param) > -1; + } + + static T GetParameter(string[] args, string param, T defaultValue = default(T)) + { + var index = IndexOfParam(args, param); + + if (index == -1 || index == (args.Length - 1)) + return defaultValue; + + var strParam = args[index + 1]; + + var converter = TypeDescriptor.GetConverter(typeof(T)); + if (converter != null) + { + return (T)converter.ConvertFromString(strParam); + } + + return default(T); + } + + static List GetParameterList(string[] args, string param) + { + var list = new List(); + var index = IndexOfParam(args, param); + + if (index == -1 || index == (args.Length - 1)) + return list; + + index++; + + while (index < args.Length) + { + var strParam = args[index]; + + if (strParam[0] == '-') break; + + var converter = TypeDescriptor.GetConverter(typeof(T)); + if (converter != null) + { + list.Add((T)converter.ConvertFromString(strParam)); + } + + index++; + } + + return list; + } + + static void PrintUsage() + { + Console.WriteLine(); + Console.WriteLine("Usage - downloading one or all depots for an app:"); + Console.WriteLine("\tdepotdownloader -app [-depot [-manifest ]]"); + Console.WriteLine("\t\t[-username [-password ]] [other options]"); + Console.WriteLine(); + Console.WriteLine("Usage - downloading a workshop item using pubfile id"); + Console.WriteLine("\tdepotdownloader -app -pubfile [-username [-password ]]"); + Console.WriteLine("Usage - downloading a workshop item using ugc id"); + Console.WriteLine("\tdepotdownloader -app -ugc [-username [-password ]]"); + Console.WriteLine(); + Console.WriteLine("Parameters:"); + Console.WriteLine("\t-app <#>\t\t\t\t- the AppID to download."); + Console.WriteLine("\t-depot <#>\t\t\t\t- the DepotID to download."); + Console.WriteLine("\t-manifest \t\t\t- manifest id of content to download (requires -depot, default: current for branch)."); + Console.WriteLine("\t-beta \t\t\t- download from specified branch if available (default: Public)."); + Console.WriteLine("\t-betapassword \t\t- branch password if applicable."); + Console.WriteLine("\t-all-platforms\t\t\t- downloads all platform-specific depots when -app is used."); + Console.WriteLine("\t-os \t\t\t\t- the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)"); + Console.WriteLine("\t-osarch \t\t\t\t- the architecture for which to download the game (32 or 64, default: the host's architecture)"); + Console.WriteLine("\t-all-languages\t\t\t\t- download all language-specific depots when -app is used."); + Console.WriteLine("\t-language \t\t\t\t- the language for which to download the game (default: english)"); + Console.WriteLine("\t-lowviolence\t\t\t\t- download low violence depots when -app is used."); + Console.WriteLine(); + Console.WriteLine("\t-ugc <#>\t\t\t\t- the UGC ID to download."); + Console.WriteLine("\t-pubfile <#>\t\t\t- the PublishedFileId to download. (Will automatically resolve to UGC id)"); + Console.WriteLine(); + Console.WriteLine("\t-username \t\t- the username of the account to login to for restricted content."); + Console.WriteLine("\t-password \t\t- the password of the account to login to for restricted content."); + Console.WriteLine("\t-remember-password\t\t- if set, remember the password for subsequent logins of this user."); + Console.WriteLine(); + Console.WriteLine("\t-dir \t\t- the directory in which to place downloaded files."); + Console.WriteLine("\t-filelist \t- a list of files to download (from the manifest). Prefix file path with 'regex:' if you want to match with regex."); + Console.WriteLine("\t-validate\t\t\t\t- Include checksum verification of files already downloaded"); + Console.WriteLine(); + Console.WriteLine("\t-manifest-only\t\t\t- downloads a human readable manifest for any depots that would be downloaded."); + Console.WriteLine("\t-cellid <#>\t\t\t\t- the overridden CellID of the content server to download from."); + Console.WriteLine("\t-max-servers <#>\t\t- maximum number of content servers to use. (default: 20)."); + Console.WriteLine("\t-max-downloads <#>\t\t- maximum number of chunks to download concurrently. (default: 8)."); + Console.WriteLine("\t-loginid <#>\t\t- a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently."); + } + } +} diff --git a/DepotDownloader/ProtoManifest.cs b/DepotDownloader/ProtoManifest.cs index f521c5cc..70389a05 100644 --- a/DepotDownloader/ProtoManifest.cs +++ b/DepotDownloader/ProtoManifest.cs @@ -2,13 +2,12 @@ using System.Collections.Generic; using System.IO; using System.IO.Compression; - using ProtoBuf; using SteamKit2; namespace DepotDownloader { - [ProtoContract()] + [ProtoContract] class ProtoManifest { // Proto ctor @@ -24,7 +23,7 @@ namespace DepotDownloader CreationTime = sourceManifest.CreationTime; } - [ProtoContract()] + [ProtoContract] public class FileData { // Proto ctor @@ -130,32 +129,31 @@ namespace DepotDownloader return null; } - using (MemoryStream ms = new MemoryStream()) + using (var ms = new MemoryStream()) { - using (FileStream fs = File.Open(filename, FileMode.Open)) - using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress)) + using (var fs = File.Open(filename, FileMode.Open)) + using (var ds = new DeflateStream(fs, CompressionMode.Decompress)) ds.CopyTo(ms); checksum = Util.SHAHash(ms.ToArray()); ms.Seek(0, SeekOrigin.Begin); - return ProtoBuf.Serializer.Deserialize(ms); + return Serializer.Deserialize(ms); } } public void SaveToFile(string filename, out byte[] checksum) { - - using (MemoryStream ms = new MemoryStream()) + using (var ms = new MemoryStream()) { - ProtoBuf.Serializer.Serialize(ms, this); + Serializer.Serialize(ms, this); checksum = Util.SHAHash(ms.ToArray()); ms.Seek(0, SeekOrigin.Begin); - using (FileStream fs = File.Open(filename, FileMode.Create)) - using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Compress)) + using (var fs = File.Open(filename, FileMode.Create)) + using (var ds = new DeflateStream(fs, CompressionMode.Compress)) ms.CopyTo(ds); } } diff --git a/DepotDownloader/Steam3Session.cs b/DepotDownloader/Steam3Session.cs index 72cb8f89..0b5fa07a 100644 --- a/DepotDownloader/Steam3Session.cs +++ b/DepotDownloader/Steam3Session.cs @@ -1,740 +1,740 @@ -using SteamKit2; -using SteamKit2.Internal; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace DepotDownloader -{ - - class Steam3Session - { - public class Credentials - { - public bool LoggedOn { get; set; } - public ulong SessionToken { get; set; } - - public bool IsValid - { - get { return LoggedOn; } - } - } - - public ReadOnlyCollection Licenses - { - get; - private set; - } - - public Dictionary AppTokens { get; private set; } - public Dictionary PackageTokens { get; private set; } - public Dictionary DepotKeys { get; private set; } - public ConcurrentDictionary> CDNAuthTokens { get; private set; } - public Dictionary AppInfo { get; private set; } - public Dictionary PackageInfo { get; private set; } - public Dictionary AppBetaPasswords { get; private set; } - - public SteamClient steamClient; - public SteamUser steamUser; - SteamApps steamApps; - SteamCloud steamCloud; - SteamUnifiedMessages.UnifiedService steamPublishedFile; - - CallbackManager callbacks; - - bool authenticatedUser; - bool bConnected; - bool bConnecting; - bool bAborted; - bool bExpectingDisconnectRemote; - bool bDidDisconnect; - bool bDidReceiveLoginKey; - bool bIsConnectionRecovery; - int connectionBackoff; - int seq; // more hack fixes - DateTime connectTime; - - // input - SteamUser.LogOnDetails logonDetails; - - // output - Credentials credentials; - - static readonly TimeSpan STEAM3_TIMEOUT = TimeSpan.FromSeconds( 30 ); - - - public Steam3Session( SteamUser.LogOnDetails details ) - { - this.logonDetails = details; - - this.authenticatedUser = details.Username != null; - this.credentials = new Credentials(); - this.bConnected = false; - this.bConnecting = false; - this.bAborted = false; - this.bExpectingDisconnectRemote = false; - this.bDidDisconnect = false; - this.bDidReceiveLoginKey = false; - this.seq = 0; - - this.AppTokens = new Dictionary(); - this.PackageTokens = new Dictionary(); - this.DepotKeys = new Dictionary(); - this.CDNAuthTokens = new ConcurrentDictionary>(); - this.AppInfo = new Dictionary(); - this.PackageInfo = new Dictionary(); - this.AppBetaPasswords = new Dictionary(); - - var clientConfiguration = SteamConfiguration.Create(config => - config - .WithHttpClientFactory(HttpClientFactory.CreateHttpClient) - ); - - this.steamClient = new SteamClient(clientConfiguration); - - this.steamUser = this.steamClient.GetHandler(); - this.steamApps = this.steamClient.GetHandler(); - this.steamCloud = this.steamClient.GetHandler(); - var steamUnifiedMessages = this.steamClient.GetHandler(); - this.steamPublishedFile = steamUnifiedMessages.CreateService(); - - this.callbacks = new CallbackManager( this.steamClient ); - - this.callbacks.Subscribe( ConnectedCallback ); - this.callbacks.Subscribe( DisconnectedCallback ); - this.callbacks.Subscribe( LogOnCallback ); - this.callbacks.Subscribe( SessionTokenCallback ); - this.callbacks.Subscribe( LicenseListCallback ); - this.callbacks.Subscribe( UpdateMachineAuthCallback ); - this.callbacks.Subscribe( LoginKeyCallback ); - - Console.Write( "Connecting to Steam3..." ); - - if ( authenticatedUser ) - { - FileInfo fi = new FileInfo( String.Format( "{0}.sentryFile", logonDetails.Username ) ); - if ( AccountSettingsStore.Instance.SentryData != null && AccountSettingsStore.Instance.SentryData.ContainsKey( logonDetails.Username ) ) - { - logonDetails.SentryFileHash = Util.SHAHash( AccountSettingsStore.Instance.SentryData[ logonDetails.Username ] ); - } - else if ( fi.Exists && fi.Length > 0 ) - { - var sentryData = File.ReadAllBytes( fi.FullName ); - logonDetails.SentryFileHash = Util.SHAHash( sentryData ); - AccountSettingsStore.Instance.SentryData[ logonDetails.Username ] = sentryData; - AccountSettingsStore.Save(); - } - } - - Connect(); - } - - public delegate bool WaitCondition(); - private object steamLock = new object(); - - public bool WaitUntilCallback( Action submitter, WaitCondition waiter ) - { - while ( !bAborted && !waiter() ) - { - lock (steamLock) - { - submitter(); - } - - int seq = this.seq; - do - { - lock (steamLock) - { - WaitForCallbacks(); - } - } - while ( !bAborted && this.seq == seq && !waiter() ); - } - - return bAborted; - } - - public Credentials WaitForCredentials() - { - if ( credentials.IsValid || bAborted ) - return credentials; - - WaitUntilCallback( () => { }, () => { return credentials.IsValid; } ); - - return credentials; - } - - public void RequestAppInfo( uint appId, bool bForce = false ) - { - if ( ( AppInfo.ContainsKey( appId ) && !bForce ) || bAborted ) - return; - - bool completed = false; - Action cbMethodTokens = ( appTokens ) => - { - completed = true; - if ( appTokens.AppTokensDenied.Contains( appId ) ) - { - Console.WriteLine( "Insufficient privileges to get access token for app {0}", appId ); - } - - foreach ( var token_dict in appTokens.AppTokens ) - { - this.AppTokens[ token_dict.Key ] = token_dict.Value; - } - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.PICSGetAccessTokens( new List() { appId }, new List() { } ), cbMethodTokens ); - }, () => { return completed; } ); - - completed = false; - Action cbMethod = ( appInfo ) => - { - completed = !appInfo.ResponsePending; - - foreach ( var app_value in appInfo.Apps ) - { - var app = app_value.Value; - - Console.WriteLine( "Got AppInfo for {0}", app.ID ); - AppInfo[ app.ID ] = app; - } - - foreach ( var app in appInfo.UnknownApps ) - { - AppInfo[ app ] = null; - } - }; - - SteamApps.PICSRequest request = new SteamApps.PICSRequest( appId ); - if ( AppTokens.ContainsKey( appId ) ) - { - request.AccessToken = AppTokens[ appId ]; - request.Public = false; - } - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.PICSGetProductInfo( new List() { request }, new List() { } ), cbMethod ); - }, () => { return completed; } ); - } - - public void RequestPackageInfo( IEnumerable packageIds ) - { - List packages = packageIds.ToList(); - packages.RemoveAll( pid => PackageInfo.ContainsKey( pid ) ); - - if ( packages.Count == 0 || bAborted ) - return; - - bool completed = false; - Action cbMethod = ( packageInfo ) => - { - completed = !packageInfo.ResponsePending; - - foreach ( var package_value in packageInfo.Packages ) - { - var package = package_value.Value; - PackageInfo[ package.ID ] = package; - } - - foreach ( var package in packageInfo.UnknownPackages ) - { - PackageInfo[package] = null; - } - }; - - var packageRequests = new List(); - - foreach ( var package in packages ) - { - var request = new SteamApps.PICSRequest( package ); - - if ( PackageTokens.TryGetValue( package, out var token ) ) - { - request.AccessToken = token; - request.Public = false; - } - - packageRequests.Add( request ); - } - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.PICSGetProductInfo( new List(), packageRequests ), cbMethod ); - }, () => { return completed; } ); - } - - public bool RequestFreeAppLicense( uint appId ) - { - bool success = false; - bool completed = false; - Action cbMethod = ( resultInfo ) => - { - completed = true; - success = resultInfo.GrantedApps.Contains( appId ); - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.RequestFreeLicense( appId ), cbMethod ); - }, () => { return completed; } ); - - return success; - } - - public void RequestDepotKey( uint depotId, uint appid = 0 ) - { - if ( DepotKeys.ContainsKey( depotId ) || bAborted ) - return; - - bool completed = false; - - Action cbMethod = ( depotKey ) => - { - completed = true; - Console.WriteLine( "Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result ); - - if ( depotKey.Result != EResult.OK ) - { - Abort(); - return; - } - - DepotKeys[ depotKey.DepotID ] = depotKey.DepotKey; - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.GetDepotDecryptionKey( depotId, appid ), cbMethod ); - }, () => { return completed; } ); - } - - public string ResolveCDNTopLevelHost(string host) - { - // SteamPipe CDN shares tokens with all hosts - if (host.EndsWith( ".steampipe.steamcontent.com" ) ) - { - return "steampipe.steamcontent.com"; - } - else if (host.EndsWith(".steamcontent.com")) - { - return "steamcontent.com"; - } - - return host; - } - - public void RequestCDNAuthToken( uint appid, uint depotid, string host, string cdnKey ) - { - if ( CDNAuthTokens.ContainsKey( cdnKey ) || bAborted ) - return; - - if ( !CDNAuthTokens.TryAdd( cdnKey, new TaskCompletionSource() ) ) - return; - - bool completed = false; - var timeoutDate = DateTime.Now.AddSeconds( 10 ); - Action cbMethod = ( cdnAuth ) => - { - completed = true; - Console.WriteLine( "Got CDN auth token for {0} result: {1} (expires {2})", host, cdnAuth.Result, cdnAuth.Expiration ); - - if ( cdnAuth.Result != EResult.OK ) - { - Abort(); - return; - } - - CDNAuthTokens[cdnKey].TrySetResult( cdnAuth ); - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.GetCDNAuthToken( appid, depotid, host ), cbMethod ); - }, () => { return completed || DateTime.Now >= timeoutDate; } ); - } - - public void CheckAppBetaPassword( uint appid, string password ) - { - bool completed = false; - Action cbMethod = ( appPassword ) => - { - completed = true; - - Console.WriteLine( "Retrieved {0} beta keys with result: {1}", appPassword.BetaPasswords.Count, appPassword.Result ); - - foreach ( var entry in appPassword.BetaPasswords ) - { - AppBetaPasswords[ entry.Key ] = entry.Value; - } - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.CheckAppBetaPassword( appid, password ), cbMethod ); - }, () => { return completed; } ); - } - - public PublishedFileDetails GetPublishedFileDetails(uint appId, PublishedFileID pubFile) - { - var pubFileRequest = new CPublishedFile_GetDetails_Request() { appid = appId }; - pubFileRequest.publishedfileids.Add( pubFile ); - - bool completed = false; - PublishedFileDetails details = null; - - Action cbMethod = callback => - { - completed = true; - if (callback.Result == EResult.OK) - { - var response = callback.GetDeserializedResponse(); - details = response.publishedfiledetails.FirstOrDefault(); - } - else - { - throw new Exception($"EResult {(int)callback.Result} ({callback.Result}) while retrieving file details for pubfile {pubFile}."); - } - }; - - WaitUntilCallback(() => - { - callbacks.Subscribe(steamPublishedFile.SendMessage(api => api.GetDetails(pubFileRequest)), cbMethod); - }, () => { return completed; }); - - return details; - } - - - public SteamCloud.UGCDetailsCallback GetUGCDetails(UGCHandle ugcHandle) - { - bool completed = false; - SteamCloud.UGCDetailsCallback details = null; - - Action cbMethod = callback => - { - completed = true; - if (callback.Result == EResult.OK) - { - details = callback; - } - else if (callback.Result == EResult.FileNotFound) - { - details = null; - } - else - { - throw new Exception($"EResult {(int)callback.Result} ({callback.Result}) while retrieving UGC details for {ugcHandle}."); - } - }; - - WaitUntilCallback(() => - { - callbacks.Subscribe(steamCloud.RequestUGCDetails(ugcHandle), cbMethod); - }, () => { return completed; }); - - return details; - } - - private void ResetConnectionFlags() - { - bExpectingDisconnectRemote = false; - bDidDisconnect = false; - bIsConnectionRecovery = false; - bDidReceiveLoginKey = false; - } - - void Connect() - { - bAborted = false; - bConnected = false; - bConnecting = true; - connectionBackoff = 0; - - ResetConnectionFlags(); - - this.connectTime = DateTime.Now; - this.steamClient.Connect(); - } - - private void Abort( bool sendLogOff = true ) - { - Disconnect( sendLogOff ); - } - public void Disconnect( bool sendLogOff = true ) - { - if ( sendLogOff ) - { - steamUser.LogOff(); - } - - bAborted = true; - bConnected = false; - bConnecting = false; - bIsConnectionRecovery = false; - steamClient.Disconnect(); - - // flush callbacks until our disconnected event - while ( !bDidDisconnect ) - { - callbacks.RunWaitAllCallbacks( TimeSpan.FromMilliseconds( 100 ) ); - } - } - - private void Reconnect() - { - bIsConnectionRecovery = true; - steamClient.Disconnect(); - } - - public void TryWaitForLoginKey() - { - if ( logonDetails.Username == null || !credentials.LoggedOn || !ContentDownloader.Config.RememberPassword ) return; - - var totalWaitPeriod = DateTime.Now.AddSeconds( 3 ); - - while ( true ) - { - DateTime now = DateTime.Now; - if ( now >= totalWaitPeriod ) break; - - if ( bDidReceiveLoginKey ) break; - - callbacks.RunWaitAllCallbacks( TimeSpan.FromMilliseconds( 100 ) ); - } - } - - private void WaitForCallbacks() - { - callbacks.RunWaitCallbacks( TimeSpan.FromSeconds( 1 ) ); - - TimeSpan diff = DateTime.Now - connectTime; - - if ( diff > STEAM3_TIMEOUT && !bConnected ) - { - Console.WriteLine( "Timeout connecting to Steam3." ); - Abort(); - - return; - } - } - - private void ConnectedCallback( SteamClient.ConnectedCallback connected ) - { - Console.WriteLine( " Done!" ); - bConnecting = false; - bConnected = true; - if ( !authenticatedUser ) - { - Console.Write( "Logging anonymously into Steam3..." ); - steamUser.LogOnAnonymous(); - } - else - { - Console.Write( "Logging '{0}' into Steam3...", logonDetails.Username ); - steamUser.LogOn( logonDetails ); - } - } - - private void DisconnectedCallback( SteamClient.DisconnectedCallback disconnected ) - { - bDidDisconnect = true; - - // When recovering the connection, we want to reconnect even if the remote disconnects us - if ( !bIsConnectionRecovery && ( disconnected.UserInitiated || bExpectingDisconnectRemote ) ) - { - Console.WriteLine( "Disconnected from Steam" ); - - // Any operations outstanding need to be aborted - bAborted = true; - } - else if ( connectionBackoff >= 10 ) - { - Console.WriteLine( "Could not connect to Steam after 10 tries" ); - Abort( false ); - } - else if ( !bAborted ) - { - if ( bConnecting ) - { - Console.WriteLine( "Connection to Steam failed. Trying again" ); - } - else - { - Console.WriteLine( "Lost connection to Steam. Reconnecting" ); - } - - Thread.Sleep( 1000 * ++connectionBackoff ); - - // Any connection related flags need to be reset here to match the state after Connect - ResetConnectionFlags(); - steamClient.Connect(); - } - } - - private void LogOnCallback( SteamUser.LoggedOnCallback loggedOn ) - { - bool isSteamGuard = loggedOn.Result == EResult.AccountLogonDenied; - bool is2FA = loggedOn.Result == EResult.AccountLoginDeniedNeedTwoFactor; - bool isLoginKey = ContentDownloader.Config.RememberPassword && logonDetails.LoginKey != null && loggedOn.Result == EResult.InvalidPassword; - - if ( isSteamGuard || is2FA || isLoginKey ) - { - bExpectingDisconnectRemote = true; - Abort( false ); - - if ( !isLoginKey ) - { - Console.WriteLine( "This account is protected by Steam Guard." ); - } - - if ( is2FA ) - { - Console.Write( "Please enter your 2 factor auth code from your authenticator app: " ); - logonDetails.TwoFactorCode = Console.ReadLine(); - } - else if ( isLoginKey ) - { - AccountSettingsStore.Instance.LoginKeys.Remove( logonDetails.Username ); - AccountSettingsStore.Save(); - - logonDetails.LoginKey = null; - - if ( ContentDownloader.Config.SuppliedPassword != null ) - { - Console.WriteLine( "Login key was expired. Connecting with supplied password." ); - logonDetails.Password = ContentDownloader.Config.SuppliedPassword; - } - else - { - Console.Write( "Login key was expired. Please enter your password: " ); - logonDetails.Password = Util.ReadPassword(); - } - } - else - { - Console.Write( "Please enter the authentication code sent to your email address: " ); - logonDetails.AuthCode = Console.ReadLine(); - } - - Console.Write( "Retrying Steam3 connection..." ); - Connect(); - - return; - } - else if ( loggedOn.Result == EResult.TryAnotherCM ) - { - Console.Write( "Retrying Steam3 connection (TryAnotherCM)..." ); - - Reconnect(); - - return; - } - else if ( loggedOn.Result == EResult.ServiceUnavailable ) - { - Console.WriteLine( "Unable to login to Steam3: {0}", loggedOn.Result ); - Abort( false ); - - return; - } - else if ( loggedOn.Result != EResult.OK ) - { - Console.WriteLine( "Unable to login to Steam3: {0}", loggedOn.Result ); - Abort(); - - return; - } - - Console.WriteLine( " Done!" ); - - this.seq++; - credentials.LoggedOn = true; - - if ( ContentDownloader.Config.CellID == 0 ) - { - Console.WriteLine( "Using Steam3 suggested CellID: " + loggedOn.CellID ); - ContentDownloader.Config.CellID = ( int )loggedOn.CellID; - } - } - - private void SessionTokenCallback( SteamUser.SessionTokenCallback sessionToken ) - { - Console.WriteLine( "Got session token!" ); - credentials.SessionToken = sessionToken.SessionToken; - } - - private void LicenseListCallback( SteamApps.LicenseListCallback licenseList ) - { - if ( licenseList.Result != EResult.OK ) - { - Console.WriteLine( "Unable to get license list: {0} ", licenseList.Result ); - Abort(); - - return; - } - - Console.WriteLine( "Got {0} licenses for account!", licenseList.LicenseList.Count ); - Licenses = licenseList.LicenseList; - - foreach ( var license in licenseList.LicenseList ) - { - if ( license.AccessToken > 0 ) - { - PackageTokens.TryAdd( license.PackageID, license.AccessToken ); - } - } - } - - private void UpdateMachineAuthCallback( SteamUser.UpdateMachineAuthCallback machineAuth ) - { - byte[] hash = Util.SHAHash( machineAuth.Data ); - Console.WriteLine( "Got Machine Auth: {0} {1} {2} {3}", machineAuth.FileName, machineAuth.Offset, machineAuth.BytesToWrite, machineAuth.Data.Length, hash ); - - AccountSettingsStore.Instance.SentryData[ logonDetails.Username ] = machineAuth.Data; - AccountSettingsStore.Save(); - - var authResponse = new SteamUser.MachineAuthDetails - { - BytesWritten = machineAuth.BytesToWrite, - FileName = machineAuth.FileName, - FileSize = machineAuth.BytesToWrite, - Offset = machineAuth.Offset, - - SentryFileHash = hash, // should be the sha1 hash of the sentry file we just wrote - - OneTimePassword = machineAuth.OneTimePassword, // not sure on this one yet, since we've had no examples of steam using OTPs - - LastError = 0, // result from win32 GetLastError - Result = EResult.OK, // if everything went okay, otherwise ~who knows~ - - JobID = machineAuth.JobID, // so we respond to the correct server job - }; - - // send off our response - steamUser.SendMachineAuthResponse( authResponse ); - } - - private void LoginKeyCallback( SteamUser.LoginKeyCallback loginKey ) - { - Console.WriteLine( "Accepted new login key for account {0}", logonDetails.Username ); - - AccountSettingsStore.Instance.LoginKeys[ logonDetails.Username ] = loginKey.LoginKey; - AccountSettingsStore.Save(); - - steamUser.AcceptNewLoginKey( loginKey ); - - bDidReceiveLoginKey = true; - } - - - } -} +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using SteamKit2; +using SteamKit2.Internal; + +namespace DepotDownloader +{ + class Steam3Session + { + public class Credentials + { + public bool LoggedOn { get; set; } + public ulong SessionToken { get; set; } + + public bool IsValid + { + get { return LoggedOn; } + } + } + + public ReadOnlyCollection Licenses + { + get; + private set; + } + + public Dictionary AppTokens { get; private set; } + public Dictionary PackageTokens { get; private set; } + public Dictionary DepotKeys { get; private set; } + public ConcurrentDictionary> CDNAuthTokens { get; private set; } + public Dictionary AppInfo { get; private set; } + public Dictionary PackageInfo { get; private set; } + public Dictionary AppBetaPasswords { get; private set; } + + public SteamClient steamClient; + public SteamUser steamUser; + readonly SteamApps steamApps; + readonly SteamCloud steamCloud; + readonly SteamUnifiedMessages.UnifiedService steamPublishedFile; + + readonly CallbackManager callbacks; + + readonly bool authenticatedUser; + bool bConnected; + bool bConnecting; + bool bAborted; + bool bExpectingDisconnectRemote; + bool bDidDisconnect; + bool bDidReceiveLoginKey; + bool bIsConnectionRecovery; + int connectionBackoff; + int seq; // more hack fixes + DateTime connectTime; + + // input + readonly SteamUser.LogOnDetails logonDetails; + + // output + readonly Credentials credentials; + + static readonly TimeSpan STEAM3_TIMEOUT = TimeSpan.FromSeconds(30); + + + public Steam3Session(SteamUser.LogOnDetails details) + { + this.logonDetails = details; + + this.authenticatedUser = details.Username != null; + this.credentials = new Credentials(); + this.bConnected = false; + this.bConnecting = false; + this.bAborted = false; + this.bExpectingDisconnectRemote = false; + this.bDidDisconnect = false; + this.bDidReceiveLoginKey = false; + this.seq = 0; + + this.AppTokens = new Dictionary(); + this.PackageTokens = new Dictionary(); + this.DepotKeys = new Dictionary(); + this.CDNAuthTokens = new ConcurrentDictionary>(); + this.AppInfo = new Dictionary(); + this.PackageInfo = new Dictionary(); + this.AppBetaPasswords = new Dictionary(); + + var clientConfiguration = SteamConfiguration.Create(config => + config + .WithHttpClientFactory(HttpClientFactory.CreateHttpClient) + ); + + this.steamClient = new SteamClient(clientConfiguration); + + this.steamUser = this.steamClient.GetHandler(); + this.steamApps = this.steamClient.GetHandler(); + this.steamCloud = this.steamClient.GetHandler(); + var steamUnifiedMessages = this.steamClient.GetHandler(); + this.steamPublishedFile = steamUnifiedMessages.CreateService(); + + this.callbacks = new CallbackManager(this.steamClient); + + this.callbacks.Subscribe(ConnectedCallback); + this.callbacks.Subscribe(DisconnectedCallback); + this.callbacks.Subscribe(LogOnCallback); + this.callbacks.Subscribe(SessionTokenCallback); + this.callbacks.Subscribe(LicenseListCallback); + this.callbacks.Subscribe(UpdateMachineAuthCallback); + this.callbacks.Subscribe(LoginKeyCallback); + + Console.Write("Connecting to Steam3..."); + + if (authenticatedUser) + { + var fi = new FileInfo(String.Format("{0}.sentryFile", logonDetails.Username)); + if (AccountSettingsStore.Instance.SentryData != null && AccountSettingsStore.Instance.SentryData.ContainsKey(logonDetails.Username)) + { + logonDetails.SentryFileHash = Util.SHAHash(AccountSettingsStore.Instance.SentryData[logonDetails.Username]); + } + else if (fi.Exists && fi.Length > 0) + { + var sentryData = File.ReadAllBytes(fi.FullName); + logonDetails.SentryFileHash = Util.SHAHash(sentryData); + AccountSettingsStore.Instance.SentryData[logonDetails.Username] = sentryData; + AccountSettingsStore.Save(); + } + } + + Connect(); + } + + public delegate bool WaitCondition(); + + private readonly object steamLock = new object(); + + public bool WaitUntilCallback(Action submitter, WaitCondition waiter) + { + while (!bAborted && !waiter()) + { + lock (steamLock) + { + submitter(); + } + + var seq = this.seq; + do + { + lock (steamLock) + { + WaitForCallbacks(); + } + } while (!bAborted && this.seq == seq && !waiter()); + } + + return bAborted; + } + + public Credentials WaitForCredentials() + { + if (credentials.IsValid || bAborted) + return credentials; + + WaitUntilCallback(() => { }, () => { return credentials.IsValid; }); + + return credentials; + } + + public void RequestAppInfo(uint appId, bool bForce = false) + { + if ((AppInfo.ContainsKey(appId) && !bForce) || bAborted) + return; + + var completed = false; + Action cbMethodTokens = appTokens => + { + completed = true; + if (appTokens.AppTokensDenied.Contains(appId)) + { + Console.WriteLine("Insufficient privileges to get access token for app {0}", appId); + } + + foreach (var token_dict in appTokens.AppTokens) + { + this.AppTokens[token_dict.Key] = token_dict.Value; + } + }; + + WaitUntilCallback(() => + { + callbacks.Subscribe(steamApps.PICSGetAccessTokens(new List { appId }, new List()), cbMethodTokens); + }, () => { return completed; }); + + completed = false; + Action cbMethod = appInfo => + { + completed = !appInfo.ResponsePending; + + foreach (var app_value in appInfo.Apps) + { + var app = app_value.Value; + + Console.WriteLine("Got AppInfo for {0}", app.ID); + AppInfo[app.ID] = app; + } + + foreach (var app in appInfo.UnknownApps) + { + AppInfo[app] = null; + } + }; + + var request = new SteamApps.PICSRequest(appId); + if (AppTokens.ContainsKey(appId)) + { + request.AccessToken = AppTokens[appId]; + request.Public = false; + } + + WaitUntilCallback(() => + { + callbacks.Subscribe(steamApps.PICSGetProductInfo(new List { request }, new List()), cbMethod); + }, () => { return completed; }); + } + + public void RequestPackageInfo(IEnumerable packageIds) + { + var packages = packageIds.ToList(); + packages.RemoveAll(pid => PackageInfo.ContainsKey(pid)); + + if (packages.Count == 0 || bAborted) + return; + + var completed = false; + Action cbMethod = packageInfo => + { + completed = !packageInfo.ResponsePending; + + foreach (var package_value in packageInfo.Packages) + { + var package = package_value.Value; + PackageInfo[package.ID] = package; + } + + foreach (var package in packageInfo.UnknownPackages) + { + PackageInfo[package] = null; + } + }; + + var packageRequests = new List(); + + foreach (var package in packages) + { + var request = new SteamApps.PICSRequest(package); + + if (PackageTokens.TryGetValue(package, out var token)) + { + request.AccessToken = token; + request.Public = false; + } + + packageRequests.Add(request); + } + + WaitUntilCallback(() => + { + callbacks.Subscribe(steamApps.PICSGetProductInfo(new List(), packageRequests), cbMethod); + }, () => { return completed; }); + } + + public bool RequestFreeAppLicense(uint appId) + { + var success = false; + var completed = false; + Action cbMethod = resultInfo => + { + completed = true; + success = resultInfo.GrantedApps.Contains(appId); + }; + + WaitUntilCallback(() => + { + callbacks.Subscribe(steamApps.RequestFreeLicense(appId), cbMethod); + }, () => { return completed; }); + + return success; + } + + public void RequestDepotKey(uint depotId, uint appid = 0) + { + if (DepotKeys.ContainsKey(depotId) || bAborted) + return; + + var completed = false; + + Action cbMethod = depotKey => + { + completed = true; + Console.WriteLine("Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result); + + if (depotKey.Result != EResult.OK) + { + Abort(); + return; + } + + DepotKeys[depotKey.DepotID] = depotKey.DepotKey; + }; + + WaitUntilCallback(() => + { + callbacks.Subscribe(steamApps.GetDepotDecryptionKey(depotId, appid), cbMethod); + }, () => { return completed; }); + } + + public string ResolveCDNTopLevelHost(string host) + { + // SteamPipe CDN shares tokens with all hosts + if (host.EndsWith(".steampipe.steamcontent.com")) + { + return "steampipe.steamcontent.com"; + } + + if (host.EndsWith(".steamcontent.com")) + { + return "steamcontent.com"; + } + + return host; + } + + public void RequestCDNAuthToken(uint appid, uint depotid, string host, string cdnKey) + { + if (CDNAuthTokens.ContainsKey(cdnKey) || bAborted) + return; + + if (!CDNAuthTokens.TryAdd(cdnKey, new TaskCompletionSource())) + return; + + var completed = false; + var timeoutDate = DateTime.Now.AddSeconds(10); + Action cbMethod = cdnAuth => + { + completed = true; + Console.WriteLine("Got CDN auth token for {0} result: {1} (expires {2})", host, cdnAuth.Result, cdnAuth.Expiration); + + if (cdnAuth.Result != EResult.OK) + { + Abort(); + return; + } + + CDNAuthTokens[cdnKey].TrySetResult(cdnAuth); + }; + + WaitUntilCallback(() => + { + callbacks.Subscribe(steamApps.GetCDNAuthToken(appid, depotid, host), cbMethod); + }, () => { return completed || DateTime.Now >= timeoutDate; }); + } + + public void CheckAppBetaPassword(uint appid, string password) + { + var completed = false; + Action cbMethod = appPassword => + { + completed = true; + + Console.WriteLine("Retrieved {0} beta keys with result: {1}", appPassword.BetaPasswords.Count, appPassword.Result); + + foreach (var entry in appPassword.BetaPasswords) + { + AppBetaPasswords[entry.Key] = entry.Value; + } + }; + + WaitUntilCallback(() => + { + callbacks.Subscribe(steamApps.CheckAppBetaPassword(appid, password), cbMethod); + }, () => { return completed; }); + } + + public PublishedFileDetails GetPublishedFileDetails(uint appId, PublishedFileID pubFile) + { + var pubFileRequest = new CPublishedFile_GetDetails_Request { appid = appId }; + pubFileRequest.publishedfileids.Add(pubFile); + + var completed = false; + PublishedFileDetails details = null; + + Action cbMethod = callback => + { + completed = true; + if (callback.Result == EResult.OK) + { + var response = callback.GetDeserializedResponse(); + details = response.publishedfiledetails.FirstOrDefault(); + } + else + { + throw new Exception($"EResult {(int)callback.Result} ({callback.Result}) while retrieving file details for pubfile {pubFile}."); + } + }; + + WaitUntilCallback(() => + { + callbacks.Subscribe(steamPublishedFile.SendMessage(api => api.GetDetails(pubFileRequest)), cbMethod); + }, () => { return completed; }); + + return details; + } + + + public SteamCloud.UGCDetailsCallback GetUGCDetails(UGCHandle ugcHandle) + { + var completed = false; + SteamCloud.UGCDetailsCallback details = null; + + Action cbMethod = callback => + { + completed = true; + if (callback.Result == EResult.OK) + { + details = callback; + } + else if (callback.Result == EResult.FileNotFound) + { + details = null; + } + else + { + throw new Exception($"EResult {(int)callback.Result} ({callback.Result}) while retrieving UGC details for {ugcHandle}."); + } + }; + + WaitUntilCallback(() => + { + callbacks.Subscribe(steamCloud.RequestUGCDetails(ugcHandle), cbMethod); + }, () => { return completed; }); + + return details; + } + + private void ResetConnectionFlags() + { + bExpectingDisconnectRemote = false; + bDidDisconnect = false; + bIsConnectionRecovery = false; + bDidReceiveLoginKey = false; + } + + void Connect() + { + bAborted = false; + bConnected = false; + bConnecting = true; + connectionBackoff = 0; + + ResetConnectionFlags(); + + this.connectTime = DateTime.Now; + this.steamClient.Connect(); + } + + private void Abort(bool sendLogOff = true) + { + Disconnect(sendLogOff); + } + + public void Disconnect(bool sendLogOff = true) + { + if (sendLogOff) + { + steamUser.LogOff(); + } + + bAborted = true; + bConnected = false; + bConnecting = false; + bIsConnectionRecovery = false; + steamClient.Disconnect(); + + // flush callbacks until our disconnected event + while (!bDidDisconnect) + { + callbacks.RunWaitAllCallbacks(TimeSpan.FromMilliseconds(100)); + } + } + + private void Reconnect() + { + bIsConnectionRecovery = true; + steamClient.Disconnect(); + } + + public void TryWaitForLoginKey() + { + if (logonDetails.Username == null || !credentials.LoggedOn || !ContentDownloader.Config.RememberPassword) return; + + var totalWaitPeriod = DateTime.Now.AddSeconds(3); + + while (true) + { + var now = DateTime.Now; + if (now >= totalWaitPeriod) break; + + if (bDidReceiveLoginKey) break; + + callbacks.RunWaitAllCallbacks(TimeSpan.FromMilliseconds(100)); + } + } + + private void WaitForCallbacks() + { + callbacks.RunWaitCallbacks(TimeSpan.FromSeconds(1)); + + var diff = DateTime.Now - connectTime; + + if (diff > STEAM3_TIMEOUT && !bConnected) + { + Console.WriteLine("Timeout connecting to Steam3."); + Abort(); + } + } + + private void ConnectedCallback(SteamClient.ConnectedCallback connected) + { + Console.WriteLine(" Done!"); + bConnecting = false; + bConnected = true; + if (!authenticatedUser) + { + Console.Write("Logging anonymously into Steam3..."); + steamUser.LogOnAnonymous(); + } + else + { + Console.Write("Logging '{0}' into Steam3...", logonDetails.Username); + steamUser.LogOn(logonDetails); + } + } + + private void DisconnectedCallback(SteamClient.DisconnectedCallback disconnected) + { + bDidDisconnect = true; + + // When recovering the connection, we want to reconnect even if the remote disconnects us + if (!bIsConnectionRecovery && (disconnected.UserInitiated || bExpectingDisconnectRemote)) + { + Console.WriteLine("Disconnected from Steam"); + + // Any operations outstanding need to be aborted + bAborted = true; + } + else if (connectionBackoff >= 10) + { + Console.WriteLine("Could not connect to Steam after 10 tries"); + Abort(false); + } + else if (!bAborted) + { + if (bConnecting) + { + Console.WriteLine("Connection to Steam failed. Trying again"); + } + else + { + Console.WriteLine("Lost connection to Steam. Reconnecting"); + } + + Thread.Sleep(1000 * ++connectionBackoff); + + // Any connection related flags need to be reset here to match the state after Connect + ResetConnectionFlags(); + steamClient.Connect(); + } + } + + private void LogOnCallback(SteamUser.LoggedOnCallback loggedOn) + { + var isSteamGuard = loggedOn.Result == EResult.AccountLogonDenied; + var is2FA = loggedOn.Result == EResult.AccountLoginDeniedNeedTwoFactor; + var isLoginKey = ContentDownloader.Config.RememberPassword && logonDetails.LoginKey != null && loggedOn.Result == EResult.InvalidPassword; + + if (isSteamGuard || is2FA || isLoginKey) + { + bExpectingDisconnectRemote = true; + Abort(false); + + if (!isLoginKey) + { + Console.WriteLine("This account is protected by Steam Guard."); + } + + if (is2FA) + { + Console.Write("Please enter your 2 factor auth code from your authenticator app: "); + logonDetails.TwoFactorCode = Console.ReadLine(); + } + else if (isLoginKey) + { + AccountSettingsStore.Instance.LoginKeys.Remove(logonDetails.Username); + AccountSettingsStore.Save(); + + logonDetails.LoginKey = null; + + if (ContentDownloader.Config.SuppliedPassword != null) + { + Console.WriteLine("Login key was expired. Connecting with supplied password."); + logonDetails.Password = ContentDownloader.Config.SuppliedPassword; + } + else + { + Console.Write("Login key was expired. Please enter your password: "); + logonDetails.Password = Util.ReadPassword(); + } + } + else + { + Console.Write("Please enter the authentication code sent to your email address: "); + logonDetails.AuthCode = Console.ReadLine(); + } + + Console.Write("Retrying Steam3 connection..."); + Connect(); + + return; + } + + if (loggedOn.Result == EResult.TryAnotherCM) + { + Console.Write("Retrying Steam3 connection (TryAnotherCM)..."); + + Reconnect(); + + return; + } + + if (loggedOn.Result == EResult.ServiceUnavailable) + { + Console.WriteLine("Unable to login to Steam3: {0}", loggedOn.Result); + Abort(false); + + return; + } + + if (loggedOn.Result != EResult.OK) + { + Console.WriteLine("Unable to login to Steam3: {0}", loggedOn.Result); + Abort(); + + return; + } + + Console.WriteLine(" Done!"); + + this.seq++; + credentials.LoggedOn = true; + + if (ContentDownloader.Config.CellID == 0) + { + Console.WriteLine("Using Steam3 suggested CellID: " + loggedOn.CellID); + ContentDownloader.Config.CellID = (int)loggedOn.CellID; + } + } + + private void SessionTokenCallback(SteamUser.SessionTokenCallback sessionToken) + { + Console.WriteLine("Got session token!"); + credentials.SessionToken = sessionToken.SessionToken; + } + + private void LicenseListCallback(SteamApps.LicenseListCallback licenseList) + { + if (licenseList.Result != EResult.OK) + { + Console.WriteLine("Unable to get license list: {0} ", licenseList.Result); + Abort(); + + return; + } + + Console.WriteLine("Got {0} licenses for account!", licenseList.LicenseList.Count); + Licenses = licenseList.LicenseList; + + foreach (var license in licenseList.LicenseList) + { + if (license.AccessToken > 0) + { + PackageTokens.TryAdd(license.PackageID, license.AccessToken); + } + } + } + + private void UpdateMachineAuthCallback(SteamUser.UpdateMachineAuthCallback machineAuth) + { + var hash = Util.SHAHash(machineAuth.Data); + Console.WriteLine("Got Machine Auth: {0} {1} {2} {3}", machineAuth.FileName, machineAuth.Offset, machineAuth.BytesToWrite, machineAuth.Data.Length, hash); + + AccountSettingsStore.Instance.SentryData[logonDetails.Username] = machineAuth.Data; + AccountSettingsStore.Save(); + + var authResponse = new SteamUser.MachineAuthDetails + { + BytesWritten = machineAuth.BytesToWrite, + FileName = machineAuth.FileName, + FileSize = machineAuth.BytesToWrite, + Offset = machineAuth.Offset, + + SentryFileHash = hash, // should be the sha1 hash of the sentry file we just wrote + + OneTimePassword = machineAuth.OneTimePassword, // not sure on this one yet, since we've had no examples of steam using OTPs + + LastError = 0, // result from win32 GetLastError + Result = EResult.OK, // if everything went okay, otherwise ~who knows~ + + JobID = machineAuth.JobID, // so we respond to the correct server job + }; + + // send off our response + steamUser.SendMachineAuthResponse(authResponse); + } + + private void LoginKeyCallback(SteamUser.LoginKeyCallback loginKey) + { + Console.WriteLine("Accepted new login key for account {0}", logonDetails.Username); + + AccountSettingsStore.Instance.LoginKeys[logonDetails.Username] = loginKey.LoginKey; + AccountSettingsStore.Save(); + + steamUser.AcceptNewLoginKey(loginKey); + + bDidReceiveLoginKey = true; + } + } +} diff --git a/DepotDownloader/Util.cs b/DepotDownloader/Util.cs index 6ef9bb48..4f98df9f 100644 --- a/DepotDownloader/Util.cs +++ b/DepotDownloader/Util.cs @@ -1,176 +1,179 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using System.Security.Cryptography; -using System.Text; -using System.Threading.Tasks; - -namespace DepotDownloader -{ - static class Util - { - public static string GetSteamOS() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - return "windows"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - return "macos"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - return "linux"; - } - - return "unknown"; - } - - public static string GetSteamArch() - { - return Environment.Is64BitOperatingSystem ? "64" : "32"; - } - - public static string ReadPassword() - { - ConsoleKeyInfo keyInfo; - StringBuilder password = new StringBuilder(); - - do - { - keyInfo = Console.ReadKey( true ); - - if ( keyInfo.Key == ConsoleKey.Backspace ) - { - if ( password.Length > 0 ) - { - password.Remove( password.Length - 1, 1 ); - Console.Write( "\x1B[1D\x1B[1P" ); - } - continue; - } - - /* Printable ASCII characters only */ - char c = keyInfo.KeyChar; - if ( c >= ' ' && c <= '~' ) - { - password.Append( c ); - Console.Write( '*' ); - } - } while ( keyInfo.Key != ConsoleKey.Enter ); - - return password.ToString(); - } - - // Validate a file against Steam3 Chunk data - public static List ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata) - { - var neededChunks = new List(); - int read; - - foreach (var data in chunkdata) - { - byte[] chunk = new byte[data.UncompressedLength]; - fs.Seek((long)data.Offset, SeekOrigin.Begin); - read = fs.Read(chunk, 0, (int)data.UncompressedLength); - - byte[] tempchunk; - if (read < data.UncompressedLength) - { - tempchunk = new byte[read]; - Array.Copy(chunk, 0, tempchunk, 0, read); - } - else - { - tempchunk = chunk; - } - - byte[] adler = AdlerHash(tempchunk); - if (!adler.SequenceEqual(data.Checksum)) - { - neededChunks.Add(data); - } - } - - return neededChunks; - } - - public static byte[] AdlerHash(byte[] input) - { - uint a = 0, b = 0; - for (int i = 0; i < input.Length; i++) - { - a = (a + input[i]) % 65521; - b = (b + a) % 65521; - } - return BitConverter.GetBytes(a | (b << 16)); - } - - public static byte[] SHAHash( byte[] input ) - { - using (var sha = SHA1.Create()) - { - var output = sha.ComputeHash( input ); - - return output; - } - } - - public static byte[] DecodeHexString( string hex ) - { - if ( hex == null ) - return null; - - int chars = hex.Length; - byte[] bytes = new byte[ chars / 2 ]; - - for ( int i = 0 ; i < chars ; i += 2 ) - bytes[ i / 2 ] = Convert.ToByte( hex.Substring( i, 2 ), 16 ); - - return bytes; - } - - public static string EncodeHexString( byte[] input ) - { - return input.Aggregate( new StringBuilder(), - ( sb, v ) => sb.Append( v.ToString( "x2" ) ) - ).ToString(); - } - - public static async Task InvokeAsync(IEnumerable> taskFactories, int maxDegreeOfParallelism) - { - if (taskFactories == null) throw new ArgumentNullException(nameof(taskFactories)); - if (maxDegreeOfParallelism <= 0) throw new ArgumentException(nameof(maxDegreeOfParallelism)); - - Func[] queue = taskFactories.ToArray(); - - if (queue.Length == 0) - { - return; - } - - List tasksInFlight = new List(maxDegreeOfParallelism); - int index = 0; - - do - { - while (tasksInFlight.Count < maxDegreeOfParallelism && index < queue.Length) - { - Func taskFactory = queue[index++]; - - tasksInFlight.Add(taskFactory()); - } - - Task completedTask = await Task.WhenAny(tasksInFlight).ConfigureAwait(false); - - await completedTask.ConfigureAwait(false); - - tasksInFlight.Remove(completedTask); - } - while (index < queue.Length || tasksInFlight.Count != 0); - } - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; + +namespace DepotDownloader +{ + static class Util + { + public static string GetSteamOS() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return "windows"; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return "macos"; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return "linux"; + } + + return "unknown"; + } + + public static string GetSteamArch() + { + return Environment.Is64BitOperatingSystem ? "64" : "32"; + } + + public static string ReadPassword() + { + ConsoleKeyInfo keyInfo; + var password = new StringBuilder(); + + do + { + keyInfo = Console.ReadKey(true); + + if (keyInfo.Key == ConsoleKey.Backspace) + { + if (password.Length > 0) + { + password.Remove(password.Length - 1, 1); + Console.Write("\x1B[1D\x1B[1P"); + } + + continue; + } + + /* Printable ASCII characters only */ + var c = keyInfo.KeyChar; + if (c >= ' ' && c <= '~') + { + password.Append(c); + Console.Write('*'); + } + } while (keyInfo.Key != ConsoleKey.Enter); + + return password.ToString(); + } + + // Validate a file against Steam3 Chunk data + public static List ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata) + { + var neededChunks = new List(); + int read; + + foreach (var data in chunkdata) + { + var chunk = new byte[data.UncompressedLength]; + fs.Seek((long)data.Offset, SeekOrigin.Begin); + read = fs.Read(chunk, 0, (int)data.UncompressedLength); + + byte[] tempchunk; + if (read < data.UncompressedLength) + { + tempchunk = new byte[read]; + Array.Copy(chunk, 0, tempchunk, 0, read); + } + else + { + tempchunk = chunk; + } + + var adler = AdlerHash(tempchunk); + if (!adler.SequenceEqual(data.Checksum)) + { + neededChunks.Add(data); + } + } + + return neededChunks; + } + + public static byte[] AdlerHash(byte[] input) + { + uint a = 0, b = 0; + for (var i = 0; i < input.Length; i++) + { + a = (a + input[i]) % 65521; + b = (b + a) % 65521; + } + + return BitConverter.GetBytes(a | (b << 16)); + } + + public static byte[] SHAHash(byte[] input) + { + using (var sha = SHA1.Create()) + { + var output = sha.ComputeHash(input); + + return output; + } + } + + public static byte[] DecodeHexString(string hex) + { + if (hex == null) + return null; + + var chars = hex.Length; + var bytes = new byte[chars / 2]; + + for (var i = 0; i < chars; i += 2) + bytes[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16); + + return bytes; + } + + public static string EncodeHexString(byte[] input) + { + return input.Aggregate(new StringBuilder(), + (sb, v) => sb.Append(v.ToString("x2")) + ).ToString(); + } + + public static async Task InvokeAsync(IEnumerable> taskFactories, int maxDegreeOfParallelism) + { + if (taskFactories == null) throw new ArgumentNullException(nameof(taskFactories)); + if (maxDegreeOfParallelism <= 0) throw new ArgumentException(nameof(maxDegreeOfParallelism)); + + var queue = taskFactories.ToArray(); + + if (queue.Length == 0) + { + return; + } + + var tasksInFlight = new List(maxDegreeOfParallelism); + var index = 0; + + do + { + while (tasksInFlight.Count < maxDegreeOfParallelism && index < queue.Length) + { + var taskFactory = queue[index++]; + + tasksInFlight.Add(taskFactory()); + } + + var completedTask = await Task.WhenAny(tasksInFlight).ConfigureAwait(false); + + await completedTask.ConfigureAwait(false); + + tasksInFlight.Remove(completedTask); + } while (index < queue.Length || tasksInFlight.Count != 0); + } + } +}