diff --git a/DepotDownloader/AccountSettingsStore.cs b/DepotDownloader.Core/AccountSettingsStore.cs similarity index 97% rename from DepotDownloader/AccountSettingsStore.cs rename to DepotDownloader.Core/AccountSettingsStore.cs index 2a8c9836..7355a44b 100644 --- a/DepotDownloader/AccountSettingsStore.cs +++ b/DepotDownloader.Core/AccountSettingsStore.cs @@ -8,10 +8,10 @@ using System.Linq; using SteamKit2; using SteamKit2.Discovery; -namespace DepotDownloader +namespace DepotDownloader.Core { [ProtoContract] - class AccountSettingsStore + public class AccountSettingsStore { [ProtoMember(1, IsRequired=false)] public Dictionary SentryData { get; private set; } diff --git a/DepotDownloader/CDNClientPool.cs b/DepotDownloader.Core/CDNClientPool.cs similarity index 99% rename from DepotDownloader/CDNClientPool.cs rename to DepotDownloader.Core/CDNClientPool.cs index 654d3f54..07a8af26 100644 --- a/DepotDownloader/CDNClientPool.cs +++ b/DepotDownloader.Core/CDNClientPool.cs @@ -7,12 +7,12 @@ using System.Net; using System.Threading; using System.Threading.Tasks; -namespace DepotDownloader +namespace DepotDownloader.Core { /// /// CDNClientPool provides a pool of connections to CDN endpoints, requesting CDN tokens as needed /// - class CDNClientPool + public class CDNClientPool { private const int ServerEndpointMinimumSize = 8; diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader.Core/ContentDownloader.cs similarity index 97% rename from DepotDownloader/ContentDownloader.cs rename to DepotDownloader.Core/ContentDownloader.cs index 66b829e0..e9af6e0d 100644 --- a/DepotDownloader/ContentDownloader.cs +++ b/DepotDownloader.Core/ContentDownloader.cs @@ -1,1019 +1,1019 @@ -using SteamKit2; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; - -namespace DepotDownloader -{ - public class ContentDownloaderException : System.Exception - { - public ContentDownloaderException( String value ) : base( value ) {} - } - - static class ContentDownloader - { - public const uint INVALID_APP_ID = uint.MaxValue; - public const uint INVALID_DEPOT_ID = uint.MaxValue; - public const ulong INVALID_MANIFEST_ID = ulong.MaxValue; - public const string DEFAULT_BRANCH = "Public"; - - public static DownloadConfig Config = new DownloadConfig(); - - private static Steam3Session steam3; - private static Steam3Session.Credentials steam3Credentials; - private static CDNClientPool cdnPool; - - private const string DEFAULT_DOWNLOAD_DIR = "depots"; - private const string CONFIG_DIR = ".DepotDownloader"; - private static readonly string STAGING_DIR = Path.Combine( CONFIG_DIR, "staging" ); - - private sealed class DepotDownloadInfo - { - public uint id { get; private set; } - public string installDir { get; private set; } - public string contentName { get; private set; } - - public ulong manifestId { get; private set; } - public byte[] depotKey; - - public DepotDownloadInfo( uint depotid, ulong manifestId, string installDir, string contentName ) - { - this.id = depotid; - this.manifestId = manifestId; - this.installDir = installDir; - this.contentName = contentName; - } - } - - static bool CreateDirectories( uint depotId, uint depotVersion, out string installDir ) - { - installDir = null; - try - { - if ( string.IsNullOrWhiteSpace( ContentDownloader.Config.InstallDirectory ) ) - { - Directory.CreateDirectory( DEFAULT_DOWNLOAD_DIR ); - - string depotPath = Path.Combine( DEFAULT_DOWNLOAD_DIR, depotId.ToString() ); - Directory.CreateDirectory( depotPath ); - - installDir = Path.Combine( depotPath, depotVersion.ToString() ); - Directory.CreateDirectory( installDir ); - - Directory.CreateDirectory( Path.Combine( installDir, CONFIG_DIR ) ); - Directory.CreateDirectory( Path.Combine( installDir, STAGING_DIR ) ); - } - else - { - Directory.CreateDirectory( ContentDownloader.Config.InstallDirectory ); - - installDir = ContentDownloader.Config.InstallDirectory; - - Directory.CreateDirectory( Path.Combine( installDir, CONFIG_DIR ) ); - Directory.CreateDirectory( Path.Combine( installDir, STAGING_DIR ) ); - } - } - catch - { - return false; - } - - return true; - } - - static bool TestIsFileIncluded( string filename ) - { - if ( !Config.UsingFileList ) - return true; - - foreach ( string fileListEntry in Config.FilesToDownload ) - { - if ( fileListEntry.Equals( filename, StringComparison.OrdinalIgnoreCase ) ) - return true; - } - - foreach ( Regex rgx in Config.FilesToDownloadRegex ) - { - Match m = rgx.Match( filename ); - - if ( m.Success ) - return true; - } - - return false; - } - - static bool AccountHasAccess( uint depotId ) - { - if ( steam3 == null || steam3.steamUser.SteamID == null || ( steam3.Licenses == null && steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser ) ) - return false; - - IEnumerable licenseQuery; - if ( steam3.steamUser.SteamID.AccountType == EAccountType.AnonUser ) - { - licenseQuery = new List() { 17906 }; - } - else - { - licenseQuery = steam3.Licenses.Select( x => x.PackageID ).Distinct(); - } - - steam3.RequestPackageInfo( licenseQuery ); - - foreach ( var license in licenseQuery ) - { - SteamApps.PICSProductInfoCallback.PICSProductInfo package; - if ( steam3.PackageInfo.TryGetValue( license, out package ) && package != null ) - { - if ( package.KeyValues[ "appids" ].Children.Any( child => child.AsUnsignedInteger() == depotId ) ) - return true; - - if ( package.KeyValues[ "depotids" ].Children.Any( child => child.AsUnsignedInteger() == depotId ) ) - return true; - } - } - - return false; - } - - internal static KeyValue GetSteam3AppSection( uint appId, EAppInfoSection section ) - { - if ( steam3 == null || steam3.AppInfo == null ) - { - return null; - } - - SteamApps.PICSProductInfoCallback.PICSProductInfo app; - if ( !steam3.AppInfo.TryGetValue( appId, out app ) || app == null ) - { - return null; - } - - KeyValue appinfo = app.KeyValues; - string section_key; - - switch ( section ) - { - case EAppInfoSection.Common: - section_key = "common"; - break; - case EAppInfoSection.Extended: - section_key = "extended"; - break; - case EAppInfoSection.Config: - section_key = "config"; - break; - case EAppInfoSection.Depots: - section_key = "depots"; - break; - default: - throw new NotImplementedException(); - } - - KeyValue section_kv = appinfo.Children.Where( c => c.Name == section_key ).FirstOrDefault(); - return section_kv; - } - - static uint GetSteam3AppBuildNumber( uint appId, string branch ) - { - if ( appId == INVALID_APP_ID ) - return 0; - - - KeyValue depots = ContentDownloader.GetSteam3AppSection( appId, EAppInfoSection.Depots ); - KeyValue branches = depots[ "branches" ]; - KeyValue node = branches[ branch ]; - - if ( node == KeyValue.Invalid ) - return 0; - - KeyValue buildid = node[ "buildid" ]; - - if ( buildid == KeyValue.Invalid ) - return 0; - - return uint.Parse( buildid.Value ); - } - - static ulong GetSteam3DepotManifest( uint depotId, uint appId, string branch ) - { - KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); - KeyValue depotChild = depots[ depotId.ToString() ]; - - if ( depotChild == KeyValue.Invalid ) - return INVALID_MANIFEST_ID; - - // Shared depots can either provide manifests, or leave you relying on their parent app. - // It seems that with the latter, "sharedinstall" will exist (and equals 2 in the one existance I know of). - // Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680. - if ( depotChild[ "manifests" ] == KeyValue.Invalid && depotChild[ "depotfromapp" ] != KeyValue.Invalid ) - { - uint otherAppId = depotChild["depotfromapp"].AsUnsignedInteger(); - if ( otherAppId == appId ) - { - // This shouldn't ever happen, but ya never know with Valve. Don't infinite loop. - Console.WriteLine( "App {0}, Depot {1} has depotfromapp of {2}!", - appId, depotId, otherAppId ); - return INVALID_MANIFEST_ID; - } - - steam3.RequestAppInfo( otherAppId ); - - return GetSteam3DepotManifest( depotId, otherAppId, branch ); - } - - var manifests = depotChild[ "manifests" ]; - var manifests_encrypted = depotChild[ "encryptedmanifests" ]; - - if ( manifests.Children.Count == 0 && manifests_encrypted.Children.Count == 0 ) - return INVALID_MANIFEST_ID; - - var node = manifests[ branch ]; - - if ( branch != "Public" && node == KeyValue.Invalid ) - { - var node_encrypted = manifests_encrypted[ branch ]; - if ( node_encrypted != KeyValue.Invalid ) - { - string password = Config.BetaPassword; - if ( password == null ) - { - Console.Write( "Please enter the password for branch {0}: ", branch ); - Config.BetaPassword = password = Console.ReadLine(); - } - - var encrypted_v1 = node_encrypted[ "encrypted_gid" ]; - var encrypted_v2 = node_encrypted[ "encrypted_gid_2" ]; - - if ( encrypted_v1 != KeyValue.Invalid ) - { - byte[] input = Util.DecodeHexString( encrypted_v1.Value ); - byte[] manifest_bytes = CryptoHelper.VerifyAndDecryptPassword( input, password ); - - if ( manifest_bytes == null ) - { - Console.WriteLine( "Password was invalid for branch {0}", branch ); - return INVALID_MANIFEST_ID; - } - - return BitConverter.ToUInt64( manifest_bytes, 0 ); - } - else if ( encrypted_v2 != KeyValue.Invalid ) - { - // Submit the password to Steam now to get encryption keys - steam3.CheckAppBetaPassword( appId, Config.BetaPassword ); - - if ( !steam3.AppBetaPasswords.ContainsKey( branch ) ) - { - Console.WriteLine( "Password was invalid for branch {0}", branch ); - return INVALID_MANIFEST_ID; - } - - byte[] input = Util.DecodeHexString( encrypted_v2.Value ); - byte[] manifest_bytes; - try - { - manifest_bytes = CryptoHelper.SymmetricDecryptECB( input, steam3.AppBetaPasswords[ branch ] ); - } - catch ( Exception e ) - { - Console.WriteLine( "Failed to decrypt branch {0}: {1}", branch, e.Message ); - return INVALID_MANIFEST_ID; - } - - return BitConverter.ToUInt64( manifest_bytes, 0 ); - } - else - { - Console.WriteLine( "Unhandled depot encryption for depotId {0}", depotId ); - return INVALID_MANIFEST_ID; - } - - } - - return INVALID_MANIFEST_ID; - } - - if ( node.Value == null ) - return INVALID_MANIFEST_ID; - - return UInt64.Parse( node.Value ); - } - - static string GetAppOrDepotName( uint depotId, uint appId ) - { - if ( depotId == INVALID_DEPOT_ID ) - { - KeyValue info = GetSteam3AppSection( appId, EAppInfoSection.Common ); - - if ( info == null ) - return String.Empty; - - return info[ "name" ].AsString(); - } - else - { - KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); - - if ( depots == null ) - return String.Empty; - - KeyValue depotChild = depots[ depotId.ToString() ]; - - if ( depotChild == null ) - return String.Empty; - - return depotChild[ "name" ].AsString(); - } - } - - public static bool InitializeSteam3( string username, string password ) - { - string loginKey = null; - - if ( username != null && Config.RememberPassword ) - { - _ = AccountSettingsStore.Instance.LoginKeys.TryGetValue( username, out loginKey ); - } - - steam3 = new Steam3Session( - new SteamUser.LogOnDetails() - { - Username = username, - Password = loginKey == null ? password : null, - ShouldRememberPassword = Config.RememberPassword, - LoginKey = loginKey, - LoginID = Config.LoginID ?? 0x534B32, // "SK2" - } - ); - - steam3Credentials = steam3.WaitForCredentials(); - - if ( !steam3Credentials.IsValid ) - { - Console.WriteLine( "Unable to get steam3 credentials." ); - return false; - } - - cdnPool = new CDNClientPool( steam3 ); - return true; - } - - public static void ShutdownSteam3() - { - if (cdnPool != null) - { - cdnPool.Shutdown(); - cdnPool = null; - } - - if ( steam3 == null ) - return; - - steam3.TryWaitForLoginKey(); - steam3.Disconnect(); - } - - public static async Task DownloadPubfileAsync( uint appId, ulong publishedFileId ) - { - var details = steam3.GetPubfileItemInfo( appId, publishedFileId ); - - if ( details?.manifest_id > 0 ) - { - await DownloadAppAsync( appId, appId, details.manifest_id, DEFAULT_BRANCH, null, null, null, false, true ); - } - else - { - Console.WriteLine( "Unable to locate manifest ID for published file {0}", publishedFileId ); - } - } - - public static async Task DownloadAppAsync( uint appId, uint depotId, ulong manifestId, string branch, string os, string arch, string language, bool lv, bool isUgc ) - { - // Load our configuration data containing the depots currently installed - string configPath = ContentDownloader.Config.InstallDirectory; - if (string.IsNullOrWhiteSpace(configPath)) - { - configPath = DEFAULT_DOWNLOAD_DIR; - } - - Directory.CreateDirectory(Path.Combine(configPath, CONFIG_DIR)); - DepotConfigStore.LoadFromFile(Path.Combine(configPath, CONFIG_DIR, "depot.config")); - - if ( steam3 != null ) - steam3.RequestAppInfo( appId ); - - if ( !AccountHasAccess( appId ) ) - { - if ( steam3.RequestFreeAppLicense( appId ) ) - { - Console.WriteLine( "Obtained FreeOnDemand license for app {0}", appId ); - - // Fetch app info again in case we didn't get it fully without a license. - steam3.RequestAppInfo( appId, true ); - } - else - { - string contentName = GetAppOrDepotName( INVALID_DEPOT_ID, appId ); - throw new ContentDownloaderException( String.Format( "App {0} ({1}) is not available from this account.", appId, contentName ) ); - } - } - - var depotIDs = new List(); - KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); - - if ( isUgc ) - { - var workshopDepot = depots["workshopdepot"].AsUnsignedInteger(); - if (workshopDepot != 0) - depotId = workshopDepot; - - depotIDs.Add( depotId ); - } - else - { - Console.WriteLine( "Using app branch: '{0}'.", branch ); - - if ( depots != null ) - { - foreach ( var depotSection in depots.Children ) - { - uint id = INVALID_DEPOT_ID; - if ( depotSection.Children.Count == 0 ) - continue; - - if ( !uint.TryParse( depotSection.Name, out id ) ) - continue; - - if ( depotId != INVALID_DEPOT_ID && id != depotId ) - continue; - - if ( depotId == INVALID_DEPOT_ID ) - { - var depotConfig = depotSection[ "config" ]; - if ( depotConfig != KeyValue.Invalid ) - { - if ( !Config.DownloadAllPlatforms && - depotConfig["oslist"] != KeyValue.Invalid && - !string.IsNullOrWhiteSpace( depotConfig["oslist"].Value ) ) - { - var oslist = depotConfig["oslist"].Value.Split( ',' ); - if ( Array.IndexOf( oslist, os ?? Util.GetSteamOS() ) == -1 ) - continue; - } - - if ( depotConfig["osarch"] != KeyValue.Invalid && - !string.IsNullOrWhiteSpace( depotConfig["osarch"].Value ) ) - { - var depotArch = depotConfig["osarch"].Value; - if ( depotArch != ( arch ?? Util.GetSteamArch() ) ) - continue; - } - - if ( !Config.DownloadAllLanguages && - depotConfig["language"] != KeyValue.Invalid && - !string.IsNullOrWhiteSpace( depotConfig["language"].Value ) ) - { - var depotLang = depotConfig["language"].Value; - if ( depotLang != ( language ?? "english" ) ) - continue; - } - - if ( !lv && - depotConfig["lowviolence"] != KeyValue.Invalid && - depotConfig["lowviolence"].AsBoolean() ) - continue; - } - } - - depotIDs.Add( id ); - } - } - if ( depotIDs == null || ( depotIDs.Count == 0 && depotId == INVALID_DEPOT_ID ) ) - { - throw new ContentDownloaderException( String.Format( "Couldn't find any depots to download for app {0}", appId ) ); - } - else if ( depotIDs.Count == 0 ) - { - throw new ContentDownloaderException( String.Format( "Depot {0} not listed for app {1}", depotId, appId ) ); - } - } - - var infos = new List(); - - foreach ( var depot in depotIDs ) - { - var info = GetDepotInfo( depot, appId, manifestId, branch ); - if ( info != null ) - { - infos.Add( info ); - } - } - - try - { - await DownloadSteam3Async( appId, infos ).ConfigureAwait( false ); - } - catch ( OperationCanceledException ) - { - Console.WriteLine( "App {0} was not completely downloaded.", appId ); - throw; - } - } - - static DepotDownloadInfo GetDepotInfo( uint depotId, uint appId, ulong manifestId, string branch ) - { - if ( steam3 != null && appId != INVALID_APP_ID ) - steam3.RequestAppInfo( ( uint )appId ); - - string contentName = GetAppOrDepotName( depotId, appId ); - - if ( !AccountHasAccess( depotId ) ) - { - Console.WriteLine( "Depot {0} ({1}) is not available from this account.", depotId, contentName ); - - return null; - } - - // Skip requesting an app ticket - steam3.AppTickets[ depotId ] = null; - - if (manifestId == INVALID_MANIFEST_ID) - { - manifestId = GetSteam3DepotManifest(depotId, appId, branch); - if (manifestId == INVALID_MANIFEST_ID && branch != "public") - { - Console.WriteLine("Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch); - branch = "public"; - manifestId = GetSteam3DepotManifest(depotId, appId, branch); - } - - if (manifestId == INVALID_MANIFEST_ID) - { - Console.WriteLine("Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName); - return null; - } - } - - uint uVersion = GetSteam3AppBuildNumber( appId, branch ); - - string installDir; - if ( !CreateDirectories( depotId, uVersion, out installDir ) ) - { - Console.WriteLine( "Error: Unable to create install directories!" ); - return null; - } - - steam3.RequestDepotKey( depotId, appId ); - if ( !steam3.DepotKeys.ContainsKey( depotId ) ) - { - Console.WriteLine( "No valid depot key for {0}, unable to download.", depotId ); - return null; - } - - byte[] depotKey = steam3.DepotKeys[ depotId ]; - - var info = new DepotDownloadInfo( depotId, manifestId, installDir, contentName ); - info.depotKey = depotKey; - return info; - } - - private class ChunkMatch - { - public ChunkMatch( ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk ) - { - OldChunk = oldChunk; - NewChunk = newChunk; - } - public ProtoManifest.ChunkData OldChunk { get; private set; } - public ProtoManifest.ChunkData NewChunk { get; private set; } - } - - private static async Task DownloadSteam3Async( uint appId, List depots ) - { - ulong TotalBytesCompressed = 0; - ulong TotalBytesUncompressed = 0; - - foreach ( var depot in depots ) - { - ulong DepotBytesCompressed = 0; - ulong DepotBytesUncompressed = 0; - - Console.WriteLine( "Downloading depot {0} - {1}", depot.id, depot.contentName ); - - CancellationTokenSource cts = new CancellationTokenSource(); - cdnPool.ExhaustedToken = cts; - - ProtoManifest oldProtoManifest = null; - ProtoManifest newProtoManifest = null; - string configDir = Path.Combine( depot.installDir, CONFIG_DIR ); - - ulong lastManifestId = INVALID_MANIFEST_ID; - DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue( depot.id, out lastManifestId ); - - // In case we have an early exit, this will force equiv of verifyall next run. - DepotConfigStore.Instance.InstalledManifestIDs[ depot.id ] = INVALID_MANIFEST_ID; - DepotConfigStore.Save(); - - if ( lastManifestId != INVALID_MANIFEST_ID ) - { - var oldManifestFileName = Path.Combine( configDir, string.Format( "{0}_{1}.bin", depot.id, lastManifestId ) ); - - if (File.Exists(oldManifestFileName)) - { - byte[] expectedChecksum, currentChecksum; - - try - { - expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); - } - catch (IOException) - { - expectedChecksum = null; - } - - oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum); - - if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) - { - // We only have to show this warning if the old manifest ID was different - if (lastManifestId != depot.manifestId) - Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); - oldProtoManifest = null; - } - } - } - - if ( lastManifestId == depot.manifestId && oldProtoManifest != null ) - { - newProtoManifest = oldProtoManifest; - Console.WriteLine( "Already have manifest {0} for depot {1}.", depot.manifestId, depot.id ); - } - else - { - var newManifestFileName = Path.Combine( configDir, string.Format( "{0}_{1}.bin", depot.id, depot.manifestId ) ); - if ( newManifestFileName != null ) - { - byte[] expectedChecksum, currentChecksum; - - try - { - expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); - } - catch (IOException) - { - expectedChecksum = null; - } - - newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum); - - if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) - { - Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId); - newProtoManifest = null; - } - } - - if ( newProtoManifest != null ) - { - Console.WriteLine( "Already have manifest {0} for depot {1}.", depot.manifestId, depot.id ); - } - else - { - Console.Write( "Downloading depot manifest..." ); - - DepotManifest depotManifest = null; - - while ( depotManifest == null ) - { - Tuple connection = null; - try - { - connection = await cdnPool.GetConnectionForDepot( appId, depot.id, CancellationToken.None ); - - depotManifest = await cdnPool.CDNClient.DownloadManifestAsync( depot.id, depot.manifestId, - connection.Item1, connection.Item2, depot.depotKey ).ConfigureAwait(false); - - cdnPool.ReturnConnection( connection ); - } - catch ( SteamKitWebRequestException e ) - { - cdnPool.ReturnBrokenConnection( connection ); - - if ( e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden ) - { - Console.WriteLine( "Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId ); - break; - } - else - { - Console.WriteLine( "Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode ); - } - } - catch ( Exception e ) - { - cdnPool.ReturnBrokenConnection( connection ); - Console.WriteLine( "Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message ); - } - } - - if ( depotManifest == null ) - { - Console.WriteLine( "\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id ); - return; - } - - byte[] checksum; - - newProtoManifest = new ProtoManifest( depotManifest, depot.manifestId ); - newProtoManifest.SaveToFile( newManifestFileName, out checksum ); - File.WriteAllBytes( newManifestFileName + ".sha", checksum ); - - Console.WriteLine( " Done!" ); - } - } - - newProtoManifest.Files.Sort( ( x, y ) => string.Compare( x.FileName, y.FileName, StringComparison.Ordinal ) ); - - Console.WriteLine( "Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime ); - - if ( Config.DownloadManifestOnly ) - { - StringBuilder manifestBuilder = new StringBuilder(); - string txtManifest = Path.Combine( depot.installDir, string.Format( "manifest_{0}_{1}.txt", depot.id, depot.manifestId ) ); - manifestBuilder.Append( string.Format( "{0}\n\n", newProtoManifest.CreationTime ) ); - - foreach ( var file in newProtoManifest.Files ) - { - if ( file.Flags.HasFlag( EDepotFileFlag.Directory ) ) - continue; - - manifestBuilder.Append( string.Format( "{0}\n", file.FileName ) ); - manifestBuilder.Append( string.Format( "\t{0}\n", file.TotalSize ) ); - manifestBuilder.Append( string.Format( "\t{0}\n", BitConverter.ToString( file.FileHash ).Replace( "-", "" ) ) ); - } - - File.WriteAllText( txtManifest, manifestBuilder.ToString() ); - continue; - } - - ulong complete_download_size = 0; - ulong size_downloaded = 0; - string stagingDir = Path.Combine( depot.installDir, STAGING_DIR ); - - var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where( f => TestIsFileIncluded( f.FileName ) ).ToList(); - - // Pre-process - filesAfterExclusions.ForEach( file => - { - var fileFinalPath = Path.Combine( depot.installDir, file.FileName ); - var fileStagingPath = Path.Combine( stagingDir, file.FileName ); - - if ( file.Flags.HasFlag( EDepotFileFlag.Directory ) ) - { - Directory.CreateDirectory( fileFinalPath ); - Directory.CreateDirectory( fileStagingPath ); - } - else - { - // Some manifests don't explicitly include all necessary directories - Directory.CreateDirectory( Path.GetDirectoryName( fileFinalPath ) ); - Directory.CreateDirectory( Path.GetDirectoryName( fileStagingPath ) ); - - complete_download_size += file.TotalSize; - } - } ); - - var semaphore = new SemaphoreSlim( Config.MaxDownloads ); - var files = filesAfterExclusions.Where( f => !f.Flags.HasFlag( EDepotFileFlag.Directory ) ).ToArray(); - var tasks = new Task[ files.Length ]; - for ( var i = 0; i < files.Length; i++ ) - { - var file = files[ i ]; - var task = Task.Run( async () => - { - cts.Token.ThrowIfCancellationRequested(); - - try - { - await semaphore.WaitAsync().ConfigureAwait( false ); - cts.Token.ThrowIfCancellationRequested(); - - string fileFinalPath = Path.Combine( depot.installDir, file.FileName ); - string fileStagingPath = Path.Combine( stagingDir, file.FileName ); - - // This may still exist if the previous run exited before cleanup - if ( File.Exists( fileStagingPath ) ) - { - File.Delete( fileStagingPath ); - } - - FileStream fs = null; - List neededChunks; - FileInfo fi = new FileInfo( fileFinalPath ); - if ( !fi.Exists ) - { - // create new file. need all chunks - fs = File.Create( fileFinalPath ); - fs.SetLength( ( long )file.TotalSize ); - neededChunks = new List( file.Chunks ); - } - else - { - // open existing - ProtoManifest.FileData oldManifestFile = null; - if ( oldProtoManifest != null ) - { - oldManifestFile = oldProtoManifest.Files.SingleOrDefault( f => f.FileName == file.FileName ); - } - - if ( oldManifestFile != null ) - { - neededChunks = new List(); - - if ( Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual( file.FileHash ) ) - { - // we have a version of this file, but it doesn't fully match what we want - - var matchingChunks = new List(); - - foreach ( var chunk in file.Chunks ) - { - var oldChunk = oldManifestFile.Chunks.FirstOrDefault( c => c.ChunkID.SequenceEqual( chunk.ChunkID ) ); - if ( oldChunk != null ) - { - matchingChunks.Add( new ChunkMatch( oldChunk, chunk ) ); - } - else - { - neededChunks.Add( chunk ); - } - } - - File.Move( fileFinalPath, fileStagingPath ); - - fs = File.Open( fileFinalPath, FileMode.Create ); - fs.SetLength( ( long )file.TotalSize ); - - using ( var fsOld = File.Open( fileStagingPath, FileMode.Open ) ) - { - foreach ( var match in matchingChunks ) - { - fsOld.Seek( ( long )match.OldChunk.Offset, SeekOrigin.Begin ); - - byte[] tmp = new byte[ match.OldChunk.UncompressedLength ]; - fsOld.Read( tmp, 0, tmp.Length ); - - byte[] adler = Util.AdlerHash( tmp ); - if ( !adler.SequenceEqual( match.OldChunk.Checksum ) ) - { - neededChunks.Add( match.NewChunk ); - } - else - { - fs.Seek( ( long )match.NewChunk.Offset, SeekOrigin.Begin ); - fs.Write( tmp, 0, tmp.Length ); - } - } - } - - File.Delete( fileStagingPath ); - } - } - else - { - // No old manifest or file not in old manifest. We must validate. - - fs = File.Open( fileFinalPath, FileMode.Open ); - if ( ( ulong )fi.Length != file.TotalSize ) - { - fs.SetLength( ( long )file.TotalSize ); - } - - neededChunks = Util.ValidateSteam3FileChecksums( fs, file.Chunks.OrderBy( x => x.Offset ).ToArray() ); - } - - if ( neededChunks.Count() == 0 ) - { - size_downloaded += file.TotalSize; - Console.WriteLine( "{0,6:#00.00}% {1}", ( ( float )size_downloaded / ( float )complete_download_size ) * 100.0f, fileFinalPath ); - if ( fs != null ) - fs.Dispose(); - return; - } - else - { - size_downloaded += ( file.TotalSize - ( ulong )neededChunks.Select( x => ( long )x.UncompressedLength ).Sum() ); - } - } - - foreach ( var chunk in neededChunks ) - { - if ( cts.IsCancellationRequested ) break; - - string chunkID = Util.EncodeHexString( chunk.ChunkID ); - CDNClient.DepotChunk chunkData = null; - - while ( !cts.IsCancellationRequested ) - { - Tuple connection; - try - { - connection = await cdnPool.GetConnectionForDepot( appId, depot.id, cts.Token ); - } - catch ( OperationCanceledException ) - { - break; - } - - DepotManifest.ChunkData data = new DepotManifest.ChunkData(); - data.ChunkID = chunk.ChunkID; - data.Checksum = chunk.Checksum; - data.Offset = chunk.Offset; - data.CompressedLength = chunk.CompressedLength; - data.UncompressedLength = chunk.UncompressedLength; - - try - { - chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync( depot.id, data, - connection.Item1, connection.Item2, depot.depotKey ).ConfigureAwait( false ); - cdnPool.ReturnConnection( connection ); - break; - } - catch ( SteamKitWebRequestException e ) - { - cdnPool.ReturnBrokenConnection( connection ); - - if ( e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden ) - { - Console.WriteLine( "Encountered 401 for chunk {0}. Aborting.", chunkID ); - cts.Cancel(); - break; - } - else - { - Console.WriteLine( "Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode ); - } - } - catch ( TaskCanceledException ) - { - Console.WriteLine( "Connection timeout downloading chunk {0}", chunkID ); - } - catch ( Exception e ) - { - cdnPool.ReturnBrokenConnection( connection ); - Console.WriteLine( "Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message ); - } - } - - if ( chunkData == null ) - { - Console.WriteLine( "Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id ); - cts.Cancel(); - } - - // Throw the cancellation exception if requested so that this task is marked failed - cts.Token.ThrowIfCancellationRequested(); - - TotalBytesCompressed += chunk.CompressedLength; - DepotBytesCompressed += chunk.CompressedLength; - TotalBytesUncompressed += chunk.UncompressedLength; - DepotBytesUncompressed += chunk.UncompressedLength; - - fs.Seek( ( long )chunk.Offset, SeekOrigin.Begin ); - fs.Write( chunkData.Data, 0, chunkData.Data.Length ); - - size_downloaded += chunk.UncompressedLength; - } - - fs.Dispose(); - - Console.WriteLine( "{0,6:#00.00}% {1}", ( ( float )size_downloaded / ( float )complete_download_size ) * 100.0f, fileFinalPath ); - } - finally - { - semaphore.Release(); - } - } ); - - tasks[ i ] = task; - } - - await Task.WhenAll( tasks ).ConfigureAwait( false ); - - DepotConfigStore.Instance.InstalledManifestIDs[ depot.id ] = depot.manifestId; - DepotConfigStore.Save(); - - Console.WriteLine( "Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed ); - } - - Console.WriteLine( "Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count ); - } - } -} +using SteamKit2; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace DepotDownloader.Core +{ + public class ContentDownloaderException : System.Exception + { + public ContentDownloaderException( String value ) : base( value ) {} + } + + public static class ContentDownloader + { + public const uint INVALID_APP_ID = uint.MaxValue; + public const uint INVALID_DEPOT_ID = uint.MaxValue; + public const ulong INVALID_MANIFEST_ID = ulong.MaxValue; + public const string DEFAULT_BRANCH = "Public"; + + public static DownloadConfig Config = new DownloadConfig(); + + private static Steam3Session steam3; + private static Steam3Session.Credentials steam3Credentials; + private static CDNClientPool cdnPool; + + private const string DEFAULT_DOWNLOAD_DIR = "depots"; + private const string CONFIG_DIR = ".DepotDownloader"; + private static readonly string STAGING_DIR = Path.Combine( CONFIG_DIR, "staging" ); + + private sealed class DepotDownloadInfo + { + public uint id { get; private set; } + public string installDir { get; private set; } + public string contentName { get; private set; } + + public ulong manifestId { get; private set; } + public byte[] depotKey; + + public DepotDownloadInfo( uint depotid, ulong manifestId, string installDir, string contentName ) + { + this.id = depotid; + this.manifestId = manifestId; + this.installDir = installDir; + this.contentName = contentName; + } + } + + static bool CreateDirectories( uint depotId, uint depotVersion, out string installDir ) + { + installDir = null; + try + { + if ( string.IsNullOrWhiteSpace( ContentDownloader.Config.InstallDirectory ) ) + { + Directory.CreateDirectory( DEFAULT_DOWNLOAD_DIR ); + + string depotPath = Path.Combine( DEFAULT_DOWNLOAD_DIR, depotId.ToString() ); + Directory.CreateDirectory( depotPath ); + + installDir = Path.Combine( depotPath, depotVersion.ToString() ); + Directory.CreateDirectory( installDir ); + + Directory.CreateDirectory( Path.Combine( installDir, CONFIG_DIR ) ); + Directory.CreateDirectory( Path.Combine( installDir, STAGING_DIR ) ); + } + else + { + Directory.CreateDirectory( ContentDownloader.Config.InstallDirectory ); + + installDir = ContentDownloader.Config.InstallDirectory; + + Directory.CreateDirectory( Path.Combine( installDir, CONFIG_DIR ) ); + Directory.CreateDirectory( Path.Combine( installDir, STAGING_DIR ) ); + } + } + catch + { + return false; + } + + return true; + } + + static bool TestIsFileIncluded( string filename ) + { + if ( !Config.UsingFileList ) + return true; + + foreach ( string fileListEntry in Config.FilesToDownload ) + { + if ( fileListEntry.Equals( filename, StringComparison.OrdinalIgnoreCase ) ) + return true; + } + + foreach ( Regex rgx in Config.FilesToDownloadRegex ) + { + Match m = rgx.Match( filename ); + + if ( m.Success ) + return true; + } + + return false; + } + + static bool AccountHasAccess( uint depotId ) + { + if ( steam3 == null || steam3.steamUser.SteamID == null || ( steam3.Licenses == null && steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser ) ) + return false; + + IEnumerable licenseQuery; + if ( steam3.steamUser.SteamID.AccountType == EAccountType.AnonUser ) + { + licenseQuery = new List() { 17906 }; + } + else + { + licenseQuery = steam3.Licenses.Select( x => x.PackageID ).Distinct(); + } + + steam3.RequestPackageInfo( licenseQuery ); + + foreach ( var license in licenseQuery ) + { + SteamApps.PICSProductInfoCallback.PICSProductInfo package; + if ( steam3.PackageInfo.TryGetValue( license, out package ) && package != null ) + { + if ( package.KeyValues[ "appids" ].Children.Any( child => child.AsUnsignedInteger() == depotId ) ) + return true; + + if ( package.KeyValues[ "depotids" ].Children.Any( child => child.AsUnsignedInteger() == depotId ) ) + return true; + } + } + + return false; + } + + internal static KeyValue GetSteam3AppSection( uint appId, EAppInfoSection section ) + { + if ( steam3 == null || steam3.AppInfo == null ) + { + return null; + } + + SteamApps.PICSProductInfoCallback.PICSProductInfo app; + if ( !steam3.AppInfo.TryGetValue( appId, out app ) || app == null ) + { + return null; + } + + KeyValue appinfo = app.KeyValues; + string section_key; + + switch ( section ) + { + case EAppInfoSection.Common: + section_key = "common"; + break; + case EAppInfoSection.Extended: + section_key = "extended"; + break; + case EAppInfoSection.Config: + section_key = "config"; + break; + case EAppInfoSection.Depots: + section_key = "depots"; + break; + default: + throw new NotImplementedException(); + } + + KeyValue section_kv = appinfo.Children.Where( c => c.Name == section_key ).FirstOrDefault(); + return section_kv; + } + + static uint GetSteam3AppBuildNumber( uint appId, string branch ) + { + if ( appId == INVALID_APP_ID ) + return 0; + + + KeyValue depots = ContentDownloader.GetSteam3AppSection( appId, EAppInfoSection.Depots ); + KeyValue branches = depots[ "branches" ]; + KeyValue node = branches[ branch ]; + + if ( node == KeyValue.Invalid ) + return 0; + + KeyValue buildid = node[ "buildid" ]; + + if ( buildid == KeyValue.Invalid ) + return 0; + + return uint.Parse( buildid.Value ); + } + + static ulong GetSteam3DepotManifest( uint depotId, uint appId, string branch ) + { + KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); + KeyValue depotChild = depots[ depotId.ToString() ]; + + if ( depotChild == KeyValue.Invalid ) + return INVALID_MANIFEST_ID; + + // Shared depots can either provide manifests, or leave you relying on their parent app. + // It seems that with the latter, "sharedinstall" will exist (and equals 2 in the one existance I know of). + // Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680. + if ( depotChild[ "manifests" ] == KeyValue.Invalid && depotChild[ "depotfromapp" ] != KeyValue.Invalid ) + { + uint otherAppId = depotChild["depotfromapp"].AsUnsignedInteger(); + if ( otherAppId == appId ) + { + // This shouldn't ever happen, but ya never know with Valve. Don't infinite loop. + Console.WriteLine( "App {0}, Depot {1} has depotfromapp of {2}!", + appId, depotId, otherAppId ); + return INVALID_MANIFEST_ID; + } + + steam3.RequestAppInfo( otherAppId ); + + return GetSteam3DepotManifest( depotId, otherAppId, branch ); + } + + var manifests = depotChild[ "manifests" ]; + var manifests_encrypted = depotChild[ "encryptedmanifests" ]; + + if ( manifests.Children.Count == 0 && manifests_encrypted.Children.Count == 0 ) + return INVALID_MANIFEST_ID; + + var node = manifests[ branch ]; + + if ( branch != "Public" && node == KeyValue.Invalid ) + { + var node_encrypted = manifests_encrypted[ branch ]; + if ( node_encrypted != KeyValue.Invalid ) + { + string password = Config.BetaPassword; + if ( password == null ) + { + Console.Write( "Please enter the password for branch {0}: ", branch ); + Config.BetaPassword = password = Console.ReadLine(); + } + + var encrypted_v1 = node_encrypted[ "encrypted_gid" ]; + var encrypted_v2 = node_encrypted[ "encrypted_gid_2" ]; + + if ( encrypted_v1 != KeyValue.Invalid ) + { + byte[] input = Util.DecodeHexString( encrypted_v1.Value ); + byte[] manifest_bytes = CryptoHelper.VerifyAndDecryptPassword( input, password ); + + if ( manifest_bytes == null ) + { + Console.WriteLine( "Password was invalid for branch {0}", branch ); + return INVALID_MANIFEST_ID; + } + + return BitConverter.ToUInt64( manifest_bytes, 0 ); + } + else if ( encrypted_v2 != KeyValue.Invalid ) + { + // Submit the password to Steam now to get encryption keys + steam3.CheckAppBetaPassword( appId, Config.BetaPassword ); + + if ( !steam3.AppBetaPasswords.ContainsKey( branch ) ) + { + Console.WriteLine( "Password was invalid for branch {0}", branch ); + return INVALID_MANIFEST_ID; + } + + byte[] input = Util.DecodeHexString( encrypted_v2.Value ); + byte[] manifest_bytes; + try + { + manifest_bytes = CryptoHelper.SymmetricDecryptECB( input, steam3.AppBetaPasswords[ branch ] ); + } + catch ( Exception e ) + { + Console.WriteLine( "Failed to decrypt branch {0}: {1}", branch, e.Message ); + return INVALID_MANIFEST_ID; + } + + return BitConverter.ToUInt64( manifest_bytes, 0 ); + } + else + { + Console.WriteLine( "Unhandled depot encryption for depotId {0}", depotId ); + return INVALID_MANIFEST_ID; + } + + } + + return INVALID_MANIFEST_ID; + } + + if ( node.Value == null ) + return INVALID_MANIFEST_ID; + + return UInt64.Parse( node.Value ); + } + + static string GetAppOrDepotName( uint depotId, uint appId ) + { + if ( depotId == INVALID_DEPOT_ID ) + { + KeyValue info = GetSteam3AppSection( appId, EAppInfoSection.Common ); + + if ( info == null ) + return String.Empty; + + return info[ "name" ].AsString(); + } + else + { + KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); + + if ( depots == null ) + return String.Empty; + + KeyValue depotChild = depots[ depotId.ToString() ]; + + if ( depotChild == null ) + return String.Empty; + + return depotChild[ "name" ].AsString(); + } + } + + public static bool InitializeSteam3( string username, string password ) + { + string loginKey = null; + + if ( username != null && Config.RememberPassword ) + { + _ = AccountSettingsStore.Instance.LoginKeys.TryGetValue( username, out loginKey ); + } + + steam3 = new Steam3Session( + new SteamUser.LogOnDetails() + { + Username = username, + Password = loginKey == null ? password : null, + ShouldRememberPassword = Config.RememberPassword, + LoginKey = loginKey, + LoginID = Config.LoginID ?? 0x534B32, // "SK2" + } + ); + + steam3Credentials = steam3.WaitForCredentials(); + + if ( !steam3Credentials.IsValid ) + { + Console.WriteLine( "Unable to get steam3 credentials." ); + return false; + } + + cdnPool = new CDNClientPool( steam3 ); + return true; + } + + public static void ShutdownSteam3() + { + if (cdnPool != null) + { + cdnPool.Shutdown(); + cdnPool = null; + } + + if ( steam3 == null ) + return; + + steam3.TryWaitForLoginKey(); + steam3.Disconnect(); + } + + public static async Task DownloadPubfileAsync( uint appId, ulong publishedFileId ) + { + var details = steam3.GetPubfileItemInfo( appId, publishedFileId ); + + if ( details?.manifest_id > 0 ) + { + await DownloadAppAsync( appId, appId, details.manifest_id, DEFAULT_BRANCH, null, null, null, false, true ); + } + else + { + Console.WriteLine( "Unable to locate manifest ID for published file {0}", publishedFileId ); + } + } + + public static async Task DownloadAppAsync( uint appId, uint depotId, ulong manifestId, string branch, string os, string arch, string language, bool lv, bool isUgc ) + { + // Load our configuration data containing the depots currently installed + string configPath = ContentDownloader.Config.InstallDirectory; + if (string.IsNullOrWhiteSpace(configPath)) + { + configPath = DEFAULT_DOWNLOAD_DIR; + } + + Directory.CreateDirectory(Path.Combine(configPath, CONFIG_DIR)); + DepotConfigStore.LoadFromFile(Path.Combine(configPath, CONFIG_DIR, "depot.config")); + + if ( steam3 != null ) + steam3.RequestAppInfo( appId ); + + if ( !AccountHasAccess( appId ) ) + { + if ( steam3.RequestFreeAppLicense( appId ) ) + { + Console.WriteLine( "Obtained FreeOnDemand license for app {0}", appId ); + + // Fetch app info again in case we didn't get it fully without a license. + steam3.RequestAppInfo( appId, true ); + } + else + { + string contentName = GetAppOrDepotName( INVALID_DEPOT_ID, appId ); + throw new ContentDownloaderException( String.Format( "App {0} ({1}) is not available from this account.", appId, contentName ) ); + } + } + + var depotIDs = new List(); + KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); + + if ( isUgc ) + { + var workshopDepot = depots["workshopdepot"].AsUnsignedInteger(); + if (workshopDepot != 0) + depotId = workshopDepot; + + depotIDs.Add( depotId ); + } + else + { + Console.WriteLine( "Using app branch: '{0}'.", branch ); + + if ( depots != null ) + { + foreach ( var depotSection in depots.Children ) + { + uint id = INVALID_DEPOT_ID; + if ( depotSection.Children.Count == 0 ) + continue; + + if ( !uint.TryParse( depotSection.Name, out id ) ) + continue; + + if ( depotId != INVALID_DEPOT_ID && id != depotId ) + continue; + + if ( depotId == INVALID_DEPOT_ID ) + { + var depotConfig = depotSection[ "config" ]; + if ( depotConfig != KeyValue.Invalid ) + { + if ( !Config.DownloadAllPlatforms && + depotConfig["oslist"] != KeyValue.Invalid && + !string.IsNullOrWhiteSpace( depotConfig["oslist"].Value ) ) + { + var oslist = depotConfig["oslist"].Value.Split( ',' ); + if ( Array.IndexOf( oslist, os ?? Util.GetSteamOS() ) == -1 ) + continue; + } + + if ( depotConfig["osarch"] != KeyValue.Invalid && + !string.IsNullOrWhiteSpace( depotConfig["osarch"].Value ) ) + { + var depotArch = depotConfig["osarch"].Value; + if ( depotArch != ( arch ?? Util.GetSteamArch() ) ) + continue; + } + + if ( !Config.DownloadAllLanguages && + depotConfig["language"] != KeyValue.Invalid && + !string.IsNullOrWhiteSpace( depotConfig["language"].Value ) ) + { + var depotLang = depotConfig["language"].Value; + if ( depotLang != ( language ?? "english" ) ) + continue; + } + + if ( !lv && + depotConfig["lowviolence"] != KeyValue.Invalid && + depotConfig["lowviolence"].AsBoolean() ) + continue; + } + } + + depotIDs.Add( id ); + } + } + if ( depotIDs == null || ( depotIDs.Count == 0 && depotId == INVALID_DEPOT_ID ) ) + { + throw new ContentDownloaderException( String.Format( "Couldn't find any depots to download for app {0}", appId ) ); + } + else if ( depotIDs.Count == 0 ) + { + throw new ContentDownloaderException( String.Format( "Depot {0} not listed for app {1}", depotId, appId ) ); + } + } + + var infos = new List(); + + foreach ( var depot in depotIDs ) + { + var info = GetDepotInfo( depot, appId, manifestId, branch ); + if ( info != null ) + { + infos.Add( info ); + } + } + + try + { + await DownloadSteam3Async( appId, infos ).ConfigureAwait( false ); + } + catch ( OperationCanceledException ) + { + Console.WriteLine( "App {0} was not completely downloaded.", appId ); + throw; + } + } + + static DepotDownloadInfo GetDepotInfo( uint depotId, uint appId, ulong manifestId, string branch ) + { + if ( steam3 != null && appId != INVALID_APP_ID ) + steam3.RequestAppInfo( ( uint )appId ); + + string contentName = GetAppOrDepotName( depotId, appId ); + + if ( !AccountHasAccess( depotId ) ) + { + Console.WriteLine( "Depot {0} ({1}) is not available from this account.", depotId, contentName ); + + return null; + } + + // Skip requesting an app ticket + steam3.AppTickets[ depotId ] = null; + + if (manifestId == INVALID_MANIFEST_ID) + { + manifestId = GetSteam3DepotManifest(depotId, appId, branch); + if (manifestId == INVALID_MANIFEST_ID && branch != "public") + { + Console.WriteLine("Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch); + branch = "public"; + manifestId = GetSteam3DepotManifest(depotId, appId, branch); + } + + if (manifestId == INVALID_MANIFEST_ID) + { + Console.WriteLine("Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName); + return null; + } + } + + uint uVersion = GetSteam3AppBuildNumber( appId, branch ); + + string installDir; + if ( !CreateDirectories( depotId, uVersion, out installDir ) ) + { + Console.WriteLine( "Error: Unable to create install directories!" ); + return null; + } + + steam3.RequestDepotKey( depotId, appId ); + if ( !steam3.DepotKeys.ContainsKey( depotId ) ) + { + Console.WriteLine( "No valid depot key for {0}, unable to download.", depotId ); + return null; + } + + byte[] depotKey = steam3.DepotKeys[ depotId ]; + + var info = new DepotDownloadInfo( depotId, manifestId, installDir, contentName ); + info.depotKey = depotKey; + return info; + } + + private class ChunkMatch + { + public ChunkMatch( ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk ) + { + OldChunk = oldChunk; + NewChunk = newChunk; + } + public ProtoManifest.ChunkData OldChunk { get; private set; } + public ProtoManifest.ChunkData NewChunk { get; private set; } + } + + private static async Task DownloadSteam3Async( uint appId, List depots ) + { + ulong TotalBytesCompressed = 0; + ulong TotalBytesUncompressed = 0; + + foreach ( var depot in depots ) + { + ulong DepotBytesCompressed = 0; + ulong DepotBytesUncompressed = 0; + + Console.WriteLine( "Downloading depot {0} - {1}", depot.id, depot.contentName ); + + CancellationTokenSource cts = new CancellationTokenSource(); + cdnPool.ExhaustedToken = cts; + + ProtoManifest oldProtoManifest = null; + ProtoManifest newProtoManifest = null; + string configDir = Path.Combine( depot.installDir, CONFIG_DIR ); + + ulong lastManifestId = INVALID_MANIFEST_ID; + DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue( depot.id, out lastManifestId ); + + // In case we have an early exit, this will force equiv of verifyall next run. + DepotConfigStore.Instance.InstalledManifestIDs[ depot.id ] = INVALID_MANIFEST_ID; + DepotConfigStore.Save(); + + if ( lastManifestId != INVALID_MANIFEST_ID ) + { + var oldManifestFileName = Path.Combine( configDir, string.Format( "{0}_{1}.bin", depot.id, lastManifestId ) ); + + if (File.Exists(oldManifestFileName)) + { + byte[] expectedChecksum, currentChecksum; + + try + { + expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha"); + } + catch (IOException) + { + expectedChecksum = null; + } + + oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum); + + if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)) + { + // We only have to show this warning if the old manifest ID was different + if (lastManifestId != depot.manifestId) + Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId); + oldProtoManifest = null; + } + } + } + + if ( lastManifestId == depot.manifestId && oldProtoManifest != null ) + { + newProtoManifest = oldProtoManifest; + Console.WriteLine( "Already have manifest {0} for depot {1}.", depot.manifestId, depot.id ); + } + else + { + var newManifestFileName = Path.Combine( configDir, string.Format( "{0}_{1}.bin", depot.id, depot.manifestId ) ); + if ( newManifestFileName != null ) + { + byte[] expectedChecksum, currentChecksum; + + try + { + expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha"); + } + catch (IOException) + { + expectedChecksum = null; + } + + newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum); + + if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))) + { + Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId); + newProtoManifest = null; + } + } + + if ( newProtoManifest != null ) + { + Console.WriteLine( "Already have manifest {0} for depot {1}.", depot.manifestId, depot.id ); + } + else + { + Console.Write( "Downloading depot manifest..." ); + + DepotManifest depotManifest = null; + + while ( depotManifest == null ) + { + Tuple connection = null; + try + { + connection = await cdnPool.GetConnectionForDepot( appId, depot.id, CancellationToken.None ); + + depotManifest = await cdnPool.CDNClient.DownloadManifestAsync( depot.id, depot.manifestId, + connection.Item1, connection.Item2, depot.depotKey ).ConfigureAwait(false); + + cdnPool.ReturnConnection( connection ); + } + catch ( SteamKitWebRequestException e ) + { + cdnPool.ReturnBrokenConnection( connection ); + + if ( e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden ) + { + Console.WriteLine( "Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId ); + break; + } + else + { + Console.WriteLine( "Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode ); + } + } + catch ( Exception e ) + { + cdnPool.ReturnBrokenConnection( connection ); + Console.WriteLine( "Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message ); + } + } + + if ( depotManifest == null ) + { + Console.WriteLine( "\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id ); + return; + } + + byte[] checksum; + + newProtoManifest = new ProtoManifest( depotManifest, depot.manifestId ); + newProtoManifest.SaveToFile( newManifestFileName, out checksum ); + File.WriteAllBytes( newManifestFileName + ".sha", checksum ); + + Console.WriteLine( " Done!" ); + } + } + + newProtoManifest.Files.Sort( ( x, y ) => string.Compare( x.FileName, y.FileName, StringComparison.Ordinal ) ); + + Console.WriteLine( "Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime ); + + if ( Config.DownloadManifestOnly ) + { + StringBuilder manifestBuilder = new StringBuilder(); + string txtManifest = Path.Combine( depot.installDir, string.Format( "manifest_{0}_{1}.txt", depot.id, depot.manifestId ) ); + manifestBuilder.Append( string.Format( "{0}\n\n", newProtoManifest.CreationTime ) ); + + foreach ( var file in newProtoManifest.Files ) + { + if ( file.Flags.HasFlag( EDepotFileFlag.Directory ) ) + continue; + + manifestBuilder.Append( string.Format( "{0}\n", file.FileName ) ); + manifestBuilder.Append( string.Format( "\t{0}\n", file.TotalSize ) ); + manifestBuilder.Append( string.Format( "\t{0}\n", BitConverter.ToString( file.FileHash ).Replace( "-", "" ) ) ); + } + + File.WriteAllText( txtManifest, manifestBuilder.ToString() ); + continue; + } + + ulong complete_download_size = 0; + ulong size_downloaded = 0; + string stagingDir = Path.Combine( depot.installDir, STAGING_DIR ); + + var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where( f => TestIsFileIncluded( f.FileName ) ).ToList(); + + // Pre-process + filesAfterExclusions.ForEach( file => + { + var fileFinalPath = Path.Combine( depot.installDir, file.FileName ); + var fileStagingPath = Path.Combine( stagingDir, file.FileName ); + + if ( file.Flags.HasFlag( EDepotFileFlag.Directory ) ) + { + Directory.CreateDirectory( fileFinalPath ); + Directory.CreateDirectory( fileStagingPath ); + } + else + { + // Some manifests don't explicitly include all necessary directories + Directory.CreateDirectory( Path.GetDirectoryName( fileFinalPath ) ); + Directory.CreateDirectory( Path.GetDirectoryName( fileStagingPath ) ); + + complete_download_size += file.TotalSize; + } + } ); + + var semaphore = new SemaphoreSlim( Config.MaxDownloads ); + var files = filesAfterExclusions.Where( f => !f.Flags.HasFlag( EDepotFileFlag.Directory ) ).ToArray(); + var tasks = new Task[ files.Length ]; + for ( var i = 0; i < files.Length; i++ ) + { + var file = files[ i ]; + var task = Task.Run( async () => + { + cts.Token.ThrowIfCancellationRequested(); + + try + { + await semaphore.WaitAsync().ConfigureAwait( false ); + cts.Token.ThrowIfCancellationRequested(); + + string fileFinalPath = Path.Combine( depot.installDir, file.FileName ); + string fileStagingPath = Path.Combine( stagingDir, file.FileName ); + + // This may still exist if the previous run exited before cleanup + if ( File.Exists( fileStagingPath ) ) + { + File.Delete( fileStagingPath ); + } + + FileStream fs = null; + List neededChunks; + FileInfo fi = new FileInfo( fileFinalPath ); + if ( !fi.Exists ) + { + // create new file. need all chunks + fs = File.Create( fileFinalPath ); + fs.SetLength( ( long )file.TotalSize ); + neededChunks = new List( file.Chunks ); + } + else + { + // open existing + ProtoManifest.FileData oldManifestFile = null; + if ( oldProtoManifest != null ) + { + oldManifestFile = oldProtoManifest.Files.SingleOrDefault( f => f.FileName == file.FileName ); + } + + if ( oldManifestFile != null ) + { + neededChunks = new List(); + + if ( Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual( file.FileHash ) ) + { + // we have a version of this file, but it doesn't fully match what we want + + var matchingChunks = new List(); + + foreach ( var chunk in file.Chunks ) + { + var oldChunk = oldManifestFile.Chunks.FirstOrDefault( c => c.ChunkID.SequenceEqual( chunk.ChunkID ) ); + if ( oldChunk != null ) + { + matchingChunks.Add( new ChunkMatch( oldChunk, chunk ) ); + } + else + { + neededChunks.Add( chunk ); + } + } + + File.Move( fileFinalPath, fileStagingPath ); + + fs = File.Open( fileFinalPath, FileMode.Create ); + fs.SetLength( ( long )file.TotalSize ); + + using ( var fsOld = File.Open( fileStagingPath, FileMode.Open ) ) + { + foreach ( var match in matchingChunks ) + { + fsOld.Seek( ( long )match.OldChunk.Offset, SeekOrigin.Begin ); + + byte[] tmp = new byte[ match.OldChunk.UncompressedLength ]; + fsOld.Read( tmp, 0, tmp.Length ); + + byte[] adler = Util.AdlerHash( tmp ); + if ( !adler.SequenceEqual( match.OldChunk.Checksum ) ) + { + neededChunks.Add( match.NewChunk ); + } + else + { + fs.Seek( ( long )match.NewChunk.Offset, SeekOrigin.Begin ); + fs.Write( tmp, 0, tmp.Length ); + } + } + } + + File.Delete( fileStagingPath ); + } + } + else + { + // No old manifest or file not in old manifest. We must validate. + + fs = File.Open( fileFinalPath, FileMode.Open ); + if ( ( ulong )fi.Length != file.TotalSize ) + { + fs.SetLength( ( long )file.TotalSize ); + } + + neededChunks = Util.ValidateSteam3FileChecksums( fs, file.Chunks.OrderBy( x => x.Offset ).ToArray() ); + } + + if ( neededChunks.Count() == 0 ) + { + size_downloaded += file.TotalSize; + Console.WriteLine( "{0,6:#00.00}% {1}", ( ( float )size_downloaded / ( float )complete_download_size ) * 100.0f, fileFinalPath ); + if ( fs != null ) + fs.Dispose(); + return; + } + else + { + size_downloaded += ( file.TotalSize - ( ulong )neededChunks.Select( x => ( long )x.UncompressedLength ).Sum() ); + } + } + + foreach ( var chunk in neededChunks ) + { + if ( cts.IsCancellationRequested ) break; + + string chunkID = Util.EncodeHexString( chunk.ChunkID ); + CDNClient.DepotChunk chunkData = null; + + while ( !cts.IsCancellationRequested ) + { + Tuple connection; + try + { + connection = await cdnPool.GetConnectionForDepot( appId, depot.id, cts.Token ); + } + catch ( OperationCanceledException ) + { + break; + } + + DepotManifest.ChunkData data = new DepotManifest.ChunkData(); + data.ChunkID = chunk.ChunkID; + data.Checksum = chunk.Checksum; + data.Offset = chunk.Offset; + data.CompressedLength = chunk.CompressedLength; + data.UncompressedLength = chunk.UncompressedLength; + + try + { + chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync( depot.id, data, + connection.Item1, connection.Item2, depot.depotKey ).ConfigureAwait( false ); + cdnPool.ReturnConnection( connection ); + break; + } + catch ( SteamKitWebRequestException e ) + { + cdnPool.ReturnBrokenConnection( connection ); + + if ( e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden ) + { + Console.WriteLine( "Encountered 401 for chunk {0}. Aborting.", chunkID ); + cts.Cancel(); + break; + } + else + { + Console.WriteLine( "Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode ); + } + } + catch ( TaskCanceledException ) + { + Console.WriteLine( "Connection timeout downloading chunk {0}", chunkID ); + } + catch ( Exception e ) + { + cdnPool.ReturnBrokenConnection( connection ); + Console.WriteLine( "Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message ); + } + } + + if ( chunkData == null ) + { + Console.WriteLine( "Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id ); + cts.Cancel(); + } + + // Throw the cancellation exception if requested so that this task is marked failed + cts.Token.ThrowIfCancellationRequested(); + + TotalBytesCompressed += chunk.CompressedLength; + DepotBytesCompressed += chunk.CompressedLength; + TotalBytesUncompressed += chunk.UncompressedLength; + DepotBytesUncompressed += chunk.UncompressedLength; + + fs.Seek( ( long )chunk.Offset, SeekOrigin.Begin ); + fs.Write( chunkData.Data, 0, chunkData.Data.Length ); + + size_downloaded += chunk.UncompressedLength; + } + + fs.Dispose(); + + Console.WriteLine( "{0,6:#00.00}% {1}", ( ( float )size_downloaded / ( float )complete_download_size ) * 100.0f, fileFinalPath ); + } + finally + { + semaphore.Release(); + } + } ); + + tasks[ i ] = task; + } + + await Task.WhenAll( tasks ).ConfigureAwait( false ); + + DepotConfigStore.Instance.InstalledManifestIDs[ depot.id ] = depot.manifestId; + DepotConfigStore.Save(); + + Console.WriteLine( "Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed ); + } + + Console.WriteLine( "Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count ); + } + } +} diff --git a/DepotDownloader/DepotConfigStore.cs b/DepotDownloader.Core/DepotConfigStore.cs similarity index 98% rename from DepotDownloader/DepotConfigStore.cs rename to DepotDownloader.Core/DepotConfigStore.cs index 79a414ab..9305a020 100644 --- a/DepotDownloader/DepotConfigStore.cs +++ b/DepotDownloader.Core/DepotConfigStore.cs @@ -4,7 +4,7 @@ using ProtoBuf; using System.IO; using System.IO.Compression; -namespace DepotDownloader +namespace DepotDownloader.Core { [ProtoContract] class DepotConfigStore diff --git a/DepotDownloader.Core/DepotDownloader.Core.csproj b/DepotDownloader.Core/DepotDownloader.Core.csproj new file mode 100644 index 00000000..91d31204 --- /dev/null +++ b/DepotDownloader.Core/DepotDownloader.Core.csproj @@ -0,0 +1,12 @@ + + + + netstandard2.0 + + + + + + + + diff --git a/DepotDownloader/DownloadConfig.cs b/DepotDownloader.Core/DownloadConfig.cs similarity index 92% rename from DepotDownloader/DownloadConfig.cs rename to DepotDownloader.Core/DownloadConfig.cs index 780fd56a..8024b182 100644 --- a/DepotDownloader/DownloadConfig.cs +++ b/DepotDownloader.Core/DownloadConfig.cs @@ -1,33 +1,33 @@ -using System.Collections.Generic; -using System.Text.RegularExpressions; - -namespace DepotDownloader -{ - class DownloadConfig - { - public int CellID { get; set; } - public bool DownloadAllPlatforms { get; set; } - public bool DownloadAllLanguages { get; set; } - public bool DownloadManifestOnly { get; set; } - public string InstallDirectory { get; set; } - - public bool UsingFileList { get; set; } - public List FilesToDownload { get; set; } - public List FilesToDownloadRegex { get; set; } - - public bool UsingExclusionList { get; set; } - - public string BetaPassword { get; set; } - - public bool VerifyAll { get; set; } - - public int MaxServers { get; set; } - public int MaxDownloads { get; set; } - - public string SuppliedPassword { get; set; } - public bool RememberPassword { get; set; } - - // A Steam LoginID to allow multiple concurrent connections - public uint? LoginID {get; set; } - } -} +using System.Collections.Generic; +using System.Text.RegularExpressions; + +namespace DepotDownloader.Core +{ + public class DownloadConfig + { + public int CellID { get; set; } + public bool DownloadAllPlatforms { get; set; } + public bool DownloadAllLanguages { get; set; } + public bool DownloadManifestOnly { get; set; } + public string InstallDirectory { get; set; } + + public bool UsingFileList { get; set; } + public List FilesToDownload { get; set; } + public List FilesToDownloadRegex { get; set; } + + public bool UsingExclusionList { get; set; } + + public string BetaPassword { get; set; } + + public bool VerifyAll { get; set; } + + public int MaxServers { get; set; } + public int MaxDownloads { get; set; } + + public string SuppliedPassword { get; set; } + public bool RememberPassword { get; set; } + + // A Steam LoginID to allow multiple concurrent connections + public uint? LoginID {get; set; } + } +} diff --git a/DepotDownloader/ProtoManifest.cs b/DepotDownloader.Core/ProtoManifest.cs similarity index 98% rename from DepotDownloader/ProtoManifest.cs rename to DepotDownloader.Core/ProtoManifest.cs index f521c5cc..e412d67e 100644 --- a/DepotDownloader/ProtoManifest.cs +++ b/DepotDownloader.Core/ProtoManifest.cs @@ -6,10 +6,10 @@ using System.IO.Compression; using ProtoBuf; using SteamKit2; -namespace DepotDownloader +namespace DepotDownloader.Core { [ProtoContract()] - class ProtoManifest + public class ProtoManifest { // Proto ctor private ProtoManifest() diff --git a/DepotDownloader/Steam3Session.cs b/DepotDownloader.Core/Steam3Session.cs similarity index 96% rename from DepotDownloader/Steam3Session.cs rename to DepotDownloader.Core/Steam3Session.cs index f622bca1..e3d34ecb 100644 --- a/DepotDownloader/Steam3Session.cs +++ b/DepotDownloader.Core/Steam3Session.cs @@ -1,700 +1,700 @@ -using SteamKit2; -using SteamKit2.Internal; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace DepotDownloader -{ - - class Steam3Session - { - public class Credentials - { - public bool LoggedOn { get; set; } - public ulong SessionToken { get; set; } - - public bool IsValid - { - get { return LoggedOn; } - } - } - - public ReadOnlyCollection Licenses - { - get; - private set; - } - - public Dictionary AppTickets { get; private set; } - public Dictionary AppTokens { get; private set; } - public Dictionary PackageTokens { get; private set; } - public Dictionary DepotKeys { get; private set; } - public ConcurrentDictionary> CDNAuthTokens { get; private set; } - public Dictionary AppInfo { get; private set; } - public Dictionary PackageInfo { get; private set; } - public Dictionary AppBetaPasswords { get; private set; } - - public SteamClient steamClient; - public SteamUser steamUser; - SteamApps steamApps; - SteamUnifiedMessages.UnifiedService steamPublishedFile; - - CallbackManager callbacks; - - bool authenticatedUser; - bool bConnected; - bool bConnecting; - bool bAborted; - bool bExpectingDisconnectRemote; - bool bDidDisconnect; - bool bDidReceiveLoginKey; - int connectionBackoff; - int seq; // more hack fixes - DateTime connectTime; - - // input - SteamUser.LogOnDetails logonDetails; - - // output - Credentials credentials; - - static readonly TimeSpan STEAM3_TIMEOUT = TimeSpan.FromSeconds( 30 ); - - - public Steam3Session( SteamUser.LogOnDetails details ) - { - this.logonDetails = details; - - this.authenticatedUser = details.Username != null; - this.credentials = new Credentials(); - this.bConnected = false; - this.bConnecting = false; - this.bAborted = false; - this.bExpectingDisconnectRemote = false; - this.bDidDisconnect = false; - this.bDidReceiveLoginKey = false; - this.seq = 0; - - this.AppTickets = new Dictionary(); - this.AppTokens = new Dictionary(); - this.PackageTokens = new Dictionary(); - this.DepotKeys = new Dictionary(); - this.CDNAuthTokens = new ConcurrentDictionary>(); - this.AppInfo = new Dictionary(); - this.PackageInfo = new Dictionary(); - this.AppBetaPasswords = new Dictionary(); - - this.steamClient = new SteamClient(); - - this.steamUser = this.steamClient.GetHandler(); - this.steamApps = this.steamClient.GetHandler(); - var steamUnifiedMessages = this.steamClient.GetHandler(); - this.steamPublishedFile = steamUnifiedMessages.CreateService(); - - this.callbacks = new CallbackManager( this.steamClient ); - - this.callbacks.Subscribe( ConnectedCallback ); - this.callbacks.Subscribe( DisconnectedCallback ); - this.callbacks.Subscribe( LogOnCallback ); - this.callbacks.Subscribe( SessionTokenCallback ); - this.callbacks.Subscribe( LicenseListCallback ); - this.callbacks.Subscribe( UpdateMachineAuthCallback ); - this.callbacks.Subscribe( LoginKeyCallback ); - - Console.Write( "Connecting to Steam3..." ); - - if ( authenticatedUser ) - { - FileInfo fi = new FileInfo( String.Format( "{0}.sentryFile", logonDetails.Username ) ); - if ( AccountSettingsStore.Instance.SentryData != null && AccountSettingsStore.Instance.SentryData.ContainsKey( logonDetails.Username ) ) - { - logonDetails.SentryFileHash = Util.SHAHash( AccountSettingsStore.Instance.SentryData[ logonDetails.Username ] ); - } - else if ( fi.Exists && fi.Length > 0 ) - { - var sentryData = File.ReadAllBytes( fi.FullName ); - logonDetails.SentryFileHash = Util.SHAHash( sentryData ); - AccountSettingsStore.Instance.SentryData[ logonDetails.Username ] = sentryData; - AccountSettingsStore.Save(); - } - } - - Connect(); - } - - public delegate bool WaitCondition(); - public bool WaitUntilCallback( Action submitter, WaitCondition waiter ) - { - while ( !bAborted && !waiter() ) - { - submitter(); - - int seq = this.seq; - do - { - WaitForCallbacks(); - } - while ( !bAborted && this.seq == seq && !waiter() ); - } - - return bAborted; - } - - public Credentials WaitForCredentials() - { - if ( credentials.IsValid || bAborted ) - return credentials; - - WaitUntilCallback( () => { }, () => { return credentials.IsValid; } ); - - return credentials; - } - - public void RequestAppInfo( uint appId, bool bForce = false ) - { - if ( ( AppInfo.ContainsKey( appId ) && !bForce ) || bAborted ) - return; - - bool completed = false; - Action cbMethodTokens = ( appTokens ) => - { - completed = true; - if ( appTokens.AppTokensDenied.Contains( appId ) ) - { - Console.WriteLine( "Insufficient privileges to get access token for app {0}", appId ); - } - - foreach ( var token_dict in appTokens.AppTokens ) - { - this.AppTokens[ token_dict.Key ] = token_dict.Value; - } - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.PICSGetAccessTokens( new List() { appId }, new List() { } ), cbMethodTokens ); - }, () => { return completed; } ); - - completed = false; - Action cbMethod = ( appInfo ) => - { - completed = !appInfo.ResponsePending; - - foreach ( var app_value in appInfo.Apps ) - { - var app = app_value.Value; - - Console.WriteLine( "Got AppInfo for {0}", app.ID ); - AppInfo[ app.ID ] = app; - } - - foreach ( var app in appInfo.UnknownApps ) - { - AppInfo[ app ] = null; - } - }; - - SteamApps.PICSRequest request = new SteamApps.PICSRequest( appId ); - if ( AppTokens.ContainsKey( appId ) ) - { - request.AccessToken = AppTokens[ appId ]; - request.Public = false; - } - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.PICSGetProductInfo( new List() { request }, new List() { } ), cbMethod ); - }, () => { return completed; } ); - } - - public void RequestPackageInfo( IEnumerable packageIds ) - { - List packages = packageIds.ToList(); - packages.RemoveAll( pid => PackageInfo.ContainsKey( pid ) ); - - if ( packages.Count == 0 || bAborted ) - return; - - bool completed = false; - Action cbMethod = ( packageInfo ) => - { - completed = !packageInfo.ResponsePending; - - foreach ( var package_value in packageInfo.Packages ) - { - var package = package_value.Value; - PackageInfo[ package.ID ] = package; - } - - foreach ( var package in packageInfo.UnknownPackages ) - { - PackageInfo[package] = null; - } - }; - - var packageRequests = new List(); - - foreach ( var package in packages ) - { - var request = new SteamApps.PICSRequest( package ); - - if ( PackageTokens.TryGetValue( package, out var token ) ) - { - request.AccessToken = token; - request.Public = false; - } - - packageRequests.Add( request ); - } - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.PICSGetProductInfo( new List(), packageRequests ), cbMethod ); - }, () => { return completed; } ); - } - - public bool RequestFreeAppLicense( uint appId ) - { - bool success = false; - bool completed = false; - Action cbMethod = ( resultInfo ) => - { - completed = true; - success = resultInfo.GrantedApps.Contains( appId ); - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.RequestFreeLicense( appId ), cbMethod ); - }, () => { return completed; } ); - - return success; - } - - public void RequestAppTicket( uint appId ) - { - if ( AppTickets.ContainsKey( appId ) || bAborted ) - return; - - - if ( !authenticatedUser ) - { - AppTickets[ appId ] = null; - return; - } - - bool completed = false; - Action cbMethod = ( appTicket ) => - { - completed = true; - - if ( appTicket.Result != EResult.OK ) - { - Console.WriteLine( "Unable to get appticket for {0}: {1}", appTicket.AppID, appTicket.Result ); - Abort(); - } - else - { - Console.WriteLine( "Got appticket for {0}!", appTicket.AppID ); - AppTickets[ appTicket.AppID ] = appTicket.Ticket; - } - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.GetAppOwnershipTicket( appId ), cbMethod ); - }, () => { return completed; } ); - } - - public void RequestDepotKey( uint depotId, uint appid = 0 ) - { - if ( DepotKeys.ContainsKey( depotId ) || bAborted ) - return; - - bool completed = false; - - Action cbMethod = ( depotKey ) => - { - completed = true; - Console.WriteLine( "Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result ); - - if ( depotKey.Result != EResult.OK ) - { - Abort(); - return; - } - - DepotKeys[ depotKey.DepotID ] = depotKey.DepotKey; - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.GetDepotDecryptionKey( depotId, appid ), cbMethod ); - }, () => { return completed; } ); - } - - public string ResolveCDNTopLevelHost(string host) - { - // SteamPipe CDN shares tokens with all hosts - if (host.EndsWith( ".steampipe.steamcontent.com" ) ) - { - return "steampipe.steamcontent.com"; - } - else if (host.EndsWith(".steamcontent.com")) - { - return "steamcontent.com"; - } - - return host; - } - - public void RequestCDNAuthToken( uint appid, uint depotid, string host, string cdnKey ) - { - if ( CDNAuthTokens.ContainsKey( cdnKey ) || bAborted ) - return; - - if ( !CDNAuthTokens.TryAdd( cdnKey, new TaskCompletionSource() ) ) - return; - - bool completed = false; - var timeoutDate = DateTime.Now.AddSeconds( 10 ); - Action cbMethod = ( cdnAuth ) => - { - completed = true; - Console.WriteLine( "Got CDN auth token for {0} result: {1} (expires {2})", host, cdnAuth.Result, cdnAuth.Expiration ); - - if ( cdnAuth.Result != EResult.OK ) - { - Abort(); - return; - } - - CDNAuthTokens[cdnKey].TrySetResult( cdnAuth ); - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.GetCDNAuthToken( appid, depotid, host ), cbMethod ); - }, () => { return completed || DateTime.Now >= timeoutDate; } ); - } - - public void CheckAppBetaPassword( uint appid, string password ) - { - bool completed = false; - Action cbMethod = ( appPassword ) => - { - completed = true; - - Console.WriteLine( "Retrieved {0} beta keys with result: {1}", appPassword.BetaPasswords.Count, appPassword.Result ); - - foreach ( var entry in appPassword.BetaPasswords ) - { - AppBetaPasswords[ entry.Key ] = entry.Value; - } - }; - - WaitUntilCallback( () => - { - callbacks.Subscribe( steamApps.CheckAppBetaPassword( appid, password ), cbMethod ); - }, () => { return completed; } ); - } - - public CPublishedFile_GetItemInfo_Response.WorkshopItemInfo GetPubfileItemInfo( uint appId, PublishedFileID pubFile ) - { - var pubFileRequest = new CPublishedFile_GetItemInfo_Request() { app_id = appId }; - pubFileRequest.workshop_items.Add( new CPublishedFile_GetItemInfo_Request.WorkshopItem() { published_file_id = pubFile } ); - - bool completed = false; - CPublishedFile_GetItemInfo_Response.WorkshopItemInfo details = null; - - Action cbMethod = callback => - { - completed = true; - if ( callback.Result == EResult.OK ) - { - var response = callback.GetDeserializedResponse(); - details = response.workshop_items.FirstOrDefault(); - } - else - { - throw new Exception( $"EResult {(int)callback.Result} ({callback.Result}) while retrieving UGC id for pubfile {pubFile}."); - } - }; - - WaitUntilCallback(() => - { - callbacks.Subscribe( steamPublishedFile.SendMessage( api => api.GetItemInfo( pubFileRequest ) ), cbMethod ); - }, () => { return completed; }); - - return details; - } - - void Connect() - { - bAborted = false; - bConnected = false; - bConnecting = true; - connectionBackoff = 0; - bExpectingDisconnectRemote = false; - bDidDisconnect = false; - bDidReceiveLoginKey = false; - this.connectTime = DateTime.Now; - this.steamClient.Connect(); - } - - private void Abort( bool sendLogOff = true ) - { - Disconnect( sendLogOff ); - } - public void Disconnect( bool sendLogOff = true ) - { - if ( sendLogOff ) - { - steamUser.LogOff(); - } - - steamClient.Disconnect(); - bConnected = false; - bConnecting = false; - bAborted = true; - - // flush callbacks until our disconnected event - while ( !bDidDisconnect ) - { - callbacks.RunWaitAllCallbacks( TimeSpan.FromMilliseconds( 100 ) ); - } - } - - public void TryWaitForLoginKey() - { - if ( logonDetails.Username == null || !ContentDownloader.Config.RememberPassword ) return; - - var totalWaitPeriod = DateTime.Now.AddSeconds( 3 ); - - while ( true ) - { - DateTime now = DateTime.Now; - if ( now >= totalWaitPeriod ) break; - - if ( bDidReceiveLoginKey ) break; - - callbacks.RunWaitAllCallbacks( TimeSpan.FromMilliseconds( 100 ) ); - } - } - - private void WaitForCallbacks() - { - callbacks.RunWaitCallbacks( TimeSpan.FromSeconds( 1 ) ); - - TimeSpan diff = DateTime.Now - connectTime; - - if ( diff > STEAM3_TIMEOUT && !bConnected ) - { - Console.WriteLine( "Timeout connecting to Steam3." ); - Abort(); - - return; - } - } - - private void ConnectedCallback( SteamClient.ConnectedCallback connected ) - { - Console.WriteLine( " Done!" ); - bConnecting = false; - bConnected = true; - if ( !authenticatedUser ) - { - Console.Write( "Logging anonymously into Steam3..." ); - steamUser.LogOnAnonymous(); - } - else - { - Console.Write( "Logging '{0}' into Steam3...", logonDetails.Username ); - steamUser.LogOn( logonDetails ); - } - } - - private void DisconnectedCallback( SteamClient.DisconnectedCallback disconnected ) - { - bDidDisconnect = true; - - if ( disconnected.UserInitiated || bExpectingDisconnectRemote ) - { - Console.WriteLine( "Disconnected from Steam" ); - } - else if ( connectionBackoff >= 10 ) - { - Console.WriteLine( "Could not connect to Steam after 10 tries" ); - Abort( false ); - } - else if ( !bAborted ) - { - if ( bConnecting ) - { - Console.WriteLine( "Connection to Steam failed. Trying again" ); - } - else - { - Console.WriteLine( "Lost connection to Steam. Reconnecting" ); - } - - Thread.Sleep( 1000 * ++connectionBackoff ); - steamClient.Connect(); - } - } - - private void LogOnCallback( SteamUser.LoggedOnCallback loggedOn ) - { - bool isSteamGuard = loggedOn.Result == EResult.AccountLogonDenied; - bool is2FA = loggedOn.Result == EResult.AccountLoginDeniedNeedTwoFactor; - bool isLoginKey = ContentDownloader.Config.RememberPassword && logonDetails.LoginKey != null && loggedOn.Result == EResult.InvalidPassword; - - if ( isSteamGuard || is2FA || isLoginKey ) - { - bExpectingDisconnectRemote = true; - Abort( false ); - - if ( !isLoginKey ) - { - Console.WriteLine( "This account is protected by Steam Guard." ); - } - - if ( is2FA ) - { - Console.Write( "Please enter your 2 factor auth code from your authenticator app: " ); - logonDetails.TwoFactorCode = Console.ReadLine(); - } - else if ( isLoginKey ) - { - AccountSettingsStore.Instance.LoginKeys.Remove( logonDetails.Username ); - AccountSettingsStore.Save(); - - logonDetails.LoginKey = null; - - if ( ContentDownloader.Config.SuppliedPassword != null ) - { - Console.WriteLine( "Login key was expired. Connecting with supplied password." ); - logonDetails.Password = ContentDownloader.Config.SuppliedPassword; - } - else - { - Console.WriteLine( "Login key was expired. Please enter your password: " ); - logonDetails.Password = Util.ReadPassword(); - } - } - else - { - Console.Write( "Please enter the authentication code sent to your email address: " ); - logonDetails.AuthCode = Console.ReadLine(); - } - - Console.Write( "Retrying Steam3 connection..." ); - Connect(); - - return; - } - else if ( loggedOn.Result == EResult.ServiceUnavailable ) - { - Console.WriteLine( "Unable to login to Steam3: {0}", loggedOn.Result ); - Abort( false ); - - return; - } - else if ( loggedOn.Result != EResult.OK ) - { - Console.WriteLine( "Unable to login to Steam3: {0}", loggedOn.Result ); - Abort(); - - return; - } - - Console.WriteLine( " Done!" ); - - this.seq++; - credentials.LoggedOn = true; - - if ( ContentDownloader.Config.CellID == 0 ) - { - Console.WriteLine( "Using Steam3 suggested CellID: " + loggedOn.CellID ); - ContentDownloader.Config.CellID = ( int )loggedOn.CellID; - } - } - - private void SessionTokenCallback( SteamUser.SessionTokenCallback sessionToken ) - { - Console.WriteLine( "Got session token!" ); - credentials.SessionToken = sessionToken.SessionToken; - } - - private void LicenseListCallback( SteamApps.LicenseListCallback licenseList ) - { - if ( licenseList.Result != EResult.OK ) - { - Console.WriteLine( "Unable to get license list: {0} ", licenseList.Result ); - Abort(); - - return; - } - - Console.WriteLine( "Got {0} licenses for account!", licenseList.LicenseList.Count ); - Licenses = licenseList.LicenseList; - - foreach ( var license in licenseList.LicenseList ) - { - if ( license.AccessToken > 0 ) - { - PackageTokens.TryAdd( license.PackageID, license.AccessToken ); - } - } - } - - private void UpdateMachineAuthCallback( SteamUser.UpdateMachineAuthCallback machineAuth ) - { - byte[] hash = Util.SHAHash( machineAuth.Data ); - Console.WriteLine( "Got Machine Auth: {0} {1} {2} {3}", machineAuth.FileName, machineAuth.Offset, machineAuth.BytesToWrite, machineAuth.Data.Length, hash ); - - AccountSettingsStore.Instance.SentryData[ logonDetails.Username ] = machineAuth.Data; - AccountSettingsStore.Save(); - - var authResponse = new SteamUser.MachineAuthDetails - { - BytesWritten = machineAuth.BytesToWrite, - FileName = machineAuth.FileName, - FileSize = machineAuth.BytesToWrite, - Offset = machineAuth.Offset, - - SentryFileHash = hash, // should be the sha1 hash of the sentry file we just wrote - - OneTimePassword = machineAuth.OneTimePassword, // not sure on this one yet, since we've had no examples of steam using OTPs - - LastError = 0, // result from win32 GetLastError - Result = EResult.OK, // if everything went okay, otherwise ~who knows~ - - JobID = machineAuth.JobID, // so we respond to the correct server job - }; - - // send off our response - steamUser.SendMachineAuthResponse( authResponse ); - } - - private void LoginKeyCallback( SteamUser.LoginKeyCallback loginKey ) - { - Console.WriteLine( "Accepted new login key for account {0}", logonDetails.Username ); - - AccountSettingsStore.Instance.LoginKeys[ logonDetails.Username ] = loginKey.LoginKey; - AccountSettingsStore.Save(); - - steamUser.AcceptNewLoginKey( loginKey ); - - bDidReceiveLoginKey = true; - } - - - } -} +using SteamKit2; +using SteamKit2.Internal; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace DepotDownloader.Core +{ + + public class Steam3Session + { + public class Credentials + { + public bool LoggedOn { get; set; } + public ulong SessionToken { get; set; } + + public bool IsValid + { + get { return LoggedOn; } + } + } + + public ReadOnlyCollection Licenses + { + get; + private set; + } + + public Dictionary AppTickets { get; private set; } + public Dictionary AppTokens { get; private set; } + public Dictionary PackageTokens { get; private set; } + public Dictionary DepotKeys { get; private set; } + public ConcurrentDictionary> CDNAuthTokens { get; private set; } + public Dictionary AppInfo { get; private set; } + public Dictionary PackageInfo { get; private set; } + public Dictionary AppBetaPasswords { get; private set; } + + public SteamClient steamClient; + public SteamUser steamUser; + SteamApps steamApps; + SteamUnifiedMessages.UnifiedService steamPublishedFile; + + CallbackManager callbacks; + + bool authenticatedUser; + bool bConnected; + bool bConnecting; + bool bAborted; + bool bExpectingDisconnectRemote; + bool bDidDisconnect; + bool bDidReceiveLoginKey; + int connectionBackoff; + int seq; // more hack fixes + DateTime connectTime; + + // input + SteamUser.LogOnDetails logonDetails; + + // output + Credentials credentials; + + static readonly TimeSpan STEAM3_TIMEOUT = TimeSpan.FromSeconds( 30 ); + + + public Steam3Session( SteamUser.LogOnDetails details ) + { + this.logonDetails = details; + + this.authenticatedUser = details.Username != null; + this.credentials = new Credentials(); + this.bConnected = false; + this.bConnecting = false; + this.bAborted = false; + this.bExpectingDisconnectRemote = false; + this.bDidDisconnect = false; + this.bDidReceiveLoginKey = false; + this.seq = 0; + + this.AppTickets = new Dictionary(); + this.AppTokens = new Dictionary(); + this.PackageTokens = new Dictionary(); + this.DepotKeys = new Dictionary(); + this.CDNAuthTokens = new ConcurrentDictionary>(); + this.AppInfo = new Dictionary(); + this.PackageInfo = new Dictionary(); + this.AppBetaPasswords = new Dictionary(); + + this.steamClient = new SteamClient(); + + this.steamUser = this.steamClient.GetHandler(); + this.steamApps = this.steamClient.GetHandler(); + var steamUnifiedMessages = this.steamClient.GetHandler(); + this.steamPublishedFile = steamUnifiedMessages.CreateService(); + + this.callbacks = new CallbackManager( this.steamClient ); + + this.callbacks.Subscribe( ConnectedCallback ); + this.callbacks.Subscribe( DisconnectedCallback ); + this.callbacks.Subscribe( LogOnCallback ); + this.callbacks.Subscribe( SessionTokenCallback ); + this.callbacks.Subscribe( LicenseListCallback ); + this.callbacks.Subscribe( UpdateMachineAuthCallback ); + this.callbacks.Subscribe( LoginKeyCallback ); + + Console.Write( "Connecting to Steam3..." ); + + if ( authenticatedUser ) + { + FileInfo fi = new FileInfo( String.Format( "{0}.sentryFile", logonDetails.Username ) ); + if ( AccountSettingsStore.Instance.SentryData != null && AccountSettingsStore.Instance.SentryData.ContainsKey( logonDetails.Username ) ) + { + logonDetails.SentryFileHash = Util.SHAHash( AccountSettingsStore.Instance.SentryData[ logonDetails.Username ] ); + } + else if ( fi.Exists && fi.Length > 0 ) + { + var sentryData = File.ReadAllBytes( fi.FullName ); + logonDetails.SentryFileHash = Util.SHAHash( sentryData ); + AccountSettingsStore.Instance.SentryData[ logonDetails.Username ] = sentryData; + AccountSettingsStore.Save(); + } + } + + Connect(); + } + + public delegate bool WaitCondition(); + public bool WaitUntilCallback( Action submitter, WaitCondition waiter ) + { + while ( !bAborted && !waiter() ) + { + submitter(); + + int seq = this.seq; + do + { + WaitForCallbacks(); + } + while ( !bAborted && this.seq == seq && !waiter() ); + } + + return bAborted; + } + + public Credentials WaitForCredentials() + { + if ( credentials.IsValid || bAborted ) + return credentials; + + WaitUntilCallback( () => { }, () => { return credentials.IsValid; } ); + + return credentials; + } + + public void RequestAppInfo( uint appId, bool bForce = false ) + { + if ( ( AppInfo.ContainsKey( appId ) && !bForce ) || bAborted ) + return; + + bool completed = false; + Action cbMethodTokens = ( appTokens ) => + { + completed = true; + if ( appTokens.AppTokensDenied.Contains( appId ) ) + { + Console.WriteLine( "Insufficient privileges to get access token for app {0}", appId ); + } + + foreach ( var token_dict in appTokens.AppTokens ) + { + this.AppTokens[ token_dict.Key ] = token_dict.Value; + } + }; + + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.PICSGetAccessTokens( new List() { appId }, new List() { } ), cbMethodTokens ); + }, () => { return completed; } ); + + completed = false; + Action cbMethod = ( appInfo ) => + { + completed = !appInfo.ResponsePending; + + foreach ( var app_value in appInfo.Apps ) + { + var app = app_value.Value; + + Console.WriteLine( "Got AppInfo for {0}", app.ID ); + AppInfo[ app.ID ] = app; + } + + foreach ( var app in appInfo.UnknownApps ) + { + AppInfo[ app ] = null; + } + }; + + SteamApps.PICSRequest request = new SteamApps.PICSRequest( appId ); + if ( AppTokens.ContainsKey( appId ) ) + { + request.AccessToken = AppTokens[ appId ]; + request.Public = false; + } + + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.PICSGetProductInfo( new List() { request }, new List() { } ), cbMethod ); + }, () => { return completed; } ); + } + + public void RequestPackageInfo( IEnumerable packageIds ) + { + List packages = packageIds.ToList(); + packages.RemoveAll( pid => PackageInfo.ContainsKey( pid ) ); + + if ( packages.Count == 0 || bAborted ) + return; + + bool completed = false; + Action cbMethod = ( packageInfo ) => + { + completed = !packageInfo.ResponsePending; + + foreach ( var package_value in packageInfo.Packages ) + { + var package = package_value.Value; + PackageInfo[ package.ID ] = package; + } + + foreach ( var package in packageInfo.UnknownPackages ) + { + PackageInfo[package] = null; + } + }; + + var packageRequests = new List(); + + foreach ( var package in packages ) + { + var request = new SteamApps.PICSRequest( package ); + + if ( PackageTokens.TryGetValue( package, out var token ) ) + { + request.AccessToken = token; + request.Public = false; + } + + packageRequests.Add( request ); + } + + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.PICSGetProductInfo( new List(), packageRequests ), cbMethod ); + }, () => { return completed; } ); + } + + public bool RequestFreeAppLicense( uint appId ) + { + bool success = false; + bool completed = false; + Action cbMethod = ( resultInfo ) => + { + completed = true; + success = resultInfo.GrantedApps.Contains( appId ); + }; + + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.RequestFreeLicense( appId ), cbMethod ); + }, () => { return completed; } ); + + return success; + } + + public void RequestAppTicket( uint appId ) + { + if ( AppTickets.ContainsKey( appId ) || bAborted ) + return; + + + if ( !authenticatedUser ) + { + AppTickets[ appId ] = null; + return; + } + + bool completed = false; + Action cbMethod = ( appTicket ) => + { + completed = true; + + if ( appTicket.Result != EResult.OK ) + { + Console.WriteLine( "Unable to get appticket for {0}: {1}", appTicket.AppID, appTicket.Result ); + Abort(); + } + else + { + Console.WriteLine( "Got appticket for {0}!", appTicket.AppID ); + AppTickets[ appTicket.AppID ] = appTicket.Ticket; + } + }; + + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.GetAppOwnershipTicket( appId ), cbMethod ); + }, () => { return completed; } ); + } + + public void RequestDepotKey( uint depotId, uint appid = 0 ) + { + if ( DepotKeys.ContainsKey( depotId ) || bAborted ) + return; + + bool completed = false; + + Action cbMethod = ( depotKey ) => + { + completed = true; + Console.WriteLine( "Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result ); + + if ( depotKey.Result != EResult.OK ) + { + Abort(); + return; + } + + DepotKeys[ depotKey.DepotID ] = depotKey.DepotKey; + }; + + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.GetDepotDecryptionKey( depotId, appid ), cbMethod ); + }, () => { return completed; } ); + } + + public string ResolveCDNTopLevelHost(string host) + { + // SteamPipe CDN shares tokens with all hosts + if (host.EndsWith( ".steampipe.steamcontent.com" ) ) + { + return "steampipe.steamcontent.com"; + } + else if (host.EndsWith(".steamcontent.com")) + { + return "steamcontent.com"; + } + + return host; + } + + public void RequestCDNAuthToken( uint appid, uint depotid, string host, string cdnKey ) + { + if ( CDNAuthTokens.ContainsKey( cdnKey ) || bAborted ) + return; + + if ( !CDNAuthTokens.TryAdd( cdnKey, new TaskCompletionSource() ) ) + return; + + bool completed = false; + var timeoutDate = DateTime.Now.AddSeconds( 10 ); + Action cbMethod = ( cdnAuth ) => + { + completed = true; + Console.WriteLine( "Got CDN auth token for {0} result: {1} (expires {2})", host, cdnAuth.Result, cdnAuth.Expiration ); + + if ( cdnAuth.Result != EResult.OK ) + { + Abort(); + return; + } + + CDNAuthTokens[cdnKey].TrySetResult( cdnAuth ); + }; + + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.GetCDNAuthToken( appid, depotid, host ), cbMethod ); + }, () => { return completed || DateTime.Now >= timeoutDate; } ); + } + + public void CheckAppBetaPassword( uint appid, string password ) + { + bool completed = false; + Action cbMethod = ( appPassword ) => + { + completed = true; + + Console.WriteLine( "Retrieved {0} beta keys with result: {1}", appPassword.BetaPasswords.Count, appPassword.Result ); + + foreach ( var entry in appPassword.BetaPasswords ) + { + AppBetaPasswords[ entry.Key ] = entry.Value; + } + }; + + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.CheckAppBetaPassword( appid, password ), cbMethod ); + }, () => { return completed; } ); + } + + public CPublishedFile_GetItemInfo_Response.WorkshopItemInfo GetPubfileItemInfo( uint appId, PublishedFileID pubFile ) + { + var pubFileRequest = new CPublishedFile_GetItemInfo_Request() { app_id = appId }; + pubFileRequest.workshop_items.Add( new CPublishedFile_GetItemInfo_Request.WorkshopItem() { published_file_id = pubFile } ); + + bool completed = false; + CPublishedFile_GetItemInfo_Response.WorkshopItemInfo details = null; + + Action cbMethod = callback => + { + completed = true; + if ( callback.Result == EResult.OK ) + { + var response = callback.GetDeserializedResponse(); + details = response.workshop_items.FirstOrDefault(); + } + else + { + throw new Exception( $"EResult {(int)callback.Result} ({callback.Result}) while retrieving UGC id for pubfile {pubFile}."); + } + }; + + WaitUntilCallback(() => + { + callbacks.Subscribe( steamPublishedFile.SendMessage( api => api.GetItemInfo( pubFileRequest ) ), cbMethod ); + }, () => { return completed; }); + + return details; + } + + void Connect() + { + bAborted = false; + bConnected = false; + bConnecting = true; + connectionBackoff = 0; + bExpectingDisconnectRemote = false; + bDidDisconnect = false; + bDidReceiveLoginKey = false; + this.connectTime = DateTime.Now; + this.steamClient.Connect(); + } + + private void Abort( bool sendLogOff = true ) + { + Disconnect( sendLogOff ); + } + public void Disconnect( bool sendLogOff = true ) + { + if ( sendLogOff ) + { + steamUser.LogOff(); + } + + steamClient.Disconnect(); + bConnected = false; + bConnecting = false; + bAborted = true; + + // flush callbacks until our disconnected event + while ( !bDidDisconnect ) + { + callbacks.RunWaitAllCallbacks( TimeSpan.FromMilliseconds( 100 ) ); + } + } + + public void TryWaitForLoginKey() + { + if ( logonDetails.Username == null || !ContentDownloader.Config.RememberPassword ) return; + + var totalWaitPeriod = DateTime.Now.AddSeconds( 3 ); + + while ( true ) + { + DateTime now = DateTime.Now; + if ( now >= totalWaitPeriod ) break; + + if ( bDidReceiveLoginKey ) break; + + callbacks.RunWaitAllCallbacks( TimeSpan.FromMilliseconds( 100 ) ); + } + } + + private void WaitForCallbacks() + { + callbacks.RunWaitCallbacks( TimeSpan.FromSeconds( 1 ) ); + + TimeSpan diff = DateTime.Now - connectTime; + + if ( diff > STEAM3_TIMEOUT && !bConnected ) + { + Console.WriteLine( "Timeout connecting to Steam3." ); + Abort(); + + return; + } + } + + private void ConnectedCallback( SteamClient.ConnectedCallback connected ) + { + Console.WriteLine( " Done!" ); + bConnecting = false; + bConnected = true; + if ( !authenticatedUser ) + { + Console.Write( "Logging anonymously into Steam3..." ); + steamUser.LogOnAnonymous(); + } + else + { + Console.Write( "Logging '{0}' into Steam3...", logonDetails.Username ); + steamUser.LogOn( logonDetails ); + } + } + + private void DisconnectedCallback( SteamClient.DisconnectedCallback disconnected ) + { + bDidDisconnect = true; + + if ( disconnected.UserInitiated || bExpectingDisconnectRemote ) + { + Console.WriteLine( "Disconnected from Steam" ); + } + else if ( connectionBackoff >= 10 ) + { + Console.WriteLine( "Could not connect to Steam after 10 tries" ); + Abort( false ); + } + else if ( !bAborted ) + { + if ( bConnecting ) + { + Console.WriteLine( "Connection to Steam failed. Trying again" ); + } + else + { + Console.WriteLine( "Lost connection to Steam. Reconnecting" ); + } + + Thread.Sleep( 1000 * ++connectionBackoff ); + steamClient.Connect(); + } + } + + private void LogOnCallback( SteamUser.LoggedOnCallback loggedOn ) + { + bool isSteamGuard = loggedOn.Result == EResult.AccountLogonDenied; + bool is2FA = loggedOn.Result == EResult.AccountLoginDeniedNeedTwoFactor; + bool isLoginKey = ContentDownloader.Config.RememberPassword && logonDetails.LoginKey != null && loggedOn.Result == EResult.InvalidPassword; + + if ( isSteamGuard || is2FA || isLoginKey ) + { + bExpectingDisconnectRemote = true; + Abort( false ); + + if ( !isLoginKey ) + { + Console.WriteLine( "This account is protected by Steam Guard." ); + } + + if ( is2FA ) + { + Console.Write( "Please enter your 2 factor auth code from your authenticator app: " ); + logonDetails.TwoFactorCode = Console.ReadLine(); + } + else if ( isLoginKey ) + { + AccountSettingsStore.Instance.LoginKeys.Remove( logonDetails.Username ); + AccountSettingsStore.Save(); + + logonDetails.LoginKey = null; + + if ( ContentDownloader.Config.SuppliedPassword != null ) + { + Console.WriteLine( "Login key was expired. Connecting with supplied password." ); + logonDetails.Password = ContentDownloader.Config.SuppliedPassword; + } + else + { + Console.WriteLine( "Login key was expired. Please enter your password: " ); + logonDetails.Password = Util.ReadPassword(); + } + } + else + { + Console.Write( "Please enter the authentication code sent to your email address: " ); + logonDetails.AuthCode = Console.ReadLine(); + } + + Console.Write( "Retrying Steam3 connection..." ); + Connect(); + + return; + } + else if ( loggedOn.Result == EResult.ServiceUnavailable ) + { + Console.WriteLine( "Unable to login to Steam3: {0}", loggedOn.Result ); + Abort( false ); + + return; + } + else if ( loggedOn.Result != EResult.OK ) + { + Console.WriteLine( "Unable to login to Steam3: {0}", loggedOn.Result ); + Abort(); + + return; + } + + Console.WriteLine( " Done!" ); + + this.seq++; + credentials.LoggedOn = true; + + if ( ContentDownloader.Config.CellID == 0 ) + { + Console.WriteLine( "Using Steam3 suggested CellID: " + loggedOn.CellID ); + ContentDownloader.Config.CellID = ( int )loggedOn.CellID; + } + } + + private void SessionTokenCallback( SteamUser.SessionTokenCallback sessionToken ) + { + Console.WriteLine( "Got session token!" ); + credentials.SessionToken = sessionToken.SessionToken; + } + + private void LicenseListCallback( SteamApps.LicenseListCallback licenseList ) + { + if ( licenseList.Result != EResult.OK ) + { + Console.WriteLine( "Unable to get license list: {0} ", licenseList.Result ); + Abort(); + + return; + } + + Console.WriteLine( "Got {0} licenses for account!", licenseList.LicenseList.Count ); + Licenses = licenseList.LicenseList; + + foreach ( var license in licenseList.LicenseList ) + { + if ( license.AccessToken > 0 ) + { + PackageTokens.Add(license.PackageID, license.AccessToken); + } + } + } + + private void UpdateMachineAuthCallback( SteamUser.UpdateMachineAuthCallback machineAuth ) + { + byte[] hash = Util.SHAHash( machineAuth.Data ); + Console.WriteLine( "Got Machine Auth: {0} {1} {2} {3}", machineAuth.FileName, machineAuth.Offset, machineAuth.BytesToWrite, machineAuth.Data.Length, hash ); + + AccountSettingsStore.Instance.SentryData[ logonDetails.Username ] = machineAuth.Data; + AccountSettingsStore.Save(); + + var authResponse = new SteamUser.MachineAuthDetails + { + BytesWritten = machineAuth.BytesToWrite, + FileName = machineAuth.FileName, + FileSize = machineAuth.BytesToWrite, + Offset = machineAuth.Offset, + + SentryFileHash = hash, // should be the sha1 hash of the sentry file we just wrote + + OneTimePassword = machineAuth.OneTimePassword, // not sure on this one yet, since we've had no examples of steam using OTPs + + LastError = 0, // result from win32 GetLastError + Result = EResult.OK, // if everything went okay, otherwise ~who knows~ + + JobID = machineAuth.JobID, // so we respond to the correct server job + }; + + // send off our response + steamUser.SendMachineAuthResponse( authResponse ); + } + + private void LoginKeyCallback( SteamUser.LoginKeyCallback loginKey ) + { + Console.WriteLine( "Accepted new login key for account {0}", logonDetails.Username ); + + AccountSettingsStore.Instance.LoginKeys[ logonDetails.Username ] = loginKey.LoginKey; + AccountSettingsStore.Save(); + + steamUser.AcceptNewLoginKey( loginKey ); + + bDidReceiveLoginKey = true; + } + + + } +} diff --git a/DepotDownloader/Util.cs b/DepotDownloader.Core/Util.cs similarity index 95% rename from DepotDownloader/Util.cs rename to DepotDownloader.Core/Util.cs index 1c08138b..709279e2 100644 --- a/DepotDownloader/Util.cs +++ b/DepotDownloader.Core/Util.cs @@ -1,136 +1,136 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using System.Security.Cryptography; -using System.Text; - -namespace DepotDownloader -{ - static class Util - { - public static string GetSteamOS() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - return "windows"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - return "macos"; - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - return "linux"; - } - - return "unknown"; - } - - public static string GetSteamArch() - { - return Environment.Is64BitOperatingSystem ? "64" : "32"; - } - - public static string ReadPassword() - { - ConsoleKeyInfo keyInfo; - StringBuilder password = new StringBuilder(); - - do - { - keyInfo = Console.ReadKey( true ); - - if ( keyInfo.Key == ConsoleKey.Backspace ) - { - if ( password.Length > 0 ) - password.Remove( password.Length - 1, 1 ); - continue; - } - - /* Printable ASCII characters only */ - char c = keyInfo.KeyChar; - if ( c >= ' ' && c <= '~' ) - password.Append( c ); - } while ( keyInfo.Key != ConsoleKey.Enter ); - - return password.ToString(); - } - - // Validate a file against Steam3 Chunk data - public static List ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata) - { - var neededChunks = new List(); - int read; - - foreach (var data in chunkdata) - { - byte[] chunk = new byte[data.UncompressedLength]; - fs.Seek((long)data.Offset, SeekOrigin.Begin); - read = fs.Read(chunk, 0, (int)data.UncompressedLength); - - byte[] tempchunk; - if (read < data.UncompressedLength) - { - tempchunk = new byte[read]; - Array.Copy(chunk, 0, tempchunk, 0, read); - } - else - { - tempchunk = chunk; - } - - byte[] adler = AdlerHash(tempchunk); - if (!adler.SequenceEqual(data.Checksum)) - { - neededChunks.Add(data); - } - } - - return neededChunks; - } - - public static byte[] AdlerHash(byte[] input) - { - uint a = 0, b = 0; - for (int i = 0; i < input.Length; i++) - { - a = (a + input[i]) % 65521; - b = (b + a) % 65521; - } - return BitConverter.GetBytes(a | (b << 16)); - } - - public static byte[] SHAHash( byte[] input ) - { - using (var sha = SHA1.Create()) - { - var output = sha.ComputeHash( input ); - - return output; - } - } - - public static byte[] DecodeHexString( string hex ) - { - if ( hex == null ) - return null; - - int chars = hex.Length; - byte[] bytes = new byte[ chars / 2 ]; - - for ( int i = 0 ; i < chars ; i += 2 ) - bytes[ i / 2 ] = Convert.ToByte( hex.Substring( i, 2 ), 16 ); - - return bytes; - } - - public static string EncodeHexString( byte[] input ) - { - return input.Aggregate( new StringBuilder(), - ( sb, v ) => sb.Append( v.ToString( "x2" ) ) - ).ToString(); - } - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; + +namespace DepotDownloader.Core +{ + public static class Util + { + public static string GetSteamOS() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return "windows"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return "macos"; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return "linux"; + } + + return "unknown"; + } + + public static string GetSteamArch() + { + return Environment.Is64BitOperatingSystem ? "64" : "32"; + } + + public static string ReadPassword() + { + ConsoleKeyInfo keyInfo; + StringBuilder password = new StringBuilder(); + + do + { + keyInfo = Console.ReadKey( true ); + + if ( keyInfo.Key == ConsoleKey.Backspace ) + { + if ( password.Length > 0 ) + password.Remove( password.Length - 1, 1 ); + continue; + } + + /* Printable ASCII characters only */ + char c = keyInfo.KeyChar; + if ( c >= ' ' && c <= '~' ) + password.Append( c ); + } while ( keyInfo.Key != ConsoleKey.Enter ); + + return password.ToString(); + } + + // Validate a file against Steam3 Chunk data + public static List ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata) + { + var neededChunks = new List(); + int read; + + foreach (var data in chunkdata) + { + byte[] chunk = new byte[data.UncompressedLength]; + fs.Seek((long)data.Offset, SeekOrigin.Begin); + read = fs.Read(chunk, 0, (int)data.UncompressedLength); + + byte[] tempchunk; + if (read < data.UncompressedLength) + { + tempchunk = new byte[read]; + Array.Copy(chunk, 0, tempchunk, 0, read); + } + else + { + tempchunk = chunk; + } + + byte[] adler = AdlerHash(tempchunk); + if (!adler.SequenceEqual(data.Checksum)) + { + neededChunks.Add(data); + } + } + + return neededChunks; + } + + public static byte[] AdlerHash(byte[] input) + { + uint a = 0, b = 0; + for (int i = 0; i < input.Length; i++) + { + a = (a + input[i]) % 65521; + b = (b + a) % 65521; + } + return BitConverter.GetBytes(a | (b << 16)); + } + + public static byte[] SHAHash( byte[] input ) + { + using (var sha = SHA1.Create()) + { + var output = sha.ComputeHash( input ); + + return output; + } + } + + public static byte[] DecodeHexString( string hex ) + { + if ( hex == null ) + return null; + + int chars = hex.Length; + byte[] bytes = new byte[ chars / 2 ]; + + for ( int i = 0 ; i < chars ; i += 2 ) + bytes[ i / 2 ] = Convert.ToByte( hex.Substring( i, 2 ), 16 ); + + return bytes; + } + + public static string EncodeHexString( byte[] input ) + { + return input.Aggregate( new StringBuilder(), + ( sb, v ) => sb.Append( v.ToString( "x2" ) ) + ).ToString(); + } + } +} diff --git a/DepotDownloader.sln b/DepotDownloader.sln index 631e75ec..81676bda 100644 --- a/DepotDownloader.sln +++ b/DepotDownloader.sln @@ -1,9 +1,11 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26228.4 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30517.126 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DepotDownloader", "DepotDownloader\DepotDownloader.csproj", "{39159C47-ACD3-449F-96CA-4F30C8ED147A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DepotDownloader", "DepotDownloader\DepotDownloader.csproj", "{39159C47-ACD3-449F-96CA-4F30C8ED147A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DepotDownloader.Core", "DepotDownloader.Core\DepotDownloader.Core.csproj", "{8303B6A1-CE40-4C26-9376-DB08290645F7}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -15,8 +17,15 @@ Global {39159C47-ACD3-449F-96CA-4F30C8ED147A}.Debug|Any CPU.Build.0 = Debug|Any CPU {39159C47-ACD3-449F-96CA-4F30C8ED147A}.Release|Any CPU.ActiveCfg = Release|Any CPU {39159C47-ACD3-449F-96CA-4F30C8ED147A}.Release|Any CPU.Build.0 = Release|Any CPU + {8303B6A1-CE40-4C26-9376-DB08290645F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8303B6A1-CE40-4C26-9376-DB08290645F7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8303B6A1-CE40-4C26-9376-DB08290645F7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8303B6A1-CE40-4C26-9376-DB08290645F7}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {B9D19618-4471-4B6A-999B-3884575BE066} + EndGlobalSection EndGlobal diff --git a/DepotDownloader/DepotDownloader.csproj b/DepotDownloader/DepotDownloader.csproj index ac5820f3..dc1affc1 100644 --- a/DepotDownloader/DepotDownloader.csproj +++ b/DepotDownloader/DepotDownloader.csproj @@ -1,11 +1,10 @@  Exe - netcoreapp2.0 + netcoreapp3.1 false - - + \ No newline at end of file diff --git a/DepotDownloader/Program.cs b/DepotDownloader/Program.cs index b3b9b737..38411fd7 100644 --- a/DepotDownloader/Program.cs +++ b/DepotDownloader/Program.cs @@ -6,6 +6,8 @@ using SteamKit2; using System.ComponentModel; using System.Threading.Tasks; using System.Runtime.InteropServices; +using DepotDownloader.Core; + namespace DepotDownloader { class Program