diff --git a/DepotDownloader/ConfigStore.cs b/DepotDownloader/ConfigStore.cs index df580736..646c5ef7 100644 --- a/DepotDownloader/ConfigStore.cs +++ b/DepotDownloader/ConfigStore.cs @@ -18,6 +18,9 @@ namespace DepotDownloader [ProtoMember(4, IsRequired = false)] public System.Collections.Concurrent.ConcurrentDictionary ContentServerPenalty { get; private set; } + [ProtoMember(5, IsRequired = false)] + public Dictionary LoginKeys { get; private set; } + string FileName = null; ConfigStore() @@ -25,6 +28,7 @@ namespace DepotDownloader LastManifests = new Dictionary(); SentryData = new Dictionary(); ContentServerPenalty = new System.Collections.Concurrent.ConcurrentDictionary(); + LoginKeys = new Dictionary(); } static bool Loaded diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader/ContentDownloader.cs index d60ccca0..1cef730b 100644 --- a/DepotDownloader/ContentDownloader.cs +++ b/DepotDownloader/ContentDownloader.cs @@ -1,903 +1,912 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading; -using SteamKit2; -using System.Threading.Tasks; - -namespace DepotDownloader -{ - static class ContentDownloader - { - public const uint INVALID_APP_ID = uint.MaxValue; - public const uint INVALID_DEPOT_ID = uint.MaxValue; - public const ulong INVALID_MANIFEST_ID = ulong.MaxValue; - - public static DownloadConfig Config = new DownloadConfig(); - - private static Steam3Session steam3; - private static Steam3Session.Credentials steam3Credentials; - private static CDNClientPool cdnPool; - - private const string DEFAULT_DOWNLOAD_DIR = "depots"; - private const string CONFIG_DIR = ".DepotDownloader"; - private static readonly string STAGING_DIR = Path.Combine(CONFIG_DIR, "staging"); - - private sealed class DepotDownloadInfo - { - public uint id { get; private set; } - public string installDir { get; private set; } - public string contentName { get; private set; } - - public ulong manifestId { get; private set; } - public byte[] depotKey; - - public DepotDownloadInfo(uint depotid, ulong manifestId, string installDir, string contentName) - { - this.id = depotid; - this.manifestId = manifestId; - this.installDir = installDir; - this.contentName = contentName; - } - } - - static bool CreateDirectories( uint depotId, uint depotVersion, out string installDir ) - { - installDir = null; - try - { - if (string.IsNullOrWhiteSpace(ContentDownloader.Config.InstallDirectory)) - { - Directory.CreateDirectory( DEFAULT_DOWNLOAD_DIR ); - - string depotPath = Path.Combine( DEFAULT_DOWNLOAD_DIR, depotId.ToString() ); - Directory.CreateDirectory( depotPath ); - - installDir = Path.Combine(depotPath, depotVersion.ToString()); - Directory.CreateDirectory(installDir); - - Directory.CreateDirectory(Path.Combine(installDir, CONFIG_DIR)); - Directory.CreateDirectory(Path.Combine(installDir, STAGING_DIR)); - } - else - { - Directory.CreateDirectory(ContentDownloader.Config.InstallDirectory); - - installDir = ContentDownloader.Config.InstallDirectory; - - Directory.CreateDirectory(Path.Combine(installDir, CONFIG_DIR)); - Directory.CreateDirectory(Path.Combine(installDir, STAGING_DIR)); - } - } - catch - { - return false; - } - - return true; - } - - static bool TestIsFileIncluded(string filename) - { - if (!Config.UsingFileList) - return true; - - foreach (string fileListEntry in Config.FilesToDownload) - { - if (fileListEntry.Equals(filename, StringComparison.OrdinalIgnoreCase)) - return true; - } - - foreach (Regex rgx in Config.FilesToDownloadRegex) - { - Match m = rgx.Match(filename); - - if (m.Success) - return true; - } - - return false; - } - - static bool AccountHasAccess( uint depotId ) - { - if (steam3 == null || steam3.steamUser.SteamID == null || (steam3.Licenses == null && steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser)) - return false; - - IEnumerable licenseQuery; - if ( steam3.steamUser.SteamID.AccountType == EAccountType.AnonUser ) - { - licenseQuery = new List() { 17906 }; - } - else - { - licenseQuery = steam3.Licenses.Select( x => x.PackageID ); - } - - steam3.RequestPackageInfo( licenseQuery ); - - foreach ( var license in licenseQuery ) - { - SteamApps.PICSProductInfoCallback.PICSProductInfo package; - if ( steam3.PackageInfo.TryGetValue( license, out package ) && package != null ) - { - if ( package.KeyValues["appids"].Children.Any( child => child.AsInteger() == depotId ) ) - return true; - - if ( package.KeyValues["depotids"].Children.Any( child => child.AsInteger() == depotId ) ) - return true; - } - } - - return false; - } - - internal static KeyValue GetSteam3AppSection( uint appId, EAppInfoSection section ) - { - if (steam3 == null || steam3.AppInfo == null) - { - return null; - } - - SteamApps.PICSProductInfoCallback.PICSProductInfo app; - if ( !steam3.AppInfo.TryGetValue( appId, out app ) || app == null ) - { - return null; - } - - KeyValue appinfo = app.KeyValues; - string section_key; - - switch (section) - { - case EAppInfoSection.Common: - section_key = "common"; - break; - case EAppInfoSection.Extended: - section_key = "extended"; - break; - case EAppInfoSection.Config: - section_key = "config"; - break; - case EAppInfoSection.Depots: - section_key = "depots"; - break; - default: - throw new NotImplementedException(); - } - - KeyValue section_kv = appinfo.Children.Where(c => c.Name == section_key).FirstOrDefault(); - return section_kv; - } - - static uint GetSteam3AppBuildNumber(uint appId, string branch) - { - if (appId == INVALID_APP_ID) - return 0; - - - KeyValue depots = ContentDownloader.GetSteam3AppSection(appId, EAppInfoSection.Depots); - KeyValue branches = depots["branches"]; - KeyValue node = branches[branch]; - - if (node == KeyValue.Invalid) - return 0; - - KeyValue buildid = node["buildid"]; - - if (buildid == KeyValue.Invalid) - return 0; - - return uint.Parse(buildid.Value); - } - - static ulong GetSteam3DepotManifest(uint depotId, uint appId, string branch) - { - if (Config.ManifestId != INVALID_MANIFEST_ID) - return Config.ManifestId; - - KeyValue depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); - KeyValue depotChild = depots[depotId.ToString()]; - - if (depotChild == KeyValue.Invalid) - return INVALID_MANIFEST_ID; - - // Shared depots can either provide manifests, or leave you relying on their parent app. - // It seems that with the latter, "sharedinstall" will exist (and equals 2 in the one existance I know of). - // Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680. - if (depotChild["manifests"] == KeyValue.Invalid && depotChild["depotfromapp"] != KeyValue.Invalid) - { - uint otherAppId = (uint)depotChild["depotfromapp"].AsInteger(); - if (otherAppId == appId) - { - // This shouldn't ever happen, but ya never know with Valve. Don't infinite loop. - Console.WriteLine("App {0}, Depot {1} has depotfromapp of {2}!", - appId, depotId, otherAppId); - return INVALID_MANIFEST_ID; - } - - steam3.RequestAppInfo(otherAppId); - - return GetSteam3DepotManifest(depotId, otherAppId, branch); - } - - var manifests = depotChild["manifests"]; - var manifests_encrypted = depotChild["encryptedmanifests"]; - - if (manifests.Children.Count == 0 && manifests_encrypted.Children.Count == 0) - return INVALID_MANIFEST_ID; - - var node = manifests[branch]; - - if (branch != "Public" && node == KeyValue.Invalid) - { - var node_encrypted = manifests_encrypted[branch]; - if (node_encrypted != KeyValue.Invalid) - { - string password = Config.BetaPassword; - if (password == null) - { - Console.Write("Please enter the password for branch {0}: ", branch); - Config.BetaPassword = password = Console.ReadLine(); - } - - var encrypted_v1 = node_encrypted["encrypted_gid"]; - var encrypted_v2 = node_encrypted["encrypted_gid_2"]; - - if (encrypted_v1 != KeyValue.Invalid) - { - byte[] input = Util.DecodeHexString(encrypted_v1.Value); - byte[] manifest_bytes = CryptoHelper.VerifyAndDecryptPassword(input, password); - - if (manifest_bytes == null) - { - Console.WriteLine("Password was invalid for branch {0}", branch); - return INVALID_MANIFEST_ID; - } - - return BitConverter.ToUInt64(manifest_bytes, 0); - } - else if (encrypted_v2 != KeyValue.Invalid) - { - // Submit the password to Steam now to get encryption keys - steam3.CheckAppBetaPassword(appId, Config.BetaPassword); - - if (!steam3.AppBetaPasswords.ContainsKey(branch)) - { - Console.WriteLine("Password was invalid for branch {0}", branch); - return INVALID_MANIFEST_ID; - } - - byte[] input = Util.DecodeHexString(encrypted_v2.Value); - byte[] manifest_bytes; - try - { - manifest_bytes = CryptoHelper.SymmetricDecryptECB(input, steam3.AppBetaPasswords[branch]); - } - catch (Exception e) - { - Console.WriteLine("Failed to decrypt branch {0}: {1}", branch, e.Message); - return INVALID_MANIFEST_ID; - } - - return BitConverter.ToUInt64(manifest_bytes, 0); - } - else - { - Console.WriteLine("Unhandled depot encryption for depotId {0}", depotId); - return INVALID_MANIFEST_ID; - } - - } - - return INVALID_MANIFEST_ID; - } - - if (node.Value == null) - return INVALID_MANIFEST_ID; - - return UInt64.Parse(node.Value); - } - - static string GetAppOrDepotName(uint depotId, uint appId) - { - if (depotId == INVALID_DEPOT_ID) - { - KeyValue info = GetSteam3AppSection(appId, EAppInfoSection.Common); - - if (info == null) - return String.Empty; - - return info["name"].AsString(); - } - else - { - KeyValue depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); - - if (depots == null) - return String.Empty; - - KeyValue depotChild = depots[depotId.ToString()]; - - if (depotChild == null) - return String.Empty; - - return depotChild["name"].AsString(); - } - } - - public static bool InitializeSteam3(string username, string password) - { - steam3 = new Steam3Session( - new SteamUser.LogOnDetails() - { - Username = username, - Password = password, - } - ); - - steam3Credentials = steam3.WaitForCredentials(); - - if (!steam3Credentials.IsValid) - { - Console.WriteLine("Unable to get steam3 credentials."); - return false; - } - - cdnPool = new CDNClientPool(steam3); - return true; - } - - public static void ShutdownSteam3() - { - if (steam3 == null) - return; - - steam3.Disconnect(); - } - - public static async Task DownloadAppAsync(uint appId, uint depotId, string branch, bool forceDepot = false) - { - if(steam3 != null) - steam3.RequestAppInfo(appId); - - if (!AccountHasAccess(appId)) - { - if (steam3.RequestFreeAppLicense(appId)) - { - Console.WriteLine("Obtained FreeOnDemand license for app {0}", appId); - } - else - { - string contentName = GetAppOrDepotName(INVALID_DEPOT_ID, appId); - Console.WriteLine("App {0} ({1}) is not available from this account.", appId, contentName); - return; - } - } - - Console.WriteLine("Using app branch: '{0}'.", branch); - - var depotIDs = new List(); - KeyValue depots = GetSteam3AppSection(appId, EAppInfoSection.Depots); - - - if (forceDepot) - { - depotIDs.Add(depotId); - } - else - { - if (depots != null) - { - foreach (var depotSection in depots.Children) - { - uint id = INVALID_DEPOT_ID; - if (depotSection.Children.Count == 0) - continue; - - if (!uint.TryParse(depotSection.Name, out id)) - continue; - - if (depotId != INVALID_DEPOT_ID && id != depotId) - continue; - - if (!Config.DownloadAllPlatforms) - { - var depotConfig = depotSection["config"]; - if (depotConfig != KeyValue.Invalid && depotConfig["oslist"] != KeyValue.Invalid && !string.IsNullOrWhiteSpace(depotConfig["oslist"].Value)) - { - var oslist = depotConfig["oslist"].Value.Split(','); - if (Array.IndexOf(oslist, Util.GetSteamOS()) == -1) - continue; - } - } - - depotIDs.Add(id); - } - } - if (depotIDs == null || (depotIDs.Count == 0 && depotId == INVALID_DEPOT_ID)) - { - Console.WriteLine("Couldn't find any depots to download for app {0}", appId); - return; - } - else if (depotIDs.Count == 0) - { - Console.Write("Depot {0} not listed for app {1}", depotId, appId); - if (!Config.DownloadAllPlatforms) - { - Console.Write(" or not available on this platform"); - } - Console.WriteLine(); - return; - } - } - - var infos = new List(); - - foreach (var depot in depotIDs) - { - var info = GetDepotInfo(depot, appId, branch); - if (info != null) - { - infos.Add(info); - } - } - - try - { - await DownloadSteam3Async(appId, infos).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - Console.WriteLine("App {0} was not completely downloaded.", appId); - } - } - - static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, string branch) - { - if(steam3 != null && appId != INVALID_APP_ID) - steam3.RequestAppInfo((uint)appId); - - string contentName = GetAppOrDepotName(depotId, appId); - - if (!AccountHasAccess(depotId)) - { - Console.WriteLine("Depot {0} ({1}) is not available from this account.", depotId, contentName); - - return null; - } - - if (steam3 != null) - steam3.RequestAppTicket((uint)depotId); - - ulong manifestID = GetSteam3DepotManifest(depotId, appId, branch); - if (manifestID == INVALID_MANIFEST_ID && branch != "public") - { - Console.WriteLine("Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch); - branch = "public"; - manifestID = GetSteam3DepotManifest(depotId, appId, branch); - } - - if (manifestID == INVALID_MANIFEST_ID) - { - Console.WriteLine("Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName); - return null; - } - - uint uVersion = GetSteam3AppBuildNumber(appId, branch); - - string installDir; - if (!CreateDirectories(depotId, uVersion, out installDir)) - { - Console.WriteLine("Error: Unable to create install directories!"); - return null; - } - - steam3.RequestDepotKey( depotId, appId ); - if (!steam3.DepotKeys.ContainsKey(depotId)) - { - Console.WriteLine("No valid depot key for {0}, unable to download.", depotId); - return null; - } - - byte[] depotKey = steam3.DepotKeys[depotId]; - - var info = new DepotDownloadInfo( depotId, manifestID, installDir, contentName ); - info.depotKey = depotKey; - return info; - } - - private class ChunkMatch - { - public ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk) - { - OldChunk = oldChunk; - NewChunk = newChunk; - } - public ProtoManifest.ChunkData OldChunk { get; private set; } - public ProtoManifest.ChunkData NewChunk { get; private set; } - } - - private static async Task DownloadSteam3Async( uint appId, List depots ) - { - ulong TotalBytesCompressed = 0; - ulong TotalBytesUncompressed = 0; - - foreach (var depot in depots) - { - ulong DepotBytesCompressed = 0; - ulong DepotBytesUncompressed = 0; - - Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName); - - CancellationTokenSource cts = new CancellationTokenSource(); - - ProtoManifest oldProtoManifest = null; - ProtoManifest newProtoManifest = null; - string configDir = Path.Combine(depot.installDir, CONFIG_DIR); - - ulong lastManifestId = INVALID_MANIFEST_ID; - ConfigStore.TheConfig.LastManifests.TryGetValue(depot.id, out lastManifestId); - - // In case we have an early exit, this will force equiv of verifyall next run. - ConfigStore.TheConfig.LastManifests[depot.id] = INVALID_MANIFEST_ID; - ConfigStore.Save(); - - if (lastManifestId != INVALID_MANIFEST_ID) - { - var oldManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", lastManifestId)); - if (File.Exists(oldManifestFileName)) - oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName); - } - - if (lastManifestId == depot.manifestId && oldProtoManifest != null) - { - newProtoManifest = oldProtoManifest; - Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); - } - else - { - var newManifestFileName = Path.Combine(configDir, string.Format("{0}.bin", depot.manifestId)); - if (newManifestFileName != null) - { - newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName); - } - - if (newProtoManifest != null) - { - Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id); - } - else - { - Console.Write("Downloading depot manifest..."); - - DepotManifest depotManifest = null; - - while (depotManifest == null) - { - CDNClient client = null; - try { - client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None).ConfigureAwait(false); - - depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId).ConfigureAwait(false); - - cdnPool.ReturnConnection(client); - } - catch (WebException e) - { - cdnPool.ReturnBrokenConnection(client); - - if (e.Status == WebExceptionStatus.ProtocolError) - { - var response = e.Response as HttpWebResponse; - if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) - { - Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId); - break; - } - else - { - Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, response.StatusCode); - } - } - else - { - Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Status); - } - } - catch (Exception e) - { - cdnPool.ReturnBrokenConnection(client); - Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message); - } - } - - if (depotManifest == null) - { - Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id); - return; - } - - newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId); - newProtoManifest.SaveToFile(newManifestFileName); - - Console.WriteLine(" Done!"); - } - } - - newProtoManifest.Files.Sort((x, y) => { return x.FileName.CompareTo(y.FileName); }); - - if (Config.DownloadManifestOnly) - { - StringBuilder manifestBuilder = new StringBuilder(); - string txtManifest = Path.Combine(depot.installDir, string.Format("manifest_{0}.txt", depot.id)); - - foreach (var file in newProtoManifest.Files) - { - if (file.Flags.HasFlag(EDepotFileFlag.Directory)) - continue; - - manifestBuilder.Append(string.Format("{0}\n", file.FileName)); - } - - File.WriteAllText(txtManifest, manifestBuilder.ToString()); - continue; - } - - ulong complete_download_size = 0; - ulong size_downloaded = 0; - string stagingDir = Path.Combine(depot.installDir, STAGING_DIR); - - var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList(); - - // Pre-process - filesAfterExclusions.ForEach(file => - { - var fileFinalPath = Path.Combine(depot.installDir, file.FileName); - var fileStagingPath = Path.Combine(stagingDir, file.FileName); - - if (file.Flags.HasFlag(EDepotFileFlag.Directory)) - { - Directory.CreateDirectory(fileFinalPath); - Directory.CreateDirectory(fileStagingPath); - } - else - { - // Some manifests don't explicitly include all necessary directories - Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath)); - Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath)); - - complete_download_size += file.TotalSize; - } - }); - - var semaphore = new SemaphoreSlim(Config.MaxDownloads); - var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray(); - var tasks = new Task[files.Length]; - for (var i = 0; i < files.Length; i++) - { - var file = files[i]; - var task = Task.Run(async () => - { - cts.Token.ThrowIfCancellationRequested(); - - try - { - await semaphore.WaitAsync().ConfigureAwait(false); - - string fileFinalPath = Path.Combine(depot.installDir, file.FileName); - string fileStagingPath = Path.Combine(stagingDir, file.FileName); - - // This may still exist if the previous run exited before cleanup - if (File.Exists(fileStagingPath)) - { - File.Delete(fileStagingPath); - } - - FileStream fs = null; - List neededChunks; - FileInfo fi = new FileInfo(fileFinalPath); - if (!fi.Exists) - { - // create new file. need all chunks - fs = File.Create(fileFinalPath); - fs.SetLength((long)file.TotalSize); - neededChunks = new List(file.Chunks); - } - else - { - // open existing - ProtoManifest.FileData oldManifestFile = null; - if (oldProtoManifest != null) - { - oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); - } - - if (oldManifestFile != null) - { - neededChunks = new List(); - - if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) - { - // we have a version of this file, but it doesn't fully match what we want - - var matchingChunks = new List(); - - foreach (var chunk in file.Chunks) - { - var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); - if (oldChunk != null) - { - matchingChunks.Add(new ChunkMatch(oldChunk, chunk)); - } - else - { - neededChunks.Add(chunk); - } - } - - File.Move(fileFinalPath, fileStagingPath); - - fs = File.Open(fileFinalPath, FileMode.Create); - fs.SetLength((long)file.TotalSize); - - using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) - { - foreach (var match in matchingChunks) - { - fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); - - byte[] tmp = new byte[match.OldChunk.UncompressedLength]; - fsOld.Read(tmp, 0, tmp.Length); - - byte[] adler = Util.AdlerHash(tmp); - if (!adler.SequenceEqual(match.OldChunk.Checksum)) - { - neededChunks.Add(match.NewChunk); - } - else - { - fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin); - fs.Write(tmp, 0, tmp.Length); - } - } - } - - File.Delete(fileStagingPath); - } - } - else - { - // No old manifest or file not in old manifest. We must validate. - - fs = File.Open(fileFinalPath, FileMode.Open); - if ((ulong)fi.Length != file.TotalSize) - { - fs.SetLength((long)file.TotalSize); - } - - neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); - } - - if (neededChunks.Count() == 0) - { - size_downloaded += file.TotalSize; - Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath); - if (fs != null) - fs.Dispose(); - return; - } - else - { - size_downloaded += (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum()); - } - } - - foreach (var chunk in neededChunks) - { - if (cts.IsCancellationRequested) break; - - string chunkID = Util.EncodeHexString(chunk.ChunkID); - CDNClient.DepotChunk chunkData = null; - - while (!cts.IsCancellationRequested) - { - CDNClient client; - try - { - client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - break; - } - - DepotManifest.ChunkData data = new DepotManifest.ChunkData(); - data.ChunkID = chunk.ChunkID; - data.Checksum = chunk.Checksum; - data.Offset = chunk.Offset; - data.CompressedLength = chunk.CompressedLength; - data.UncompressedLength = chunk.UncompressedLength; - - try - { - chunkData = await client.DownloadDepotChunkAsync(depot.id, data).ConfigureAwait(false); - cdnPool.ReturnConnection(client); - break; - } - catch (WebException e) - { - cdnPool.ReturnBrokenConnection(client); - - if (e.Status == WebExceptionStatus.ProtocolError) - { - var response = e.Response as HttpWebResponse; - if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) - { - Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); - cts.Cancel(); - break; - } - else - { - Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode); - } - } - else - { - Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.Status); - } - } - catch (Exception e) - { - cdnPool.ReturnBrokenConnection(client); - Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); - } - } - - if (chunkData == null) - { - Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id); - return; - } - - TotalBytesCompressed += chunk.CompressedLength; - DepotBytesCompressed += chunk.CompressedLength; - TotalBytesUncompressed += chunk.UncompressedLength; - DepotBytesUncompressed += chunk.UncompressedLength; - - fs.Seek((long)chunk.Offset, SeekOrigin.Begin); - fs.Write(chunkData.Data, 0, chunkData.Data.Length); - - size_downloaded += chunk.UncompressedLength; - } - - fs.Dispose(); - - Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath); - } - finally - { - semaphore.Release(); - } - }); - - tasks[i] = task; - } - - await Task.WhenAll(tasks).ConfigureAwait(false); - - ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId; - ConfigStore.Save(); - - Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed); - } - - Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count); - } - } -} +using SteamKit2; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace DepotDownloader +{ + static class ContentDownloader + { + public const uint INVALID_APP_ID = uint.MaxValue; + public const uint INVALID_DEPOT_ID = uint.MaxValue; + public const ulong INVALID_MANIFEST_ID = ulong.MaxValue; + + public static DownloadConfig Config = new DownloadConfig(); + + private static Steam3Session steam3; + private static Steam3Session.Credentials steam3Credentials; + private static CDNClientPool cdnPool; + + private const string DEFAULT_DOWNLOAD_DIR = "depots"; + private const string CONFIG_DIR = ".DepotDownloader"; + private static readonly string STAGING_DIR = Path.Combine( CONFIG_DIR, "staging" ); + + private sealed class DepotDownloadInfo + { + public uint id { get; private set; } + public string installDir { get; private set; } + public string contentName { get; private set; } + + public ulong manifestId { get; private set; } + public byte[] depotKey; + + public DepotDownloadInfo( uint depotid, ulong manifestId, string installDir, string contentName ) + { + this.id = depotid; + this.manifestId = manifestId; + this.installDir = installDir; + this.contentName = contentName; + } + } + + static bool CreateDirectories( uint depotId, uint depotVersion, out string installDir ) + { + installDir = null; + try + { + if ( string.IsNullOrWhiteSpace( ContentDownloader.Config.InstallDirectory ) ) + { + Directory.CreateDirectory( DEFAULT_DOWNLOAD_DIR ); + + string depotPath = Path.Combine( DEFAULT_DOWNLOAD_DIR, depotId.ToString() ); + Directory.CreateDirectory( depotPath ); + + installDir = Path.Combine( depotPath, depotVersion.ToString() ); + Directory.CreateDirectory( installDir ); + + Directory.CreateDirectory( Path.Combine( installDir, CONFIG_DIR ) ); + Directory.CreateDirectory( Path.Combine( installDir, STAGING_DIR ) ); + } + else + { + Directory.CreateDirectory( ContentDownloader.Config.InstallDirectory ); + + installDir = ContentDownloader.Config.InstallDirectory; + + Directory.CreateDirectory( Path.Combine( installDir, CONFIG_DIR ) ); + Directory.CreateDirectory( Path.Combine( installDir, STAGING_DIR ) ); + } + } + catch + { + return false; + } + + return true; + } + + static bool TestIsFileIncluded( string filename ) + { + if ( !Config.UsingFileList ) + return true; + + foreach ( string fileListEntry in Config.FilesToDownload ) + { + if ( fileListEntry.Equals( filename, StringComparison.OrdinalIgnoreCase ) ) + return true; + } + + foreach ( Regex rgx in Config.FilesToDownloadRegex ) + { + Match m = rgx.Match( filename ); + + if ( m.Success ) + return true; + } + + return false; + } + + static bool AccountHasAccess( uint depotId ) + { + if ( steam3 == null || steam3.steamUser.SteamID == null || ( steam3.Licenses == null && steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser ) ) + return false; + + IEnumerable licenseQuery; + if ( steam3.steamUser.SteamID.AccountType == EAccountType.AnonUser ) + { + licenseQuery = new List() { 17906 }; + } + else + { + licenseQuery = steam3.Licenses.Select( x => x.PackageID ); + } + + steam3.RequestPackageInfo( licenseQuery ); + + foreach ( var license in licenseQuery ) + { + SteamApps.PICSProductInfoCallback.PICSProductInfo package; + if ( steam3.PackageInfo.TryGetValue( license, out package ) && package != null ) + { + if ( package.KeyValues[ "appids" ].Children.Any( child => child.AsInteger() == depotId ) ) + return true; + + if ( package.KeyValues[ "depotids" ].Children.Any( child => child.AsInteger() == depotId ) ) + return true; + } + } + + return false; + } + + internal static KeyValue GetSteam3AppSection( uint appId, EAppInfoSection section ) + { + if ( steam3 == null || steam3.AppInfo == null ) + { + return null; + } + + SteamApps.PICSProductInfoCallback.PICSProductInfo app; + if ( !steam3.AppInfo.TryGetValue( appId, out app ) || app == null ) + { + return null; + } + + KeyValue appinfo = app.KeyValues; + string section_key; + + switch ( section ) + { + case EAppInfoSection.Common: + section_key = "common"; + break; + case EAppInfoSection.Extended: + section_key = "extended"; + break; + case EAppInfoSection.Config: + section_key = "config"; + break; + case EAppInfoSection.Depots: + section_key = "depots"; + break; + default: + throw new NotImplementedException(); + } + + KeyValue section_kv = appinfo.Children.Where( c => c.Name == section_key ).FirstOrDefault(); + return section_kv; + } + + static uint GetSteam3AppBuildNumber( uint appId, string branch ) + { + if ( appId == INVALID_APP_ID ) + return 0; + + + KeyValue depots = ContentDownloader.GetSteam3AppSection( appId, EAppInfoSection.Depots ); + KeyValue branches = depots[ "branches" ]; + KeyValue node = branches[ branch ]; + + if ( node == KeyValue.Invalid ) + return 0; + + KeyValue buildid = node[ "buildid" ]; + + if ( buildid == KeyValue.Invalid ) + return 0; + + return uint.Parse( buildid.Value ); + } + + static ulong GetSteam3DepotManifest( uint depotId, uint appId, string branch ) + { + if ( Config.ManifestId != INVALID_MANIFEST_ID ) + return Config.ManifestId; + + KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); + KeyValue depotChild = depots[ depotId.ToString() ]; + + if ( depotChild == KeyValue.Invalid ) + return INVALID_MANIFEST_ID; + + // Shared depots can either provide manifests, or leave you relying on their parent app. + // It seems that with the latter, "sharedinstall" will exist (and equals 2 in the one existance I know of). + // Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680. + if ( depotChild[ "manifests" ] == KeyValue.Invalid && depotChild[ "depotfromapp" ] != KeyValue.Invalid ) + { + uint otherAppId = ( uint )depotChild[ "depotfromapp" ].AsInteger(); + if ( otherAppId == appId ) + { + // This shouldn't ever happen, but ya never know with Valve. Don't infinite loop. + Console.WriteLine( "App {0}, Depot {1} has depotfromapp of {2}!", + appId, depotId, otherAppId ); + return INVALID_MANIFEST_ID; + } + + steam3.RequestAppInfo( otherAppId ); + + return GetSteam3DepotManifest( depotId, otherAppId, branch ); + } + + var manifests = depotChild[ "manifests" ]; + var manifests_encrypted = depotChild[ "encryptedmanifests" ]; + + if ( manifests.Children.Count == 0 && manifests_encrypted.Children.Count == 0 ) + return INVALID_MANIFEST_ID; + + var node = manifests[ branch ]; + + if ( branch != "Public" && node == KeyValue.Invalid ) + { + var node_encrypted = manifests_encrypted[ branch ]; + if ( node_encrypted != KeyValue.Invalid ) + { + string password = Config.BetaPassword; + if ( password == null ) + { + Console.Write( "Please enter the password for branch {0}: ", branch ); + Config.BetaPassword = password = Console.ReadLine(); + } + + var encrypted_v1 = node_encrypted[ "encrypted_gid" ]; + var encrypted_v2 = node_encrypted[ "encrypted_gid_2" ]; + + if ( encrypted_v1 != KeyValue.Invalid ) + { + byte[] input = Util.DecodeHexString( encrypted_v1.Value ); + byte[] manifest_bytes = CryptoHelper.VerifyAndDecryptPassword( input, password ); + + if ( manifest_bytes == null ) + { + Console.WriteLine( "Password was invalid for branch {0}", branch ); + return INVALID_MANIFEST_ID; + } + + return BitConverter.ToUInt64( manifest_bytes, 0 ); + } + else if ( encrypted_v2 != KeyValue.Invalid ) + { + // Submit the password to Steam now to get encryption keys + steam3.CheckAppBetaPassword( appId, Config.BetaPassword ); + + if ( !steam3.AppBetaPasswords.ContainsKey( branch ) ) + { + Console.WriteLine( "Password was invalid for branch {0}", branch ); + return INVALID_MANIFEST_ID; + } + + byte[] input = Util.DecodeHexString( encrypted_v2.Value ); + byte[] manifest_bytes; + try + { + manifest_bytes = CryptoHelper.SymmetricDecryptECB( input, steam3.AppBetaPasswords[ branch ] ); + } + catch ( Exception e ) + { + Console.WriteLine( "Failed to decrypt branch {0}: {1}", branch, e.Message ); + return INVALID_MANIFEST_ID; + } + + return BitConverter.ToUInt64( manifest_bytes, 0 ); + } + else + { + Console.WriteLine( "Unhandled depot encryption for depotId {0}", depotId ); + return INVALID_MANIFEST_ID; + } + + } + + return INVALID_MANIFEST_ID; + } + + if ( node.Value == null ) + return INVALID_MANIFEST_ID; + + return UInt64.Parse( node.Value ); + } + + static string GetAppOrDepotName( uint depotId, uint appId ) + { + if ( depotId == INVALID_DEPOT_ID ) + { + KeyValue info = GetSteam3AppSection( appId, EAppInfoSection.Common ); + + if ( info == null ) + return String.Empty; + + return info[ "name" ].AsString(); + } + else + { + KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); + + if ( depots == null ) + return String.Empty; + + KeyValue depotChild = depots[ depotId.ToString() ]; + + if ( depotChild == null ) + return String.Empty; + + return depotChild[ "name" ].AsString(); + } + } + + public static bool InitializeSteam3( string username, string password ) + { + string loginKey = null; + + if ( username != null && Config.RememberPassword ) + { + _ = ConfigStore.TheConfig.LoginKeys.TryGetValue( username, out loginKey ); + } + + steam3 = new Steam3Session( + new SteamUser.LogOnDetails() + { + Username = username, + Password = loginKey == null ? password : null, + ShouldRememberPassword = Config.RememberPassword, + LoginKey = loginKey, + } + ); + + steam3Credentials = steam3.WaitForCredentials(); + + if ( !steam3Credentials.IsValid ) + { + Console.WriteLine( "Unable to get steam3 credentials." ); + return false; + } + + cdnPool = new CDNClientPool( steam3 ); + return true; + } + + public static void ShutdownSteam3() + { + if ( steam3 == null ) + return; + + steam3.TryWaitForLoginKey(); + steam3.Disconnect(); + } + + public static async Task DownloadAppAsync( uint appId, uint depotId, string branch, string os = null, bool forceDepot = false ) + { + if ( steam3 != null ) + steam3.RequestAppInfo( appId ); + + if ( !AccountHasAccess( appId ) ) + { + if ( steam3.RequestFreeAppLicense( appId ) ) + { + Console.WriteLine( "Obtained FreeOnDemand license for app {0}", appId ); + } + else + { + string contentName = GetAppOrDepotName( INVALID_DEPOT_ID, appId ); + Console.WriteLine( "App {0} ({1}) is not available from this account.", appId, contentName ); + return; + } + } + + Console.WriteLine( "Using app branch: '{0}'.", branch ); + + var depotIDs = new List(); + KeyValue depots = GetSteam3AppSection( appId, EAppInfoSection.Depots ); + + + if ( forceDepot ) + { + depotIDs.Add( depotId ); + } + else + { + if ( depots != null ) + { + foreach ( var depotSection in depots.Children ) + { + uint id = INVALID_DEPOT_ID; + if ( depotSection.Children.Count == 0 ) + continue; + + if ( !uint.TryParse( depotSection.Name, out id ) ) + continue; + + if ( depotId != INVALID_DEPOT_ID && id != depotId ) + continue; + + if ( !Config.DownloadAllPlatforms ) + { + var depotConfig = depotSection[ "config" ]; + if ( depotConfig != KeyValue.Invalid && depotConfig[ "oslist" ] != KeyValue.Invalid && !string.IsNullOrWhiteSpace( depotConfig[ "oslist" ].Value ) ) + { + var oslist = depotConfig[ "oslist" ].Value.Split( ',' ); + if ( Array.IndexOf( oslist, os ?? Util.GetSteamOS() ) == -1 ) + continue; + } + } + + depotIDs.Add( id ); + } + } + if ( depotIDs == null || ( depotIDs.Count == 0 && depotId == INVALID_DEPOT_ID ) ) + { + Console.WriteLine( "Couldn't find any depots to download for app {0}", appId ); + return; + } + else if ( depotIDs.Count == 0 ) + { + Console.Write( "Depot {0} not listed for app {1}", depotId, appId ); + if ( !Config.DownloadAllPlatforms ) + { + Console.Write( " or not available on this platform" ); + } + Console.WriteLine(); + return; + } + } + + var infos = new List(); + + foreach ( var depot in depotIDs ) + { + var info = GetDepotInfo( depot, appId, branch ); + if ( info != null ) + { + infos.Add( info ); + } + } + + try + { + await DownloadSteam3Async( appId, infos ).ConfigureAwait( false ); + } + catch ( OperationCanceledException ) + { + Console.WriteLine( "App {0} was not completely downloaded.", appId ); + } + } + + static DepotDownloadInfo GetDepotInfo( uint depotId, uint appId, string branch ) + { + if ( steam3 != null && appId != INVALID_APP_ID ) + steam3.RequestAppInfo( ( uint )appId ); + + string contentName = GetAppOrDepotName( depotId, appId ); + + if ( !AccountHasAccess( depotId ) ) + { + Console.WriteLine( "Depot {0} ({1}) is not available from this account.", depotId, contentName ); + + return null; + } + + if ( steam3 != null ) + steam3.RequestAppTicket( ( uint )depotId ); + + ulong manifestID = GetSteam3DepotManifest( depotId, appId, branch ); + if ( manifestID == INVALID_MANIFEST_ID && branch != "public" ) + { + Console.WriteLine( "Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch ); + branch = "public"; + manifestID = GetSteam3DepotManifest( depotId, appId, branch ); + } + + if ( manifestID == INVALID_MANIFEST_ID ) + { + Console.WriteLine( "Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName ); + return null; + } + + uint uVersion = GetSteam3AppBuildNumber( appId, branch ); + + string installDir; + if ( !CreateDirectories( depotId, uVersion, out installDir ) ) + { + Console.WriteLine( "Error: Unable to create install directories!" ); + return null; + } + + steam3.RequestDepotKey( depotId, appId ); + if ( !steam3.DepotKeys.ContainsKey( depotId ) ) + { + Console.WriteLine( "No valid depot key for {0}, unable to download.", depotId ); + return null; + } + + byte[] depotKey = steam3.DepotKeys[ depotId ]; + + var info = new DepotDownloadInfo( depotId, manifestID, installDir, contentName ); + info.depotKey = depotKey; + return info; + } + + private class ChunkMatch + { + public ChunkMatch( ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk ) + { + OldChunk = oldChunk; + NewChunk = newChunk; + } + public ProtoManifest.ChunkData OldChunk { get; private set; } + public ProtoManifest.ChunkData NewChunk { get; private set; } + } + + private static async Task DownloadSteam3Async( uint appId, List depots ) + { + ulong TotalBytesCompressed = 0; + ulong TotalBytesUncompressed = 0; + + foreach ( var depot in depots ) + { + ulong DepotBytesCompressed = 0; + ulong DepotBytesUncompressed = 0; + + Console.WriteLine( "Downloading depot {0} - {1}", depot.id, depot.contentName ); + + CancellationTokenSource cts = new CancellationTokenSource(); + + ProtoManifest oldProtoManifest = null; + ProtoManifest newProtoManifest = null; + string configDir = Path.Combine( depot.installDir, CONFIG_DIR ); + + ulong lastManifestId = INVALID_MANIFEST_ID; + ConfigStore.TheConfig.LastManifests.TryGetValue( depot.id, out lastManifestId ); + + // In case we have an early exit, this will force equiv of verifyall next run. + ConfigStore.TheConfig.LastManifests[ depot.id ] = INVALID_MANIFEST_ID; + ConfigStore.Save(); + + if ( lastManifestId != INVALID_MANIFEST_ID ) + { + var oldManifestFileName = Path.Combine( configDir, string.Format( "{0}.bin", lastManifestId ) ); + if ( File.Exists( oldManifestFileName ) ) + oldProtoManifest = ProtoManifest.LoadFromFile( oldManifestFileName ); + } + + if ( lastManifestId == depot.manifestId && oldProtoManifest != null ) + { + newProtoManifest = oldProtoManifest; + Console.WriteLine( "Already have manifest {0} for depot {1}.", depot.manifestId, depot.id ); + } + else + { + var newManifestFileName = Path.Combine( configDir, string.Format( "{0}.bin", depot.manifestId ) ); + if ( newManifestFileName != null ) + { + newProtoManifest = ProtoManifest.LoadFromFile( newManifestFileName ); + } + + if ( newProtoManifest != null ) + { + Console.WriteLine( "Already have manifest {0} for depot {1}.", depot.manifestId, depot.id ); + } + else + { + Console.Write( "Downloading depot manifest..." ); + + DepotManifest depotManifest = null; + + while ( depotManifest == null ) + { + CDNClient client = null; + try + { + client = await cdnPool.GetConnectionForDepotAsync( appId, depot.id, depot.depotKey, CancellationToken.None ).ConfigureAwait( false ); + + depotManifest = await client.DownloadManifestAsync( depot.id, depot.manifestId ).ConfigureAwait( false ); + + cdnPool.ReturnConnection( client ); + } + catch ( WebException e ) + { + cdnPool.ReturnBrokenConnection( client ); + + if ( e.Status == WebExceptionStatus.ProtocolError ) + { + var response = e.Response as HttpWebResponse; + if ( response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden ) + { + Console.WriteLine( "Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId ); + break; + } + else + { + Console.WriteLine( "Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, response.StatusCode ); + } + } + else + { + Console.WriteLine( "Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Status ); + } + } + catch ( Exception e ) + { + cdnPool.ReturnBrokenConnection( client ); + Console.WriteLine( "Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message ); + } + } + + if ( depotManifest == null ) + { + Console.WriteLine( "\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id ); + return; + } + + newProtoManifest = new ProtoManifest( depotManifest, depot.manifestId ); + newProtoManifest.SaveToFile( newManifestFileName ); + + Console.WriteLine( " Done!" ); + } + } + + newProtoManifest.Files.Sort( ( x, y ) => { return x.FileName.CompareTo( y.FileName ); } ); + + if ( Config.DownloadManifestOnly ) + { + StringBuilder manifestBuilder = new StringBuilder(); + string txtManifest = Path.Combine( depot.installDir, string.Format( "manifest_{0}.txt", depot.id ) ); + + foreach ( var file in newProtoManifest.Files ) + { + if ( file.Flags.HasFlag( EDepotFileFlag.Directory ) ) + continue; + + manifestBuilder.Append( string.Format( "{0}\n", file.FileName ) ); + } + + File.WriteAllText( txtManifest, manifestBuilder.ToString() ); + continue; + } + + ulong complete_download_size = 0; + ulong size_downloaded = 0; + string stagingDir = Path.Combine( depot.installDir, STAGING_DIR ); + + var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where( f => TestIsFileIncluded( f.FileName ) ).ToList(); + + // Pre-process + filesAfterExclusions.ForEach( file => + { + var fileFinalPath = Path.Combine( depot.installDir, file.FileName ); + var fileStagingPath = Path.Combine( stagingDir, file.FileName ); + + if ( file.Flags.HasFlag( EDepotFileFlag.Directory ) ) + { + Directory.CreateDirectory( fileFinalPath ); + Directory.CreateDirectory( fileStagingPath ); + } + else + { + // Some manifests don't explicitly include all necessary directories + Directory.CreateDirectory( Path.GetDirectoryName( fileFinalPath ) ); + Directory.CreateDirectory( Path.GetDirectoryName( fileStagingPath ) ); + + complete_download_size += file.TotalSize; + } + } ); + + var semaphore = new SemaphoreSlim( Config.MaxDownloads ); + var files = filesAfterExclusions.Where( f => !f.Flags.HasFlag( EDepotFileFlag.Directory ) ).ToArray(); + var tasks = new Task[ files.Length ]; + for ( var i = 0; i < files.Length; i++ ) + { + var file = files[ i ]; + var task = Task.Run( async () => + { + cts.Token.ThrowIfCancellationRequested(); + + try + { + string fileFinalPath = Path.Combine( depot.installDir, file.FileName ); + string fileStagingPath = Path.Combine( stagingDir, file.FileName ); + + // This may still exist if the previous run exited before cleanup + if ( File.Exists( fileStagingPath ) ) + { + File.Delete( fileStagingPath ); + } + + FileStream fs = null; + List neededChunks; + FileInfo fi = new FileInfo( fileFinalPath ); + if ( !fi.Exists ) + { + // create new file. need all chunks + fs = File.Create( fileFinalPath ); + fs.SetLength( ( long )file.TotalSize ); + neededChunks = new List( file.Chunks ); + } + else + { + // open existing + ProtoManifest.FileData oldManifestFile = null; + if ( oldProtoManifest != null ) + { + oldManifestFile = oldProtoManifest.Files.SingleOrDefault( f => f.FileName == file.FileName ); + } + + if ( oldManifestFile != null ) + { + neededChunks = new List(); + + if ( Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual( file.FileHash ) ) + { + // we have a version of this file, but it doesn't fully match what we want + + var matchingChunks = new List(); + + foreach ( var chunk in file.Chunks ) + { + var oldChunk = oldManifestFile.Chunks.FirstOrDefault( c => c.ChunkID.SequenceEqual( chunk.ChunkID ) ); + if ( oldChunk != null ) + { + matchingChunks.Add( new ChunkMatch( oldChunk, chunk ) ); + } + else + { + neededChunks.Add( chunk ); + } + } + + File.Move( fileFinalPath, fileStagingPath ); + + fs = File.Open( fileFinalPath, FileMode.Create ); + fs.SetLength( ( long )file.TotalSize ); + + using ( var fsOld = File.Open( fileStagingPath, FileMode.Open ) ) + { + foreach ( var match in matchingChunks ) + { + fsOld.Seek( ( long )match.OldChunk.Offset, SeekOrigin.Begin ); + + byte[] tmp = new byte[ match.OldChunk.UncompressedLength ]; + fsOld.Read( tmp, 0, tmp.Length ); + + byte[] adler = Util.AdlerHash( tmp ); + if ( !adler.SequenceEqual( match.OldChunk.Checksum ) ) + { + neededChunks.Add( match.NewChunk ); + } + else + { + fs.Seek( ( long )match.NewChunk.Offset, SeekOrigin.Begin ); + fs.Write( tmp, 0, tmp.Length ); + } + } + } + + File.Delete( fileStagingPath ); + } + } + else + { + // No old manifest or file not in old manifest. We must validate. + + fs = File.Open( fileFinalPath, FileMode.Open ); + if ( ( ulong )fi.Length != file.TotalSize ) + { + fs.SetLength( ( long )file.TotalSize ); + } + + neededChunks = Util.ValidateSteam3FileChecksums( fs, file.Chunks.OrderBy( x => x.Offset ).ToArray() ); + } + + if ( neededChunks.Count() == 0 ) + { + size_downloaded += file.TotalSize; + Console.WriteLine( "{0,6:#00.00}% {1}", ( ( float )size_downloaded / ( float )complete_download_size ) * 100.0f, fileFinalPath ); + if ( fs != null ) + fs.Dispose(); + return; + } + else + { + size_downloaded += ( file.TotalSize - ( ulong )neededChunks.Select( x => ( long )x.UncompressedLength ).Sum() ); + } + } + + foreach ( var chunk in neededChunks ) + { + if ( cts.IsCancellationRequested ) break; + + string chunkID = Util.EncodeHexString( chunk.ChunkID ); + CDNClient.DepotChunk chunkData = null; + + while ( !cts.IsCancellationRequested ) + { + CDNClient client; + try + { + client = await cdnPool.GetConnectionForDepotAsync( appId, depot.id, depot.depotKey, cts.Token ).ConfigureAwait( false ); + } + catch ( OperationCanceledException ) + { + break; + } + + DepotManifest.ChunkData data = new DepotManifest.ChunkData(); + data.ChunkID = chunk.ChunkID; + data.Checksum = chunk.Checksum; + data.Offset = chunk.Offset; + data.CompressedLength = chunk.CompressedLength; + data.UncompressedLength = chunk.UncompressedLength; + + try + { + chunkData = await client.DownloadDepotChunkAsync( depot.id, data ).ConfigureAwait( false ); + cdnPool.ReturnConnection( client ); + break; + } + catch ( WebException e ) + { + cdnPool.ReturnBrokenConnection( client ); + + if ( e.Status == WebExceptionStatus.ProtocolError ) + { + var response = e.Response as HttpWebResponse; + if ( response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden ) + { + Console.WriteLine( "Encountered 401 for chunk {0}. Aborting.", chunkID ); + cts.Cancel(); + break; + } + else + { + Console.WriteLine( "Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode ); + } + } + else + { + Console.WriteLine( "Encountered error downloading chunk {0}: {1}", chunkID, e.Status ); + } + } + catch ( Exception e ) + { + cdnPool.ReturnBrokenConnection( client ); + Console.WriteLine( "Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message ); + } + } + + if ( chunkData == null ) + { + Console.WriteLine( "Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id ); + return; + } + + TotalBytesCompressed += chunk.CompressedLength; + DepotBytesCompressed += chunk.CompressedLength; + TotalBytesUncompressed += chunk.UncompressedLength; + DepotBytesUncompressed += chunk.UncompressedLength; + + fs.Seek( ( long )chunk.Offset, SeekOrigin.Begin ); + fs.Write( chunkData.Data, 0, chunkData.Data.Length ); + + size_downloaded += chunk.UncompressedLength; + } + + fs.Dispose(); + + Console.WriteLine( "{0,6:#00.00}% {1}", ( ( float )size_downloaded / ( float )complete_download_size ) * 100.0f, fileFinalPath ); + } + finally + { + semaphore.Release(); + } + } ); + + tasks[ i ] = task; + } + + await Task.WhenAll( tasks ).ConfigureAwait( false ); + + ConfigStore.TheConfig.LastManifests[ depot.id ] = depot.manifestId; + ConfigStore.Save(); + + Console.WriteLine( "Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed ); + } + + Console.WriteLine( "Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count ); + } + } +} diff --git a/DepotDownloader/DownloadConfig.cs b/DepotDownloader/DownloadConfig.cs index 84fc1b22..bd2480b7 100644 --- a/DepotDownloader/DownloadConfig.cs +++ b/DepotDownloader/DownloadConfig.cs @@ -24,5 +24,8 @@ namespace DepotDownloader public int MaxServers { get; set; } public int MaxDownloads { get; set; } + + public string SuppliedPassword { get; set; } + public bool RememberPassword { get; set; } } } diff --git a/DepotDownloader/Program.cs b/DepotDownloader/Program.cs index 20eedb8a..0d847078 100644 --- a/DepotDownloader/Program.cs +++ b/DepotDownloader/Program.cs @@ -11,7 +11,7 @@ namespace DepotDownloader class Program { static void Main( string[] args ) - => MainAsync(args).GetAwaiter().GetResult(); + => MainAsync( args ).GetAwaiter().GetResult(); static async Task MainAsync( string[] args ) { @@ -23,7 +23,7 @@ namespace DepotDownloader DebugLog.Enabled = false; - ConfigStore.LoadFromFile(Path.Combine(Directory.GetCurrentDirectory(), "DepotDownloader.config")); + ConfigStore.LoadFromFile( Path.Combine( Directory.GetCurrentDirectory(), "DepotDownloader.config" ) ); bool bDumpManifest = HasParameter( args, "-manifest-only" ); uint appId = GetParameter( args, "-app", ContentDownloader.INVALID_APP_ID ); @@ -36,24 +36,24 @@ namespace DepotDownloader return; } - if (depotId == ContentDownloader.INVALID_DEPOT_ID && ContentDownloader.Config.ManifestId != ContentDownloader.INVALID_MANIFEST_ID) + if ( depotId == ContentDownloader.INVALID_DEPOT_ID && ContentDownloader.Config.ManifestId != ContentDownloader.INVALID_MANIFEST_ID ) { - Console.WriteLine("Error: -manifest requires -depot to be specified"); + Console.WriteLine( "Error: -manifest requires -depot to be specified" ); return; } ContentDownloader.Config.DownloadManifestOnly = bDumpManifest; - int cellId = GetParameter(args, "-cellid", -1); - if (cellId == -1) + int cellId = GetParameter( args, "-cellid", -1 ); + if ( cellId == -1 ) { cellId = 0; } ContentDownloader.Config.CellID = cellId; - ContentDownloader.Config.BetaPassword = GetParameter(args, "-betapassword"); + ContentDownloader.Config.BetaPassword = GetParameter( args, "-betapassword" ); - string fileList = GetParameter(args, "-filelist"); + string fileList = GetParameter( args, "-filelist" ); string[] files = null; if ( fileList != null ) @@ -67,16 +67,16 @@ namespace DepotDownloader ContentDownloader.Config.FilesToDownload = new List(); ContentDownloader.Config.FilesToDownloadRegex = new List(); - foreach (var fileEntry in files) + foreach ( var fileEntry in files ) { try { - Regex rgx = new Regex(fileEntry, RegexOptions.Compiled | RegexOptions.IgnoreCase); - ContentDownloader.Config.FilesToDownloadRegex.Add(rgx); + Regex rgx = new Regex( fileEntry, RegexOptions.Compiled | RegexOptions.IgnoreCase ); + ContentDownloader.Config.FilesToDownloadRegex.Add( rgx ); } catch { - ContentDownloader.Config.FilesToDownload.Add(fileEntry); + ContentDownloader.Config.FilesToDownload.Add( fileEntry ); continue; } } @@ -89,39 +89,50 @@ namespace DepotDownloader } } - string username = GetParameter(args, "-username") ?? GetParameter(args, "-user"); - string password = GetParameter(args, "-password") ?? GetParameter(args, "-pass"); - ContentDownloader.Config.InstallDirectory = GetParameter(args, "-dir"); - ContentDownloader.Config.DownloadAllPlatforms = HasParameter(args, "-all-platforms"); - ContentDownloader.Config.VerifyAll = HasParameter(args, "-verify-all") || HasParameter(args, "-verify_all") || HasParameter(args, "-validate"); - ContentDownloader.Config.MaxServers = GetParameter(args, "-max-servers", 20); - ContentDownloader.Config.MaxDownloads = GetParameter(args, "-max-downloads", 4); - string branch = GetParameter(args, "-branch") ?? GetParameter(args, "-beta") ?? "Public"; - var forceDepot = HasParameter(args, "-force-depot"); + string username = GetParameter( args, "-username" ) ?? GetParameter( args, "-user" ); + string password = GetParameter( args, "-password" ) ?? GetParameter( args, "-pass" ); + ContentDownloader.Config.RememberPassword = HasParameter( args, "-remember-password" ); + ContentDownloader.Config.InstallDirectory = GetParameter( args, "-dir" ); + ContentDownloader.Config.DownloadAllPlatforms = HasParameter( args, "-all-platforms" ); + ContentDownloader.Config.VerifyAll = HasParameter( args, "-verify-all" ) || HasParameter( args, "-verify_all" ) || HasParameter( args, "-validate" ); + ContentDownloader.Config.MaxServers = GetParameter( args, "-max-servers", 20 ); + ContentDownloader.Config.MaxDownloads = GetParameter( args, "-max-downloads", 4 ); + string branch = GetParameter( args, "-branch" ) ?? GetParameter( args, "-beta" ) ?? "Public"; + bool forceDepot = HasParameter( args, "-force-depot" ); + string os = GetParameter( args, "-os", null ); + + if ( ContentDownloader.Config.DownloadAllPlatforms && !String.IsNullOrEmpty( os ) ) + { + Console.WriteLine( "Error: Cannot specify -os when -all-platforms is specified." ); + return; + } - ContentDownloader.Config.MaxServers = Math.Max(ContentDownloader.Config.MaxServers, ContentDownloader.Config.MaxDownloads); + ContentDownloader.Config.MaxServers = Math.Max( ContentDownloader.Config.MaxServers, ContentDownloader.Config.MaxDownloads ); - if (username != null && password == null) + if ( username != null && password == null && ( !ContentDownloader.Config.RememberPassword || !ConfigStore.TheConfig.LoginKeys.ContainsKey( username ) ) ) { - Console.Write("Enter account password for \"{0}\": ", username); + Console.Write( "Enter account password for \"{0}\": ", username ); password = Util.ReadPassword(); Console.WriteLine(); } - else if (username == null) + else if ( username == null ) { - Console.WriteLine("No username given. Using anonymous account with dedicated server subscription."); + Console.WriteLine( "No username given. Using anonymous account with dedicated server subscription." ); } - if (ContentDownloader.InitializeSteam3(username, password)) + // capture the supplied password in case we need to re-use it after checking the login key + ContentDownloader.Config.SuppliedPassword = password; + + if ( ContentDownloader.InitializeSteam3( username, password ) ) { - await ContentDownloader.DownloadAppAsync(appId, depotId, branch, forceDepot).ConfigureAwait(false); + await ContentDownloader.DownloadAppAsync( appId, depotId, branch, os, forceDepot ).ConfigureAwait( false ); ContentDownloader.ShutdownSteam3(); } } static int IndexOfParam( string[] args, string param ) { - for ( int x = 0 ; x < args.Length ; ++x ) + for ( int x = 0; x < args.Length; ++x ) { if ( args[ x ].Equals( param, StringComparison.OrdinalIgnoreCase ) ) return x; @@ -133,22 +144,22 @@ namespace DepotDownloader return IndexOfParam( args, param ) > -1; } - static T GetParameter(string[] args, string param, T defaultValue = default(T)) + static T GetParameter( string[] args, string param, T defaultValue = default( T ) ) { - int index = IndexOfParam(args, param); + int index = IndexOfParam( args, param ); - if (index == -1 || index == (args.Length - 1)) + if ( index == -1 || index == ( args.Length - 1 ) ) return defaultValue; - string strParam = args[index + 1]; + string strParam = args[ index + 1 ]; - var converter = TypeDescriptor.GetConverter(typeof(T)); - if( converter != null ) + var converter = TypeDescriptor.GetConverter( typeof( T ) ); + if ( converter != null ) { - return (T)converter.ConvertFromString(strParam); + return ( T )converter.ConvertFromString( strParam ); } - - return default(T); + + return default( T ); } static void PrintUsage() @@ -156,7 +167,7 @@ namespace DepotDownloader Console.WriteLine( "\nUsage: depotdownloader [optional parameters]\n" ); Console.WriteLine( "Parameters:" ); - Console.WriteLine("\t-app <#>\t\t\t\t- the AppID to download."); + Console.WriteLine( "\t-app <#>\t\t\t\t- the AppID to download." ); Console.WriteLine(); Console.WriteLine( "Optional Parameters:" ); @@ -164,7 +175,9 @@ namespace DepotDownloader Console.WriteLine( "\t-cellid <#>\t\t\t- the overridden CellID of the content server to download from." ); Console.WriteLine( "\t-username \t\t\t- the username of the account to login to for restricted content." ); Console.WriteLine( "\t-password \t\t\t- the password of the account to login to for restricted content." ); + Console.WriteLine( "\t-remember-password\t\t\t- if set, remember the password for subsequent logins of this user." ); Console.WriteLine( "\t-dir \t\t\t- the directory in which to place downloaded files." ); + Console.WriteLine( "\t-os \t\t\t- the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on)" ); Console.WriteLine( "\t-filelist \t\t- a list of files to download (from the manifest). Can optionally use regex to download only certain files." ); Console.WriteLine( "\t-all-platforms\t\t\t- downloads all platform-specific depots when -app is used." ); Console.WriteLine( "\t-manifest-only\t\t\t- downloads a human readable manifest for any depots that would be downloaded." ); diff --git a/DepotDownloader/Steam3Session.cs b/DepotDownloader/Steam3Session.cs index cb098007..7ed43228 100644 --- a/DepotDownloader/Steam3Session.cs +++ b/DepotDownloader/Steam3Session.cs @@ -86,29 +86,30 @@ namespace DepotDownloader this.steamUser = this.steamClient.GetHandler(); this.steamApps = this.steamClient.GetHandler(); - this.callbacks = new CallbackManager(this.steamClient); + this.callbacks = new CallbackManager( this.steamClient ); - this.callbacks.Subscribe(ConnectedCallback); - this.callbacks.Subscribe(DisconnectedCallback); - this.callbacks.Subscribe(LogOnCallback); - this.callbacks.Subscribe(SessionTokenCallback); - this.callbacks.Subscribe(LicenseListCallback); - this.callbacks.Subscribe(UpdateMachineAuthCallback); + this.callbacks.Subscribe( ConnectedCallback ); + this.callbacks.Subscribe( DisconnectedCallback ); + this.callbacks.Subscribe( LogOnCallback ); + this.callbacks.Subscribe( SessionTokenCallback ); + this.callbacks.Subscribe( LicenseListCallback ); + this.callbacks.Subscribe( UpdateMachineAuthCallback ); + this.callbacks.Subscribe( LoginKeyCallback ); Console.Write( "Connecting to Steam3..." ); if ( authenticatedUser ) { - FileInfo fi = new FileInfo(String.Format("{0}.sentryFile", logonDetails.Username)); - if (ConfigStore.TheConfig.SentryData != null && ConfigStore.TheConfig.SentryData.ContainsKey(logonDetails.Username)) + FileInfo fi = new FileInfo( String.Format( "{0}.sentryFile", logonDetails.Username ) ); + if ( ConfigStore.TheConfig.SentryData != null && ConfigStore.TheConfig.SentryData.ContainsKey( logonDetails.Username ) ) { - logonDetails.SentryFileHash = Util.SHAHash(ConfigStore.TheConfig.SentryData[logonDetails.Username]); + logonDetails.SentryFileHash = Util.SHAHash( ConfigStore.TheConfig.SentryData[ logonDetails.Username ] ); } - else if (fi.Exists && fi.Length > 0) + else if ( fi.Exists && fi.Length > 0 ) { - var sentryData = File.ReadAllBytes(fi.FullName); - logonDetails.SentryFileHash = Util.SHAHash(sentryData); - ConfigStore.TheConfig.SentryData[logonDetails.Username] = sentryData; + var sentryData = File.ReadAllBytes( fi.FullName ); + logonDetails.SentryFileHash = Util.SHAHash( sentryData ); + ConfigStore.TheConfig.SentryData[ logonDetails.Username ] = sentryData; ConfigStore.Save(); } } @@ -117,9 +118,9 @@ namespace DepotDownloader } public delegate bool WaitCondition(); - public bool WaitUntilCallback(Action submitter, WaitCondition waiter) + public bool WaitUntilCallback( Action submitter, WaitCondition waiter ) { - while (!bAborted && !waiter()) + while ( !bAborted && !waiter() ) { submitter(); @@ -128,7 +129,7 @@ namespace DepotDownloader { WaitForCallbacks(); } - while (!bAborted && this.seq == seq && !waiter()); + while ( !bAborted && this.seq == seq && !waiter() ); } return bAborted; @@ -136,224 +137,229 @@ namespace DepotDownloader public Credentials WaitForCredentials() { - if (credentials.IsValid || bAborted) + if ( credentials.IsValid || bAborted ) return credentials; - WaitUntilCallback(() => { }, () => { return credentials.IsValid; }); + WaitUntilCallback( () => { }, () => { return credentials.IsValid; } ); return credentials; } - public void RequestAppInfo(uint appId) + public void RequestAppInfo( uint appId ) { - if (AppInfo.ContainsKey(appId) || bAborted) + if ( AppInfo.ContainsKey( appId ) || bAborted ) return; bool completed = false; - Action cbMethodTokens = (appTokens) => + Action cbMethodTokens = ( appTokens ) => { completed = true; - if (appTokens.AppTokensDenied.Contains(appId)) + if ( appTokens.AppTokensDenied.Contains( appId ) ) { - Console.WriteLine("Insufficient privileges to get access token for app {0}", appId); + Console.WriteLine( "Insufficient privileges to get access token for app {0}", appId ); } - foreach (var token_dict in appTokens.AppTokens) + foreach ( var token_dict in appTokens.AppTokens ) { - this.AppTokens.Add(token_dict.Key, token_dict.Value); + this.AppTokens.Add( token_dict.Key, token_dict.Value ); } }; - WaitUntilCallback(() => { - callbacks.Subscribe(steamApps.PICSGetAccessTokens(new List() { appId }, new List() { }), cbMethodTokens); - }, () => { return completed; }); + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.PICSGetAccessTokens( new List() { appId }, new List() { } ), cbMethodTokens ); + }, () => { return completed; } ); completed = false; - Action cbMethod = (appInfo) => + Action cbMethod = ( appInfo ) => { completed = !appInfo.ResponsePending; - foreach (var app_value in appInfo.Apps) + foreach ( var app_value in appInfo.Apps ) { var app = app_value.Value; - Console.WriteLine("Got AppInfo for {0}", app.ID); - AppInfo.Add(app.ID, app); + Console.WriteLine( "Got AppInfo for {0}", app.ID ); + AppInfo.Add( app.ID, app ); } - foreach (var app in appInfo.UnknownApps) + foreach ( var app in appInfo.UnknownApps ) { - AppInfo.Add(app, null); + AppInfo.Add( app, null ); } }; - SteamApps.PICSRequest request = new SteamApps.PICSRequest(appId); - if (AppTokens.ContainsKey(appId)) + SteamApps.PICSRequest request = new SteamApps.PICSRequest( appId ); + if ( AppTokens.ContainsKey( appId ) ) { - request.AccessToken = AppTokens[appId]; + request.AccessToken = AppTokens[ appId ]; request.Public = false; } - WaitUntilCallback(() => { - callbacks.Subscribe(steamApps.PICSGetProductInfo(new List() { request }, new List() { }), cbMethod); - }, () => { return completed; }); + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.PICSGetProductInfo( new List() { request }, new List() { } ), cbMethod ); + }, () => { return completed; } ); } - public void RequestPackageInfo(IEnumerable packageIds) + public void RequestPackageInfo( IEnumerable packageIds ) { List packages = packageIds.ToList(); - packages.RemoveAll(pid => PackageInfo.ContainsKey(pid)); + packages.RemoveAll( pid => PackageInfo.ContainsKey( pid ) ); - if (packages.Count == 0 || bAborted) + if ( packages.Count == 0 || bAborted ) return; bool completed = false; - Action cbMethod = (packageInfo) => + Action cbMethod = ( packageInfo ) => { completed = !packageInfo.ResponsePending; - foreach (var package_value in packageInfo.Packages) + foreach ( var package_value in packageInfo.Packages ) { var package = package_value.Value; - PackageInfo.Add(package.ID, package); + PackageInfo.Add( package.ID, package ); } - foreach (var package in packageInfo.UnknownPackages) + foreach ( var package in packageInfo.UnknownPackages ) { - PackageInfo.Add(package, null); + PackageInfo.Add( package, null ); } }; - WaitUntilCallback(() => { - callbacks.Subscribe(steamApps.PICSGetProductInfo(new List(), packages), cbMethod); - }, () => { return completed; }); + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.PICSGetProductInfo( new List(), packages ), cbMethod ); + }, () => { return completed; } ); } - public bool RequestFreeAppLicense(uint appId) + public bool RequestFreeAppLicense( uint appId ) { bool success = false; bool completed = false; - Action cbMethod = (resultInfo) => + Action cbMethod = ( resultInfo ) => { completed = true; - success = resultInfo.GrantedApps.Contains(appId); + success = resultInfo.GrantedApps.Contains( appId ); }; - WaitUntilCallback(() => { - callbacks.Subscribe(steamApps.RequestFreeLicense(appId), cbMethod); - }, () => { return completed; }); + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.RequestFreeLicense( appId ), cbMethod ); + }, () => { return completed; } ); return success; } - public void RequestAppTicket(uint appId) + public void RequestAppTicket( uint appId ) { - if (AppTickets.ContainsKey(appId) || bAborted) + if ( AppTickets.ContainsKey( appId ) || bAborted ) return; if ( !authenticatedUser ) { - AppTickets[appId] = null; + AppTickets[ appId ] = null; return; } bool completed = false; - Action cbMethod = (appTicket) => + Action cbMethod = ( appTicket ) => { completed = true; - if (appTicket.Result != EResult.OK) + if ( appTicket.Result != EResult.OK ) { - Console.WriteLine("Unable to get appticket for {0}: {1}", appTicket.AppID, appTicket.Result); + Console.WriteLine( "Unable to get appticket for {0}: {1}", appTicket.AppID, appTicket.Result ); Abort(); } else { - Console.WriteLine("Got appticket for {0}!", appTicket.AppID); - AppTickets[appTicket.AppID] = appTicket.Ticket; + Console.WriteLine( "Got appticket for {0}!", appTicket.AppID ); + AppTickets[ appTicket.AppID ] = appTicket.Ticket; } }; - WaitUntilCallback(() => { - callbacks.Subscribe(steamApps.GetAppOwnershipTicket(appId), cbMethod); - }, () => { return completed; }); + WaitUntilCallback( () => + { + callbacks.Subscribe( steamApps.GetAppOwnershipTicket( appId ), cbMethod ); + }, () => { return completed; } ); } - public void RequestDepotKey(uint depotId, uint appid = 0) + public void RequestDepotKey( uint depotId, uint appid = 0 ) { - if (DepotKeys.ContainsKey(depotId) || bAborted) + if ( DepotKeys.ContainsKey( depotId ) || bAborted ) return; bool completed = false; - Action cbMethod = (depotKey) => + Action cbMethod = ( depotKey ) => { completed = true; - Console.WriteLine("Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result); + Console.WriteLine( "Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result ); - if (depotKey.Result != EResult.OK) + if ( depotKey.Result != EResult.OK ) { Abort(); return; } - DepotKeys[depotKey.DepotID] = depotKey.DepotKey; + DepotKeys[ depotKey.DepotID ] = depotKey.DepotKey; }; - WaitUntilCallback(() => + WaitUntilCallback( () => { - callbacks.Subscribe(steamApps.GetDepotDecryptionKey(depotId, appid), cbMethod); - }, () => { return completed; }); + callbacks.Subscribe( steamApps.GetDepotDecryptionKey( depotId, appid ), cbMethod ); + }, () => { return completed; } ); } - public void RequestCDNAuthToken(uint appid, uint depotid, string host) + public void RequestCDNAuthToken( uint appid, uint depotid, string host ) { - var cdnKey = string.Format("{0:D}:{1}", depotid, host); + var cdnKey = string.Format( "{0:D}:{1}", depotid, host ); - if (CDNAuthTokens.ContainsKey(cdnKey) || bAborted) + if ( CDNAuthTokens.ContainsKey( cdnKey ) || bAborted ) return; bool completed = false; - Action cbMethod = (cdnAuth) => + Action cbMethod = ( cdnAuth ) => { completed = true; - Console.WriteLine("Got CDN auth token for {0} result: {1} (expires {2})", host, cdnAuth.Result, cdnAuth.Expiration); + Console.WriteLine( "Got CDN auth token for {0} result: {1} (expires {2})", host, cdnAuth.Result, cdnAuth.Expiration ); - if (cdnAuth.Result != EResult.OK) + if ( cdnAuth.Result != EResult.OK ) { Abort(); return; } - CDNAuthTokens.TryAdd(cdnKey, cdnAuth); + CDNAuthTokens.TryAdd( cdnKey, cdnAuth ); }; - WaitUntilCallback(() => + WaitUntilCallback( () => { - callbacks.Subscribe(steamApps.GetCDNAuthToken(appid, depotid, host), cbMethod); - }, () => { return completed; }); + callbacks.Subscribe( steamApps.GetCDNAuthToken( appid, depotid, host ), cbMethod ); + }, () => { return completed; } ); } - public void CheckAppBetaPassword(uint appid, string password) + public void CheckAppBetaPassword( uint appid, string password ) { bool completed = false; - Action cbMethod = (appPassword) => + Action cbMethod = ( appPassword ) => { completed = true; - Console.WriteLine("Retrieved {0} beta keys with result: {1}", appPassword.BetaPasswords.Count, appPassword.Result); + Console.WriteLine( "Retrieved {0} beta keys with result: {1}", appPassword.BetaPasswords.Count, appPassword.Result ); - foreach (var entry in appPassword.BetaPasswords) + foreach ( var entry in appPassword.BetaPasswords ) { - AppBetaPasswords[entry.Key] = entry.Value; + AppBetaPasswords[ entry.Key ] = entry.Value; } }; - WaitUntilCallback(() => + WaitUntilCallback( () => { - callbacks.Subscribe(steamApps.CheckAppBetaPassword(appid, password), cbMethod); - }, () => { return completed; }); + callbacks.Subscribe( steamApps.CheckAppBetaPassword( appid, password ), cbMethod ); + }, () => { return completed; } ); } void Connect() @@ -368,48 +374,64 @@ namespace DepotDownloader this.steamClient.Connect(); } - private void Abort(bool sendLogOff=true) + private void Abort( bool sendLogOff = true ) { - Disconnect(sendLogOff); + Disconnect( sendLogOff ); } - public void Disconnect(bool sendLogOff=true) + public void Disconnect( bool sendLogOff = true ) { - if (sendLogOff) + if ( sendLogOff ) { steamUser.LogOff(); } - + steamClient.Disconnect(); bConnected = false; bConnecting = false; bAborted = true; // flush callbacks until our disconnected event - while (!bDidDisconnect) + while ( !bDidDisconnect ) { - callbacks.RunWaitAllCallbacks(TimeSpan.FromMilliseconds(100)); + callbacks.RunWaitAllCallbacks( TimeSpan.FromMilliseconds( 100 ) ); } } + public void TryWaitForLoginKey() + { + if ( logonDetails.Username == null || !ContentDownloader.Config.RememberPassword ) return; + + DateTime waitUntil = new DateTime().AddSeconds( 10 ); + + while ( true ) + { + DateTime now = new DateTime(); + if ( now >= waitUntil ) break; + + if ( ConfigStore.TheConfig.LoginKeys.ContainsKey( logonDetails.Username ) ) break; + + callbacks.RunWaitAllCallbacks( TimeSpan.FromMilliseconds( 100 ) ); + } + } private void WaitForCallbacks() { - callbacks.RunWaitCallbacks( TimeSpan.FromSeconds(1) ); + callbacks.RunWaitCallbacks( TimeSpan.FromSeconds( 1 ) ); TimeSpan diff = DateTime.Now - connectTime; - if (diff > STEAM3_TIMEOUT && !bConnected) + if ( diff > STEAM3_TIMEOUT && !bConnected ) { - Console.WriteLine("Timeout connecting to Steam3."); + Console.WriteLine( "Timeout connecting to Steam3." ); Abort(); return; } } - private void ConnectedCallback(SteamClient.ConnectedCallback connected) + private void ConnectedCallback( SteamClient.ConnectedCallback connected ) { - Console.WriteLine(" Done!"); + Console.WriteLine( " Done!" ); bConnecting = false; bConnected = true; if ( !authenticatedUser ) @@ -424,124 +446,147 @@ namespace DepotDownloader } } - private void DisconnectedCallback(SteamClient.DisconnectedCallback disconnected) + private void DisconnectedCallback( SteamClient.DisconnectedCallback disconnected ) { bDidDisconnect = true; - if (disconnected.UserInitiated || bExpectingDisconnectRemote) + if ( disconnected.UserInitiated || bExpectingDisconnectRemote ) { - Console.WriteLine("Disconnected from Steam"); + Console.WriteLine( "Disconnected from Steam" ); } - else if (connectionBackoff >= 10) + else if ( connectionBackoff >= 10 ) { - Console.WriteLine("Could not connect to Steam after 10 tries"); - Abort(false); + Console.WriteLine( "Could not connect to Steam after 10 tries" ); + Abort( false ); } - else if (!bAborted) + else if ( !bAborted ) { - if (bConnecting) + if ( bConnecting ) { - Console.WriteLine("Connection to Steam failed. Trying again"); - } else + Console.WriteLine( "Connection to Steam failed. Trying again" ); + } + else { - Console.WriteLine("Lost connection to Steam. Reconnecting"); + Console.WriteLine( "Lost connection to Steam. Reconnecting" ); } - - Thread.Sleep(1000 * ++connectionBackoff); + + Thread.Sleep( 1000 * ++connectionBackoff ); steamClient.Connect(); } } - private void LogOnCallback(SteamUser.LoggedOnCallback loggedOn) + private void LogOnCallback( SteamUser.LoggedOnCallback loggedOn ) { bool isSteamGuard = loggedOn.Result == EResult.AccountLogonDenied; bool is2FA = loggedOn.Result == EResult.AccountLoginDeniedNeedTwoFactor; + bool isLoginKey = ContentDownloader.Config.RememberPassword && logonDetails.LoginKey != null && loggedOn.Result == EResult.InvalidPassword; - if (isSteamGuard || is2FA) + if ( isSteamGuard || is2FA || isLoginKey ) { bExpectingDisconnectRemote = true; - Abort(false); + Abort( false ); - Console.WriteLine("This account is protected by Steam Guard."); + if ( !isLoginKey ) + { + Console.WriteLine( "This account is protected by Steam Guard." ); + } - if (is2FA) + if ( is2FA ) { - Console.Write("Please enter your 2 factor auth code from your authenticator app: "); + Console.Write( "Please enter your 2 factor auth code from your authenticator app: " ); logonDetails.TwoFactorCode = Console.ReadLine(); } + else if ( isLoginKey ) + { + ConfigStore.TheConfig.LoginKeys.Remove( logonDetails.Username ); + ConfigStore.Save(); + + logonDetails.LoginKey = null; + + if ( ContentDownloader.Config.SuppliedPassword != null ) + { + Console.WriteLine( "Login key was expired. Connecting with supplied password." ); + logonDetails.Password = ContentDownloader.Config.SuppliedPassword; + } + else + { + Console.WriteLine( "Login key was expired. Please enter your password: " ); + logonDetails.Password = Util.ReadPassword(); + } + } else { - Console.Write("Please enter the authentication code sent to your email address: "); + Console.Write( "Please enter the authentication code sent to your email address: " ); logonDetails.AuthCode = Console.ReadLine(); } - Console.Write("Retrying Steam3 connection..."); + Console.Write( "Retrying Steam3 connection..." ); Connect(); return; } - else if (loggedOn.Result == EResult.ServiceUnavailable) + else if ( loggedOn.Result == EResult.ServiceUnavailable ) { - Console.WriteLine("Unable to login to Steam3: {0}", loggedOn.Result); - Abort(false); + Console.WriteLine( "Unable to login to Steam3: {0}", loggedOn.Result ); + Abort( false ); return; } - else if (loggedOn.Result != EResult.OK) + else if ( loggedOn.Result != EResult.OK ) { - Console.WriteLine("Unable to login to Steam3: {0}", loggedOn.Result); + Console.WriteLine( "Unable to login to Steam3: {0}", loggedOn.Result ); Abort(); - + return; } - Console.WriteLine(" Done!"); + Console.WriteLine( " Done!" ); this.seq++; credentials.LoggedOn = true; - if (ContentDownloader.Config.CellID == 0) + if ( ContentDownloader.Config.CellID == 0 ) { - Console.WriteLine("Using Steam3 suggested CellID: " + loggedOn.CellID); - ContentDownloader.Config.CellID = (int)loggedOn.CellID; + Console.WriteLine( "Using Steam3 suggested CellID: " + loggedOn.CellID ); + ContentDownloader.Config.CellID = ( int )loggedOn.CellID; } } - private void SessionTokenCallback(SteamUser.SessionTokenCallback sessionToken) + private void SessionTokenCallback( SteamUser.SessionTokenCallback sessionToken ) { - Console.WriteLine("Got session token!"); + Console.WriteLine( "Got session token!" ); credentials.SessionToken = sessionToken.SessionToken; } - private void LicenseListCallback(SteamApps.LicenseListCallback licenseList) + private void LicenseListCallback( SteamApps.LicenseListCallback licenseList ) { - if (licenseList.Result != EResult.OK) + if ( licenseList.Result != EResult.OK ) { - Console.WriteLine("Unable to get license list: {0} ", licenseList.Result); + Console.WriteLine( "Unable to get license list: {0} ", licenseList.Result ); Abort(); return; } - Console.WriteLine("Got {0} licenses for account!", licenseList.LicenseList.Count); + Console.WriteLine( "Got {0} licenses for account!", licenseList.LicenseList.Count ); Licenses = licenseList.LicenseList; - IEnumerable licenseQuery = Licenses.Select(lic => + IEnumerable licenseQuery = Licenses.Select( lic => { return lic.PackageID; - }); + } ); - Console.WriteLine("Licenses: {0}", string.Join(", ", licenseQuery)); + Console.WriteLine( "Licenses: {0}", string.Join( ", ", licenseQuery ) ); } - private void UpdateMachineAuthCallback(SteamUser.UpdateMachineAuthCallback machineAuth) + private void UpdateMachineAuthCallback( SteamUser.UpdateMachineAuthCallback machineAuth ) { - byte[] hash = Util.SHAHash(machineAuth.Data); - Console.WriteLine("Got Machine Auth: {0} {1} {2} {3}", machineAuth.FileName, machineAuth.Offset, machineAuth.BytesToWrite, machineAuth.Data.Length, hash); + byte[] hash = Util.SHAHash( machineAuth.Data ); + Console.WriteLine( "Got Machine Auth: {0} {1} {2} {3}", machineAuth.FileName, machineAuth.Offset, machineAuth.BytesToWrite, machineAuth.Data.Length, hash ); - ConfigStore.TheConfig.SentryData[logonDetails.Username] = machineAuth.Data; + ConfigStore.TheConfig.SentryData[ logonDetails.Username ] = machineAuth.Data; ConfigStore.Save(); - + var authResponse = new SteamUser.MachineAuthDetails { BytesWritten = machineAuth.BytesToWrite, @@ -563,6 +608,16 @@ namespace DepotDownloader steamUser.SendMachineAuthResponse( authResponse ); } + private void LoginKeyCallback( SteamUser.LoginKeyCallback loginKey ) + { + Console.WriteLine( "Accepted new login key for account {0}", logonDetails.Username ); + + ConfigStore.TheConfig.LoginKeys[ logonDetails.Username ] = loginKey.LoginKey; + ConfigStore.Save(); + + steamUser.AcceptNewLoginKey( loginKey ); + } + } } diff --git a/README.md b/README.md index c7a3b11e..a716058d 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ Optional Parameters: -cellid <#> - the overridden CellID of the content server to download from. -username - the username of the account to login to for restricted content. -password - the password of the account to login to for restricted content. + -remember-password - if set, remember the password for subsequent logins of this user. -dir - the directory in which to place downloaded files. + -os - the operating system for which to download the game (windows, macos or linux, default: OS the program is currently running on) -filelist - a list of files to download (from the manifest). Can optionally use regex to download only certain files. -all-platforms - downloads all platform-specific depots when -app is used.