KnightsOfTheArk
/
DepotDownloader
mirror of https://github.com/SteamRE/DepotDownloader.git
23
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reformat code

Browse Source
pull/231/head
js6pak 4 years ago
parent f0dcb5f57d
commit 239d31441f
No known key found for this signature in database
GPG Key ID: EFE980EFD32F8EE9
11 changed files with 486 additions and 488 deletions
Show Stats Download Patch File Download Diff File

40
DepotDownloader/AccountSettingsStore.cs
Unescape Escape View File

@ -10,16 +10,16 @@ using SteamKit2.Discovery;
namespace DepotDownloader
{
[ProtoContract]
[ ProtoContract ]
class AccountSettingsStore
{
[ProtoMember(1, IsRequired=false)]
[ ProtoMember( 1, IsRequired = false ) ]
public Dictionary<string, byte[]> SentryData { get; private set; }
[ProtoMember(2, IsRequired = false)]
[ ProtoMember( 2, IsRequired = false ) ]
public System.Collections.Concurrent.ConcurrentDictionary<string, int> ContentServerPenalty { get; private set; }
[ProtoMember(3, IsRequired = false)]
[ ProtoMember( 3, IsRequired = false ) ]
public Dictionary<string, string> LoginKeys { get; private set; }
string FileName = null;
@ -39,24 +39,24 @@ namespace DepotDownloader
public static AccountSettingsStore Instance = null;
static readonly IsolatedStorageFile IsolatedStorage = IsolatedStorageFile.GetUserStoreForAssembly();
public static void LoadFromFile(string filename)
public static void LoadFromFile( string filename )
{
if (Loaded)
throw new Exception("Config already loaded");
if ( Loaded )
throw new Exception( "Config already loaded" );
if (IsolatedStorage.FileExists(filename))
if ( IsolatedStorage.FileExists( filename ) )
{
try
{
using (var fs = IsolatedStorage.OpenFile(filename, FileMode.Open, FileAccess.Read))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress))
using ( var fs = IsolatedStorage.OpenFile( filename, FileMode.Open, FileAccess.Read ) )
using ( DeflateStream ds = new DeflateStream( fs, CompressionMode.Decompress ) )
{
Instance = ProtoBuf.Serializer.Deserialize<AccountSettingsStore>(ds);
Instance = ProtoBuf.Serializer.Deserialize<AccountSettingsStore>( ds );
}
}
catch (IOException ex)
catch ( IOException ex )
{
Console.WriteLine("Failed to load account settings: {0}", ex.Message);
Console.WriteLine( "Failed to load account settings: {0}", ex.Message );
Instance = new AccountSettingsStore();
}
}
@ -70,20 +70,20 @@ namespace DepotDownloader
public static void Save()
{
if (!Loaded)
throw new Exception("Saved config before loading");
if ( !Loaded )
throw new Exception( "Saved config before loading" );
try
{
using (var fs = IsolatedStorage.OpenFile(Instance.FileName, FileMode.Create, FileAccess.Write))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Compress))
using ( var fs = IsolatedStorage.OpenFile( Instance.FileName, FileMode.Create, FileAccess.Write ) )
using ( DeflateStream ds = new DeflateStream( fs, CompressionMode.Compress ) )
{
ProtoBuf.Serializer.Serialize<AccountSettingsStore>(ds, Instance);
ProtoBuf.Serializer.Serialize<AccountSettingsStore>( ds, Instance );
}
}
catch (IOException ex)
catch ( IOException ex )
{
Console.WriteLine("Failed to save account settings: {0}", ex.Message);
Console.WriteLine( "Failed to save account settings: {0}", ex.Message );
}
}
}

94
DepotDownloader/CDNClientPool.cs
Unescape Escape View File

@ -29,19 +29,19 @@ namespace DepotDownloader
private readonly CancellationTokenSource shutdownToken;
public CancellationTokenSource ExhaustedToken { get; set; }
public CDNClientPool(Steam3Session steamSession, uint appId)
public CDNClientPool( Steam3Session steamSession, uint appId )
{
this.steamSession = steamSession;
this.appId = appId;
CDNClient = new CDNClient(steamSession.steamClient);
CDNClient = new CDNClient( steamSession.steamClient );
activeConnectionPool = new ConcurrentStack<CDNClient.Server>();
availableServerEndpoints = new BlockingCollection<CDNClient.Server>();
populatePoolEvent = new AutoResetEvent(true);
populatePoolEvent = new AutoResetEvent( true );
shutdownToken = new CancellationTokenSource();
monitorTask = Task.Factory.StartNew(ConnectionPoolMonitorAsync).Unwrap();
monitorTask = Task.Factory.StartNew( ConnectionPoolMonitorAsync ).Unwrap();
}
public void Shutdown()
@ -54,25 +54,25 @@ namespace DepotDownloader
{
var backoffDelay = 0;
while (!shutdownToken.IsCancellationRequested)
while ( !shutdownToken.IsCancellationRequested )
{
try
{
var cdnServers = await ContentServerDirectoryService.LoadAsync(this.steamSession.steamClient.Configuration, ContentDownloader.Config.CellID, shutdownToken.Token);
if (cdnServers != null)
var cdnServers = await ContentServerDirectoryService.LoadAsync( this.steamSession.steamClient.Configuration, ContentDownloader.Config.CellID, shutdownToken.Token );
if ( cdnServers != null )
{
return cdnServers;
}
}
catch (Exception ex)
catch ( Exception ex )
{
Console.WriteLine("Failed to retrieve content server list: {0}", ex.Message);
Console.WriteLine( "Failed to retrieve content server list: {0}", ex.Message );
if (ex is SteamKitWebRequestException e && e.StatusCode == (HttpStatusCode)429)
if ( ex is SteamKitWebRequestException e && e.StatusCode == ( HttpStatusCode )429 )
{
// If we're being throttled, add a delay to the next request
backoffDelay = Math.Min(5, ++backoffDelay);
await Task.Delay(TimeSpan.FromSeconds(backoffDelay));
backoffDelay = Math.Min( 5, ++backoffDelay );
await Task.Delay( TimeSpan.FromSeconds( backoffDelay ) );
}
}
}
@ -84,48 +84,48 @@ namespace DepotDownloader
{
bool didPopulate = false;
while (!shutdownToken.IsCancellationRequested)
while ( !shutdownToken.IsCancellationRequested )
{
populatePoolEvent.WaitOne(TimeSpan.FromSeconds(1));
populatePoolEvent.WaitOne( TimeSpan.FromSeconds( 1 ) );
// We want the Steam session so we can take the CellID from the session and pass it through to the ContentServer Directory Service
if (availableServerEndpoints.Count < ServerEndpointMinimumSize && steamSession.steamClient.IsConnected)
if ( availableServerEndpoints.Count < ServerEndpointMinimumSize && steamSession.steamClient.IsConnected )
{
var servers = await FetchBootstrapServerListAsync().ConfigureAwait(false);
var servers = await FetchBootstrapServerListAsync().ConfigureAwait( false );
if (servers == null || servers.Count == 0)
if ( servers == null || servers.Count == 0 )
{
ExhaustedToken?.Cancel();
return;
}
ProxyServer = servers.Where(x => x.UseAsProxy).FirstOrDefault();
ProxyServer = servers.Where( x => x.UseAsProxy ).FirstOrDefault();
var weightedCdnServers = servers
.Where(server =>
.Where( server =>
{
var isEligibleForApp = server.AllowedAppIds == null || server.AllowedAppIds.Contains(appId);
return isEligibleForApp && (server.Type == "SteamCache" || server.Type == "CDN");
})
.Select(server =>
var isEligibleForApp = server.AllowedAppIds == null || server.AllowedAppIds.Contains( appId );
return isEligibleForApp && ( server.Type == "SteamCache" || server.Type == "CDN" );
} )
.Select( server =>
{
AccountSettingsStore.Instance.ContentServerPenalty.TryGetValue(server.Host, out var penalty);
AccountSettingsStore.Instance.ContentServerPenalty.TryGetValue( server.Host, out var penalty );
return (server, penalty);
})
.OrderBy(pair => pair.penalty).ThenBy(pair => pair.server.WeightedLoad);
return ( server, penalty );
} )
.OrderBy( pair => pair.penalty ).ThenBy( pair => pair.server.WeightedLoad );
foreach (var (server, weight) in weightedCdnServers)
foreach ( var (server, weight) in weightedCdnServers )
{
for (var i = 0; i < server.NumEntries; i++)
for ( var i = 0; i < server.NumEntries; i++ )
{
availableServerEndpoints.Add(server);
availableServerEndpoints.Add( server );
}
}
didPopulate = true;
}
else if (availableServerEndpoints.Count == 0 && !steamSession.steamClient.IsConnected && didPopulate)
else if ( availableServerEndpoints.Count == 0 && !steamSession.steamClient.IsConnected && didPopulate )
{
ExhaustedToken?.Cancel();
return;
@ -133,54 +133,54 @@ namespace DepotDownloader
}
}
private CDNClient.Server BuildConnection(CancellationToken token)
private CDNClient.Server BuildConnection( CancellationToken token )
{
if (availableServerEndpoints.Count < ServerEndpointMinimumSize)
if ( availableServerEndpoints.Count < ServerEndpointMinimumSize )
{
populatePoolEvent.Set();
}
return availableServerEndpoints.Take(token);
return availableServerEndpoints.Take( token );
}
public CDNClient.Server GetConnection(CancellationToken token)
public CDNClient.Server GetConnection( CancellationToken token )
{
if (!activeConnectionPool.TryPop(out var connection))
if ( !activeConnectionPool.TryPop( out var connection ) )
{
connection = BuildConnection(token);
connection = BuildConnection( token );
}
return connection;
}
public async Task<string> AuthenticateConnection(uint appId, uint depotId, CDNClient.Server server)
public async Task<string> AuthenticateConnection( uint appId, uint depotId, CDNClient.Server server )
{
var host = steamSession.ResolveCDNTopLevelHost(server.Host);
var host = steamSession.ResolveCDNTopLevelHost( server.Host );
var cdnKey = $"{depotId:D}:{host}";
steamSession.RequestCDNAuthToken(appId, depotId, host, cdnKey);
steamSession.RequestCDNAuthToken( appId, depotId, host, cdnKey );
if (steamSession.CDNAuthTokens.TryGetValue(cdnKey, out var authTokenCallbackPromise))
if ( steamSession.CDNAuthTokens.TryGetValue( cdnKey, out var authTokenCallbackPromise ) )
{
var result = await authTokenCallbackPromise.Task;
return result.Token;
}
else
{
throw new Exception($"Failed to retrieve CDN token for server {server.Host} depot {depotId}");
throw new Exception( $"Failed to retrieve CDN token for server {server.Host} depot {depotId}" );
}
}
public void ReturnConnection(CDNClient.Server server)
public void ReturnConnection( CDNClient.Server server )
{
if (server == null) return;
if ( server == null ) return;
activeConnectionPool.Push(server);
activeConnectionPool.Push( server );
}
public void ReturnBrokenConnection(CDNClient.Server server)
public void ReturnBrokenConnection( CDNClient.Server server )
{
if (server == null) return;
if ( server == null ) return;
// Broken connections are not returned to the pool
}

488
DepotDownloader/ContentDownloader.cs
Unescape Escape View File

@ -15,7 +15,7 @@ namespace DepotDownloader
{
public class ContentDownloaderException : System.Exception
{
public ContentDownloaderException( String value ) : base( value ) {}
public ContentDownloaderException( String value ) : base( value ) { }
}
static class ContentDownloader
@ -95,12 +95,12 @@ namespace DepotDownloader
return true;
filename = filename.Replace( '\\', '/' );
if ( Config.FilesToDownload.Contains( filename ) )
{
return true;
}
foreach ( Regex rgx in Config.FilesToDownloadRegex )
{
Match m = rgx.Match( filename );
@ -217,7 +217,7 @@ namespace DepotDownloader
// Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680.
if ( depotChild[ "manifests" ] == KeyValue.Invalid && depotChild[ "depotfromapp" ] != KeyValue.Invalid )
{
uint otherAppId = depotChild["depotfromapp"].AsUnsignedInteger();
uint otherAppId = depotChild[ "depotfromapp" ].AsUnsignedInteger();
if ( otherAppId == appId )
{
// This shouldn't ever happen, but ya never know with Valve. Don't infinite loop.
@ -297,7 +297,6 @@ namespace DepotDownloader
Console.WriteLine( "Unhandled depot encryption for depotId {0}", depotId );
return INVALID_MANIFEST_ID;
}
}
return INVALID_MANIFEST_ID;
@ -369,7 +368,7 @@ namespace DepotDownloader
public static void ShutdownSteam3()
{
if (cdnPool != null)
if ( cdnPool != null )
{
cdnPool.Shutdown();
cdnPool = null;
@ -407,7 +406,7 @@ namespace DepotDownloader
if ( steam3.steamUser.SteamID.AccountType != EAccountType.AnonUser )
{
details = steam3.GetUGCDetails( ugcId );
}
}
else
{
Console.WriteLine( $"Unable to query UGC details for {ugcId} from an anonymous account" );
@ -457,17 +456,17 @@ namespace DepotDownloader
public static async Task DownloadAppAsync( uint appId, List<(uint depotId, ulong manifestId)> depotManifestIds, string branch, string os, string arch, string language, bool lv, bool isUgc )
{
cdnPool = new CDNClientPool(steam3, appId);
cdnPool = new CDNClientPool( steam3, appId );
// Load our configuration data containing the depots currently installed
string configPath = ContentDownloader.Config.InstallDirectory;
if (string.IsNullOrWhiteSpace(configPath))
if ( string.IsNullOrWhiteSpace( configPath ) )
{
configPath = DEFAULT_DOWNLOAD_DIR;
}
Directory.CreateDirectory(Path.Combine(configPath, CONFIG_DIR));
DepotConfigStore.LoadFromFile(Path.Combine(configPath, CONFIG_DIR, "depot.config"));
Directory.CreateDirectory( Path.Combine( configPath, CONFIG_DIR ) );
DepotConfigStore.LoadFromFile( Path.Combine( configPath, CONFIG_DIR, "depot.config" ) );
if ( steam3 != null )
steam3.RequestAppInfo( appId );
@ -495,7 +494,7 @@ namespace DepotDownloader
if ( isUgc )
{
var workshopDepot = depots["workshopdepot"].AsUnsignedInteger();
var workshopDepot = depots[ "workshopdepot" ].AsUnsignedInteger();
if ( workshopDepot != 0 && !depotIdsExpected.Contains( workshopDepot ) )
{
depotIdsExpected.Add( workshopDepot );
@ -528,34 +527,34 @@ namespace DepotDownloader
if ( depotConfig != KeyValue.Invalid )
{
if ( !Config.DownloadAllPlatforms &&
depotConfig["oslist"] != KeyValue.Invalid &&
!string.IsNullOrWhiteSpace( depotConfig["oslist"].Value ) )
depotConfig[ "oslist" ] != KeyValue.Invalid &&
!string.IsNullOrWhiteSpace( depotConfig[ "oslist" ].Value ) )
{
var oslist = depotConfig["oslist"].Value.Split( ',' );
var oslist = depotConfig[ "oslist" ].Value.Split( ',' );
if ( Array.IndexOf( oslist, os ?? Util.GetSteamOS() ) == -1 )
continue;
}
if ( depotConfig["osarch"] != KeyValue.Invalid &&
!string.IsNullOrWhiteSpace( depotConfig["osarch"].Value ) )
if ( depotConfig[ "osarch" ] != KeyValue.Invalid &&
!string.IsNullOrWhiteSpace( depotConfig[ "osarch" ].Value ) )
{
var depotArch = depotConfig["osarch"].Value;
var depotArch = depotConfig[ "osarch" ].Value;
if ( depotArch != ( arch ?? Util.GetSteamArch() ) )
continue;
}
if ( !Config.DownloadAllLanguages &&
depotConfig["language"] != KeyValue.Invalid &&
!string.IsNullOrWhiteSpace( depotConfig["language"].Value ) )
depotConfig[ "language" ] != KeyValue.Invalid &&
!string.IsNullOrWhiteSpace( depotConfig[ "language" ].Value ) )
{
var depotLang = depotConfig["language"].Value;
var depotLang = depotConfig[ "language" ].Value;
if ( depotLang != ( language ?? "english" ) )
continue;
}
if ( !lv &&
depotConfig["lowviolence"] != KeyValue.Invalid &&
depotConfig["lowviolence"].AsBoolean() )
depotConfig[ "lowviolence" ] != KeyValue.Invalid &&
depotConfig[ "lowviolence" ].AsBoolean() )
continue;
}
}
@ -566,6 +565,7 @@ namespace DepotDownloader
depotManifestIds.Add( ( id, ContentDownloader.INVALID_MANIFEST_ID ) );
}
}
if ( depotManifestIds.Count == 0 && !hasSpecificDepots )
{
throw new ContentDownloaderException( String.Format( "Couldn't find any depots to download for app {0}", appId ) );
@ -573,7 +573,7 @@ namespace DepotDownloader
else if ( depotIdsFound.Count < depotIdsExpected.Count )
{
var remainingDepotIds = depotIdsExpected.Except( depotIdsFound );
throw new ContentDownloaderException( String.Format( "Depot {0} not listed for app {1}", string.Join(", ", remainingDepotIds), appId ) );
throw new ContentDownloaderException( String.Format( "Depot {0} not listed for app {1}", string.Join( ", ", remainingDepotIds ), appId ) );
}
}
@ -613,19 +613,19 @@ namespace DepotDownloader
return null;
}
if (manifestId == INVALID_MANIFEST_ID)
if ( manifestId == INVALID_MANIFEST_ID )
{
manifestId = GetSteam3DepotManifest(depotId, appId, branch);
if (manifestId == INVALID_MANIFEST_ID && branch != "public")
manifestId = GetSteam3DepotManifest( depotId, appId, branch );
if ( manifestId == INVALID_MANIFEST_ID && branch != "public" )
{
Console.WriteLine("Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch);
Console.WriteLine( "Warning: Depot {0} does not have branch named \"{1}\". Trying public branch.", depotId, branch );
branch = "public";
manifestId = GetSteam3DepotManifest(depotId, appId, branch);
manifestId = GetSteam3DepotManifest( depotId, appId, branch );
}
if (manifestId == INVALID_MANIFEST_ID)
if ( manifestId == INVALID_MANIFEST_ID )
{
Console.WriteLine("Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName);
Console.WriteLine( "Depot {0} ({1}) missing public subsection or manifest section.", depotId, contentName );
return null;
}
}
@ -660,6 +660,7 @@ namespace DepotDownloader
OldChunk = oldChunk;
NewChunk = newChunk;
}
public ProtoManifest.ChunkData OldChunk { get; private set; }
public ProtoManifest.ChunkData NewChunk { get; private set; }
}
@ -694,27 +695,26 @@ namespace DepotDownloader
public ulong SizeDownloaded;
public ulong DepotBytesCompressed;
public ulong DepotBytesUncompressed;
}
private static async Task DownloadSteam3Async(uint appId, List<DepotDownloadInfo> depots)
private static async Task DownloadSteam3Async( uint appId, List<DepotDownloadInfo> depots )
{
CancellationTokenSource cts = new CancellationTokenSource();
cdnPool.ExhaustedToken = cts;
GlobalDownloadCounter downloadCounter = new GlobalDownloadCounter();
var depotsToDownload = new List<DepotFilesData>(depots.Count);
var depotsToDownload = new List<DepotFilesData>( depots.Count );
var allFileNamesAllDepots = new HashSet<String>();
// First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
foreach (var depot in depots)
foreach ( var depot in depots )
{
var depotFileData = await ProcessDepotManifestAndFiles(cts, appId, depot);
var depotFileData = await ProcessDepotManifestAndFiles( cts, appId, depot );
if (depotFileData != null)
if ( depotFileData != null )
{
depotsToDownload.Add(depotFileData);
allFileNamesAllDepots.UnionWith(depotFileData.allFileNames);
depotsToDownload.Add( depotFileData );
allFileNamesAllDepots.UnionWith( depotFileData.allFileNames );
}
cts.Token.ThrowIfCancellationRequested();
@ -722,112 +722,112 @@ namespace DepotDownloader
// If we're about to write all the files to the same directory, we will need to first de-duplicate any files by path
// This is in last-depot-wins order, from Steam or the list of depots supplied by the user
if (!string.IsNullOrWhiteSpace(ContentDownloader.Config.InstallDirectory) && depotsToDownload.Count > 0)
if ( !string.IsNullOrWhiteSpace( ContentDownloader.Config.InstallDirectory ) && depotsToDownload.Count > 0 )
{
var claimedFileNames = new HashSet<String>();
for (var i = depotsToDownload.Count - 1; i >= 0; i--)
for ( var i = depotsToDownload.Count - 1; i >= 0; i-- )
{
// For each depot, remove all files from the list that have been claimed by a later depot
depotsToDownload[i].filteredFiles.RemoveAll(file => claimedFileNames.Contains(file.FileName));
depotsToDownload[ i ].filteredFiles.RemoveAll( file => claimedFileNames.Contains( file.FileName ) );
claimedFileNames.UnionWith(depotsToDownload[i].allFileNames);
claimedFileNames.UnionWith( depotsToDownload[ i ].allFileNames );
}
}
foreach (var depotFileData in depotsToDownload)
foreach ( var depotFileData in depotsToDownload )
{
await DownloadSteam3AsyncDepotFiles(cts, appId, downloadCounter, depotFileData, allFileNamesAllDepots);
await DownloadSteam3AsyncDepotFiles( cts, appId, downloadCounter, depotFileData, allFileNamesAllDepots );
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots",
downloadCounter.TotalBytesCompressed, downloadCounter.TotalBytesUncompressed, depots.Count);
Console.WriteLine( "Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots",
downloadCounter.TotalBytesCompressed, downloadCounter.TotalBytesUncompressed, depots.Count );
}
private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts,
uint appId, DepotDownloadInfo depot)
private static async Task<DepotFilesData> ProcessDepotManifestAndFiles( CancellationTokenSource cts,
uint appId, DepotDownloadInfo depot )
{
DepotDownloadCounter depotCounter = new DepotDownloadCounter();
Console.WriteLine("Processing depot {0} - {1}", depot.id, depot.contentName);
Console.WriteLine( "Processing depot {0} - {1}", depot.id, depot.contentName );
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
string configDir = Path.Combine( depot.installDir, CONFIG_DIR );
ulong lastManifestId = INVALID_MANIFEST_ID;
DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue(depot.id, out lastManifestId);
DepotConfigStore.Instance.InstalledManifestIDs.TryGetValue( depot.id, out lastManifestId );
// In case we have an early exit, this will force equiv of verifyall next run.
DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = INVALID_MANIFEST_ID;
DepotConfigStore.Instance.InstalledManifestIDs[ depot.id ] = INVALID_MANIFEST_ID;
DepotConfigStore.Save();
if (lastManifestId != INVALID_MANIFEST_ID)
if ( lastManifestId != INVALID_MANIFEST_ID )
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, lastManifestId));
var oldManifestFileName = Path.Combine( configDir, string.Format( "{0}_{1}.bin", depot.id, lastManifestId ) );
if (File.Exists(oldManifestFileName))
if ( File.Exists( oldManifestFileName ) )
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(oldManifestFileName + ".sha");
expectedChecksum = File.ReadAllBytes( oldManifestFileName + ".sha" );
}
catch (IOException)
catch ( IOException )
{
expectedChecksum = null;
}
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out currentChecksum);
oldProtoManifest = ProtoManifest.LoadFromFile( oldManifestFileName, out currentChecksum );
if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
if ( expectedChecksum == null || !expectedChecksum.SequenceEqual( currentChecksum ) )
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.manifestId)
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
if ( lastManifestId != depot.manifestId )
Console.WriteLine( "Manifest {0} on disk did not match the expected checksum.", lastManifestId );
oldProtoManifest = null;
}
}
}
if (lastManifestId == depot.manifestId && oldProtoManifest != null)
if ( lastManifestId == depot.manifestId && oldProtoManifest != null )
{
newProtoManifest = oldProtoManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
Console.WriteLine( "Already have manifest {0} for depot {1}.", depot.manifestId, depot.id );
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.id, depot.manifestId));
if (newManifestFileName != null)
var newManifestFileName = Path.Combine( configDir, string.Format( "{0}_{1}.bin", depot.id, depot.manifestId ) );
if ( newManifestFileName != null )
{
byte[] expectedChecksum, currentChecksum;
try
{
expectedChecksum = File.ReadAllBytes(newManifestFileName + ".sha");
expectedChecksum = File.ReadAllBytes( newManifestFileName + ".sha" );
}
catch (IOException)
catch ( IOException )
{
expectedChecksum = null;
}
newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out currentChecksum);
newProtoManifest = ProtoManifest.LoadFromFile( newManifestFileName, out currentChecksum );
if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
if ( newProtoManifest != null && ( expectedChecksum == null || !expectedChecksum.SequenceEqual( currentChecksum ) ) )
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.manifestId);
Console.WriteLine( "Manifest {0} on disk did not match the expected checksum.", depot.manifestId );
newProtoManifest = null;
}
}
if (newProtoManifest != null)
if ( newProtoManifest != null )
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.manifestId, depot.id);
Console.WriteLine( "Already have manifest {0} for depot {1}.", depot.manifestId, depot.id );
}
else
{
Console.Write("Downloading depot manifest...");
Console.Write( "Downloading depot manifest..." );
DepotManifest depotManifest = null;
@ -839,55 +839,54 @@ namespace DepotDownloader
try
{
connection = cdnPool.GetConnection(cts.Token);
connection = cdnPool.GetConnection( cts.Token );
DebugLog.WriteLine("ContentDownloader", "Authenticating connection to {0}", connection);
var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);
DebugLog.WriteLine( "ContentDownloader", "Authenticating connection to {0}", connection );
var cdnToken = await cdnPool.AuthenticateConnection( appId, depot.id, connection );
DebugLog.WriteLine("ContentDownloader", "Downloading manifest {0} from {1} with {2}", depot.manifestId, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(depot.id, depot.manifestId,
connection, cdnToken, depot.depotKey, proxyServer: cdnPool.ProxyServer).ConfigureAwait(false);
DebugLog.WriteLine( "ContentDownloader", "Downloading manifest {0} from {1} with {2}", depot.manifestId, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy" );
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync( depot.id, depot.manifestId,
connection, cdnToken, depot.depotKey, proxyServer: cdnPool.ProxyServer ).ConfigureAwait( false );
cdnPool.ReturnConnection(connection);
cdnPool.ReturnConnection( connection );
}
catch (TaskCanceledException)
catch ( TaskCanceledException )
{
Console.WriteLine("Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId);
Console.WriteLine( "Connection timeout downloading depot manifest {0} {1}", depot.id, depot.manifestId );
}
catch (SteamKitWebRequestException e)
catch ( SteamKitWebRequestException e )
{
cdnPool.ReturnBrokenConnection(connection);
cdnPool.ReturnBrokenConnection( connection );
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
if ( e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden )
{
Console.WriteLine("Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
Console.WriteLine( "Encountered 401 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId );
break;
}
else if (e.StatusCode == HttpStatusCode.NotFound)
else if ( e.StatusCode == HttpStatusCode.NotFound )
{
Console.WriteLine("Encountered 404 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId);
Console.WriteLine( "Encountered 404 for depot manifest {0} {1}. Aborting.", depot.id, depot.manifestId );
break;
}
else
{
Console.WriteLine("Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode);
Console.WriteLine( "Encountered error downloading depot manifest {0} {1}: {2}", depot.id, depot.manifestId, e.StatusCode );
}
}
catch (OperationCanceledException)
catch ( OperationCanceledException )
{
break;
}
catch (Exception e)
catch ( Exception e )
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message);
cdnPool.ReturnBrokenConnection( connection );
Console.WriteLine( "Encountered error downloading manifest for depot {0} {1}: {2}", depot.id, depot.manifestId, e.Message );
}
}
while (depotManifest == null);
} while ( depotManifest == null );
if (depotManifest == null)
if ( depotManifest == null )
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
Console.WriteLine( "\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id );
cts.Cancel();
}
@ -896,51 +895,51 @@ namespace DepotDownloader
byte[] checksum;
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
newProtoManifest.SaveToFile(newManifestFileName, out checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
newProtoManifest = new ProtoManifest( depotManifest, depot.manifestId );
newProtoManifest.SaveToFile( newManifestFileName, out checksum );
File.WriteAllBytes( newManifestFileName + ".sha", checksum );
Console.WriteLine(" Done!");
Console.WriteLine( " Done!" );
}
}
newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));
newProtoManifest.Files.Sort( ( x, y ) => string.Compare( x.FileName, y.FileName, StringComparison.Ordinal ) );
Console.WriteLine("Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime);
Console.WriteLine( "Manifest {0} ({1})", depot.manifestId, newProtoManifest.CreationTime );
if (Config.DownloadManifestOnly)
if ( Config.DownloadManifestOnly )
{
DumpManifestToTextFile(depot, newProtoManifest);
DumpManifestToTextFile( depot, newProtoManifest );
return null;
}
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
string stagingDir = Path.Combine( depot.installDir, STAGING_DIR );
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var allFileNames = new HashSet<string>(filesAfterExclusions.Count);
var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where( f => TestIsFileIncluded( f.FileName ) ).ToList();
var allFileNames = new HashSet<string>( filesAfterExclusions.Count );
// Pre-process
filesAfterExclusions.ForEach(file =>
filesAfterExclusions.ForEach( file =>
{
allFileNames.Add(file.FileName);
allFileNames.Add( file.FileName );
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
var fileStagingPath = Path.Combine(stagingDir, file.FileName);
var fileFinalPath = Path.Combine( depot.installDir, file.FileName );
var fileStagingPath = Path.Combine( stagingDir, file.FileName );
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
if ( file.Flags.HasFlag( EDepotFileFlag.Directory ) )
{
Directory.CreateDirectory(fileFinalPath);
Directory.CreateDirectory(fileStagingPath);
Directory.CreateDirectory( fileFinalPath );
Directory.CreateDirectory( fileStagingPath );
}
else
{
// Some manifests don't explicitly include all necessary directories
Directory.CreateDirectory(Path.GetDirectoryName(fileFinalPath));
Directory.CreateDirectory(Path.GetDirectoryName(fileStagingPath));
Directory.CreateDirectory( Path.GetDirectoryName( fileFinalPath ) );
Directory.CreateDirectory( Path.GetDirectoryName( fileStagingPath ) );
depotCounter.CompleteDownloadSize += file.TotalSize;
}
});
} );
return new DepotFilesData
{
@ -954,70 +953,70 @@ namespace DepotDownloader
};
}
private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource cts, uint appId,
GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, HashSet<String> allFileNamesAllDepots)
private static async Task DownloadSteam3AsyncDepotFiles( CancellationTokenSource cts, uint appId,
GlobalDownloadCounter downloadCounter, DepotFilesData depotFilesData, HashSet<String> allFileNamesAllDepots )
{
var depot = depotFilesData.depotDownloadInfo;
var depotCounter = depotFilesData.depotCounter;
Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
Console.WriteLine( "Downloading depot {0} - {1}", depot.id, depot.contentName );
var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var files = depotFilesData.filteredFiles.Where( f => !f.Flags.HasFlag( EDepotFileFlag.Directory ) ).ToArray();
var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>();
await Util.InvokeAsync(
files.Select(file => new Func<Task>(async () =>
await Task.Run(() => DownloadSteam3AsyncDepotFile(cts, depotFilesData, file, networkChunkQueue)))),
files.Select( file => new Func<Task>( async () =>
await Task.Run( () => DownloadSteam3AsyncDepotFile( cts, depotFilesData, file, networkChunkQueue ) ) ) ),
maxDegreeOfParallelism: Config.MaxDownloads
);
await Util.InvokeAsync(
networkChunkQueue.Select(q => new Func<Task>(async () =>
await Task.Run(() => DownloadSteam3AsyncDepotFileChunk(cts, appId, downloadCounter, depotFilesData,
q.fileData, q.fileStreamData, q.chunk)))),
networkChunkQueue.Select( q => new Func<Task>( async () =>
await Task.Run( () => DownloadSteam3AsyncDepotFileChunk( cts, appId, downloadCounter, depotFilesData,
q.fileData, q.fileStreamData, q.chunk ) ) ) ),
maxDegreeOfParallelism: Config.MaxDownloads
);
// Check for deleted files if updating the depot.
if (depotFilesData.previousManifest != null)
if ( depotFilesData.previousManifest != null )
{
var previousFilteredFiles = depotFilesData.previousManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).Select(f => f.FileName).ToHashSet();
var previousFilteredFiles = depotFilesData.previousManifest.Files.AsParallel().Where( f => TestIsFileIncluded( f.FileName ) ).Select( f => f.FileName ).ToHashSet();
// Check if we are writing to a single output directory. If not, each depot folder is managed independently
if (string.IsNullOrWhiteSpace(ContentDownloader.Config.InstallDirectory))
if ( string.IsNullOrWhiteSpace( ContentDownloader.Config.InstallDirectory ) )
{
// Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names
previousFilteredFiles.ExceptWith(depotFilesData.allFileNames);
previousFilteredFiles.ExceptWith( depotFilesData.allFileNames );
}
else
{
// Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names across all depots being downloaded
previousFilteredFiles.ExceptWith(allFileNamesAllDepots);
previousFilteredFiles.ExceptWith( allFileNamesAllDepots );
}
foreach(var existingFileName in previousFilteredFiles)
foreach ( var existingFileName in previousFilteredFiles )
{
string fileFinalPath = Path.Combine(depot.installDir, existingFileName);
string fileFinalPath = Path.Combine( depot.installDir, existingFileName );
if (!File.Exists(fileFinalPath))
if ( !File.Exists( fileFinalPath ) )
continue;
File.Delete(fileFinalPath);
Console.WriteLine("Deleted {0}", fileFinalPath);
File.Delete( fileFinalPath );
Console.WriteLine( "Deleted {0}", fileFinalPath );
}
}
DepotConfigStore.Instance.InstalledManifestIDs[depot.id] = depot.manifestId;
DepotConfigStore.Instance.InstalledManifestIDs[ depot.id ] = depot.manifestId;
DepotConfigStore.Save();
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed);
Console.WriteLine( "Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, depotCounter.DepotBytesCompressed, depotCounter.DepotBytesUncompressed );
}
private static void DownloadSteam3AsyncDepotFile(
CancellationTokenSource cts,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue)
ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue )
{
cts.Token.ThrowIfCancellationRequested();
@ -1026,125 +1025,126 @@ namespace DepotDownloader
var depotDownloadCounter = depotFilesData.depotCounter;
var oldProtoManifest = depotFilesData.previousManifest;
string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
string fileFinalPath = Path.Combine( depot.installDir, file.FileName );
string fileStagingPath = Path.Combine( stagingDir, file.FileName );
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
if ( File.Exists( fileStagingPath ) )
{
File.Delete(fileStagingPath);
File.Delete( fileStagingPath );
}
FileStream fs = null;
List<ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
FileInfo fi = new FileInfo( fileFinalPath );
if ( !fi.Exists )
{
Console.WriteLine("Pre-allocating {0}", fileFinalPath);
Console.WriteLine( "Pre-allocating {0}", fileFinalPath );
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs = File.Create( fileFinalPath );
try
{
fs.SetLength((long)file.TotalSize);
fs.SetLength( ( long )file.TotalSize );
}
catch (IOException ex)
catch ( IOException ex )
{
throw new ContentDownloaderException(String.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
throw new ContentDownloaderException( String.Format( "Failed to allocate file {0}: {1}", fileFinalPath, ex.Message ) );
}
neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
neededChunks = new List<ProtoManifest.ChunkData>( file.Chunks );
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
if ( oldProtoManifest != null )
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
oldManifestFile = oldProtoManifest.Files.SingleOrDefault( f => f.FileName == file.FileName );
}
if (oldManifestFile != null)
if ( oldManifestFile != null )
{
neededChunks = new List<ProtoManifest.ChunkData>();
var hashMatches = oldManifestFile.FileHash.SequenceEqual(file.FileHash);
if (Config.VerifyAll || !hashMatches)
var hashMatches = oldManifestFile.FileHash.SequenceEqual( file.FileHash );
if ( Config.VerifyAll || !hashMatches )
{
// we have a version of this file, but it doesn't fully match what we want
if (Config.VerifyAll)
if ( Config.VerifyAll )
{
Console.WriteLine("Validating {0}", fileFinalPath);
Console.WriteLine( "Validating {0}", fileFinalPath );
}
var matchingChunks = new List<ChunkMatch>();
foreach (var chunk in file.Chunks)
foreach ( var chunk in file.Chunks )
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
var oldChunk = oldManifestFile.Chunks.FirstOrDefault( c => c.ChunkID.SequenceEqual( chunk.ChunkID ) );
if ( oldChunk != null )
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
matchingChunks.Add( new ChunkMatch( oldChunk, chunk ) );
}
else
{
neededChunks.Add(chunk);
neededChunks.Add( chunk );
}
}
var orderedChunks = matchingChunks.OrderBy(x => x.OldChunk.Offset);
var orderedChunks = matchingChunks.OrderBy( x => x.OldChunk.Offset );
var copyChunks = new List<ChunkMatch>();
using (var fsOld = File.Open(fileFinalPath, FileMode.Open))
using ( var fsOld = File.Open( fileFinalPath, FileMode.Open ) )
{
foreach (var match in orderedChunks)
foreach ( var match in orderedChunks )
{
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
fsOld.Seek( ( long )match.OldChunk.Offset, SeekOrigin.Begin );
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] tmp = new byte[ match.OldChunk.UncompressedLength ];
fsOld.Read( tmp, 0, tmp.Length );
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
byte[] adler = Util.AdlerHash( tmp );
if ( !adler.SequenceEqual( match.OldChunk.Checksum ) )
{
neededChunks.Add(match.NewChunk);
neededChunks.Add( match.NewChunk );
}
else
{
copyChunks.Add(match);
copyChunks.Add( match );
}
}
}
if (!hashMatches || neededChunks.Count > 0)
if ( !hashMatches || neededChunks.Count > 0 )
{
File.Move(fileFinalPath, fileStagingPath);
File.Move( fileFinalPath, fileStagingPath );
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
using ( var fsOld = File.Open( fileStagingPath, FileMode.Open ) )
{
fs = File.Open(fileFinalPath, FileMode.Create);
fs = File.Open( fileFinalPath, FileMode.Create );
try
{
fs.SetLength((long)file.TotalSize);
fs.SetLength( ( long )file.TotalSize );
}
catch (IOException ex)
catch ( IOException ex )
{
throw new ContentDownloaderException(String.Format("Failed to resize file to expected size {0}: {1}", fileFinalPath, ex.Message));
throw new ContentDownloaderException( String.Format( "Failed to resize file to expected size {0}: {1}", fileFinalPath, ex.Message ) );
}
foreach (var match in copyChunks)
foreach ( var match in copyChunks )
{
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
fsOld.Seek( ( long )match.OldChunk.Offset, SeekOrigin.Begin );
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] tmp = new byte[ match.OldChunk.UncompressedLength ];
fsOld.Read( tmp, 0, tmp.Length );
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
fs.Seek( ( long )match.NewChunk.Offset, SeekOrigin.Begin );
fs.Write( tmp, 0, tmp.Length );
}
}
File.Delete(fileStagingPath);
File.Delete( fileStagingPath );
}
}
}
@ -1152,39 +1152,39 @@ namespace DepotDownloader
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open);
if ((ulong)fi.Length != file.TotalSize)
fs = File.Open( fileFinalPath, FileMode.Open );
if ( ( ulong )fi.Length != file.TotalSize )
{
try
{
fs.SetLength((long)file.TotalSize);
fs.SetLength( ( long )file.TotalSize );
}
catch (IOException ex)
catch ( IOException ex )
{
throw new ContentDownloaderException(String.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
throw new ContentDownloaderException( String.Format( "Failed to allocate file {0}: {1}", fileFinalPath, ex.Message ) );
}
}
Console.WriteLine("Validating {0}", fileFinalPath);
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
Console.WriteLine( "Validating {0}", fileFinalPath );
neededChunks = Util.ValidateSteam3FileChecksums( fs, file.Chunks.OrderBy( x => x.Offset ).ToArray() );
}
if (neededChunks.Count() == 0)
if ( neededChunks.Count() == 0 )
{
lock (depotDownloadCounter)
lock ( depotDownloadCounter )
{
depotDownloadCounter.SizeDownloaded += (ulong)file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", ((float)depotDownloadCounter.SizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath);
depotDownloadCounter.SizeDownloaded += ( ulong )file.TotalSize;
Console.WriteLine( "{0,6:#00.00}% {1}", ( ( float )depotDownloadCounter.SizeDownloaded / ( float )depotDownloadCounter.CompleteDownloadSize ) * 100.0f, fileFinalPath );
}
if (fs != null)
if ( fs != null )
fs.Dispose();
return;
}
else
{
var sizeOnDisk = (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum());
lock (depotDownloadCounter)
var sizeOnDisk = ( file.TotalSize - ( ulong )neededChunks.Select( x => ( long )x.UncompressedLength ).Sum() );
lock ( depotDownloadCounter )
{
depotDownloadCounter.SizeDownloaded += sizeOnDisk;
}
@ -1194,13 +1194,13 @@ namespace DepotDownloader
FileStreamData fileStreamData = new FileStreamData
{
fileStream = fs,
fileLock = new SemaphoreSlim(1),
fileLock = new SemaphoreSlim( 1 ),
chunksToDownload = neededChunks.Count
};
foreach (var chunk in neededChunks)
foreach ( var chunk in neededChunks )
{
networkChunkQueue.Enqueue((fileStreamData, file, chunk));
networkChunkQueue.Enqueue( ( fileStreamData, file, chunk ) );
}
}
@ -1208,16 +1208,16 @@ namespace DepotDownloader
CancellationTokenSource cts, uint appId,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
FileStreamData fileStreamData,
ProtoManifest.ChunkData chunk)
ProtoManifest.FileData file,
FileStreamData fileStreamData,
ProtoManifest.ChunkData chunk )
{
cts.Token.ThrowIfCancellationRequested();
var depot = depotFilesData.depotDownloadInfo;
var depotDownloadCounter = depotFilesData.depotCounter;
string chunkID = Util.EncodeHexString(chunk.ChunkID);
string chunkID = Util.EncodeHexString( chunk.ChunkID );
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
@ -1236,50 +1236,49 @@ namespace DepotDownloader
try
{
connection = cdnPool.GetConnection(cts.Token);
connection = cdnPool.GetConnection( cts.Token );
DebugLog.WriteLine("ContentDownloader", "Authenticating connection to {0}", connection);
var cdnToken = await cdnPool.AuthenticateConnection(appId, depot.id, connection);
DebugLog.WriteLine( "ContentDownloader", "Authenticating connection to {0}", connection );
var cdnToken = await cdnPool.AuthenticateConnection( appId, depot.id, connection );
DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync(depot.id, data,
connection, cdnToken, depot.depotKey, proxyServer: cdnPool.ProxyServer).ConfigureAwait(false);
DebugLog.WriteLine( "ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy" );
chunkData = await cdnPool.CDNClient.DownloadDepotChunkAsync( depot.id, data,
connection, cdnToken, depot.depotKey, proxyServer: cdnPool.ProxyServer ).ConfigureAwait( false );
cdnPool.ReturnConnection(connection);
cdnPool.ReturnConnection( connection );
}
catch (TaskCanceledException)
catch ( TaskCanceledException )
{
Console.WriteLine("Connection timeout downloading chunk {0}", chunkID);
Console.WriteLine( "Connection timeout downloading chunk {0}", chunkID );
}
catch (SteamKitWebRequestException e)
catch ( SteamKitWebRequestException e )
{
cdnPool.ReturnBrokenConnection(connection);
cdnPool.ReturnBrokenConnection( connection );
if (e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden)
if ( e.StatusCode == HttpStatusCode.Unauthorized || e.StatusCode == HttpStatusCode.Forbidden )
{
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
Console.WriteLine( "Encountered 401 for chunk {0}. Aborting.", chunkID );
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode);
Console.WriteLine( "Encountered error downloading chunk {0}: {1}", chunkID, e.StatusCode );
}
}
catch (OperationCanceledException)
catch ( OperationCanceledException )
{
break;
}
catch (Exception e)
catch ( Exception e )
{
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
cdnPool.ReturnBrokenConnection( connection );
Console.WriteLine( "Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message );
}
}
while (chunkData == null);
} while ( chunkData == null );
if (chunkData == null)
if ( chunkData == null )
{
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
Console.WriteLine( "Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id );
cts.Cancel();
}
@ -1288,44 +1287,43 @@ namespace DepotDownloader
try
{
await fileStreamData.fileLock.WaitAsync().ConfigureAwait(false);
await fileStreamData.fileLock.WaitAsync().ConfigureAwait( false );
fileStreamData.fileStream.Seek((long)chunkData.ChunkInfo.Offset, SeekOrigin.Begin);
await fileStreamData.fileStream.WriteAsync(chunkData.Data, 0, chunkData.Data.Length);
fileStreamData.fileStream.Seek( ( long )chunkData.ChunkInfo.Offset, SeekOrigin.Begin );
await fileStreamData.fileStream.WriteAsync( chunkData.Data, 0, chunkData.Data.Length );
}
finally
{
fileStreamData.fileLock.Release();
}
int remainingChunks = Interlocked.Decrement(ref fileStreamData.chunksToDownload);
if (remainingChunks == 0)
int remainingChunks = Interlocked.Decrement( ref fileStreamData.chunksToDownload );
if ( remainingChunks == 0 )
{
fileStreamData.fileStream.Dispose();
fileStreamData.fileLock.Dispose();
}
ulong sizeDownloaded = 0;
lock (depotDownloadCounter)
lock ( depotDownloadCounter )
{
sizeDownloaded = depotDownloadCounter.SizeDownloaded + (ulong)chunkData.Data.Length;
sizeDownloaded = depotDownloadCounter.SizeDownloaded + ( ulong )chunkData.Data.Length;
depotDownloadCounter.SizeDownloaded = sizeDownloaded;
depotDownloadCounter.DepotBytesCompressed += chunk.CompressedLength;
depotDownloadCounter.DepotBytesUncompressed += chunk.UncompressedLength;
}
lock (downloadCounter)
lock ( downloadCounter )
{
downloadCounter.TotalBytesCompressed += chunk.CompressedLength;
downloadCounter.TotalBytesUncompressed += chunk.UncompressedLength;
}
if (remainingChunks == 0)
if ( remainingChunks == 0 )
{
var fileFinalPath = Path.Combine(depot.installDir, file.FileName);
Console.WriteLine("{0,6:#00.00}% {1}", ((float)sizeDownloaded / (float)depotDownloadCounter.CompleteDownloadSize) * 100.0f, fileFinalPath);
var fileFinalPath = Path.Combine( depot.installDir, file.FileName );
Console.WriteLine( "{0,6:#00.00}% {1}", ( ( float )sizeDownloaded / ( float )depotDownloadCounter.CompleteDownloadSize ) * 100.0f, fileFinalPath );
}
}
static void DumpManifestToTextFile( DepotDownloadInfo depot, ProtoManifest manifest )

28
DepotDownloader/DepotConfigStore.cs
Unescape Escape View File

@ -6,10 +6,10 @@ using System.IO.Compression;
namespace DepotDownloader
{
[ProtoContract]
[ ProtoContract ]
class DepotConfigStore
{
[ProtoMember(1)]
[ ProtoMember( 1 ) ]
public Dictionary<uint, ulong> InstalledManifestIDs { get; private set; }
string FileName = null;
@ -26,16 +26,16 @@ namespace DepotDownloader
public static DepotConfigStore Instance = null;
public static void LoadFromFile(string filename)
public static void LoadFromFile( string filename )
{
if (Loaded)
throw new Exception("Config already loaded");
if ( Loaded )
throw new Exception( "Config already loaded" );
if (File.Exists(filename))
if ( File.Exists( filename ) )
{
using (FileStream fs = File.Open(filename, FileMode.Open))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress))
Instance = ProtoBuf.Serializer.Deserialize<DepotConfigStore>(ds);
using ( FileStream fs = File.Open( filename, FileMode.Open ) )
using ( DeflateStream ds = new DeflateStream( fs, CompressionMode.Decompress ) )
Instance = ProtoBuf.Serializer.Deserialize<DepotConfigStore>( ds );
}
else
{
@ -47,12 +47,12 @@ namespace DepotDownloader
public static void Save()
{
if (!Loaded)
throw new Exception("Saved config before loading");
if ( !Loaded )
throw new Exception( "Saved config before loading" );
using (FileStream fs = File.Open(Instance.FileName, FileMode.Create))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Compress))
ProtoBuf.Serializer.Serialize<DepotConfigStore>(ds, Instance);
using ( FileStream fs = File.Open( Instance.FileName, FileMode.Create ) )
using ( DeflateStream ds = new DeflateStream( fs, CompressionMode.Compress ) )
ProtoBuf.Serializer.Serialize<DepotConfigStore>( ds, Instance );
}
}
}

2
DepotDownloader/DownloadConfig.cs
Unescape Escape View File

@ -26,6 +26,6 @@ namespace DepotDownloader
public bool RememberPassword { get; set; }
// A Steam LoginID to allow multiple concurrent connections
public uint? LoginID {get; set; }
public uint? LoginID { get; set; }
}
}

16
DepotDownloader/HttpClientFactory.cs
Unescape Escape View File

@ -17,29 +17,29 @@ namespace DepotDownloader
{
public static HttpClient CreateHttpClient()
{
var client = new HttpClient(new SocketsHttpHandler
var client = new HttpClient( new SocketsHttpHandler
{
ConnectCallback = IPv4ConnectAsync
});
} );
var assemblyVersion = typeof(HttpClientFactory).Assembly.GetName().Version.ToString(fieldCount: 3);
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("DepotDownloader", assemblyVersion));
var assemblyVersion = typeof(HttpClientFactory).Assembly.GetName().Version.ToString( fieldCount: 3 );
client.DefaultRequestHeaders.UserAgent.Add( new ProductInfoHeaderValue( "DepotDownloader", assemblyVersion ) );
return client;
}
static async ValueTask<Stream> IPv4ConnectAsync(SocketsHttpConnectionContext context, CancellationToken cancellationToken)
static async ValueTask<Stream> IPv4ConnectAsync( SocketsHttpConnectionContext context, CancellationToken cancellationToken )
{
// By default, we create dual-mode sockets:
// Socket socket = new Socket(SocketType.Stream, ProtocolType.Tcp);
Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
Socket socket = new Socket( AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp );
socket.NoDelay = true;
try
{
await socket.ConnectAsync(context.DnsEndPoint, cancellationToken).ConfigureAwait(false);
return new NetworkStream(socket, ownsSocket: true);
await socket.ConnectAsync( context.DnsEndPoint, cancellationToken ).ConfigureAwait( false );
return new NetworkStream( socket, ownsSocket: true );
}
catch
{

34
DepotDownloader/HttpDiagnosticEventListener.cs
Unescape Escape View File

@ -6,37 +6,37 @@ namespace DepotDownloader
{
internal sealed class HttpDiagnosticEventListener : EventListener
{
public const EventKeywords TasksFlowActivityIds = (EventKeywords)0x80;
public const EventKeywords TasksFlowActivityIds = ( EventKeywords )0x80;
protected override void OnEventSourceCreated(EventSource eventSource)
protected override void OnEventSourceCreated( EventSource eventSource )
{
if (eventSource.Name == "System.Net.Http" ||
eventSource.Name == "System.Net.Sockets" ||
eventSource.Name == "System.Net.Security" ||
eventSource.Name == "System.Net.NameResolution")
if ( eventSource.Name == "System.Net.Http" ||
eventSource.Name == "System.Net.Sockets" ||
eventSource.Name == "System.Net.Security" ||
eventSource.Name == "System.Net.NameResolution" )
{
EnableEvents(eventSource, EventLevel.LogAlways);
EnableEvents( eventSource, EventLevel.LogAlways );
}
else if (eventSource.Name == "System.Threading.Tasks.TplEventSource")
else if ( eventSource.Name == "System.Threading.Tasks.TplEventSource" )
{
EnableEvents(eventSource, EventLevel.LogAlways, TasksFlowActivityIds);
EnableEvents( eventSource, EventLevel.LogAlways, TasksFlowActivityIds );
}
}
protected override void OnEventWritten(EventWrittenEventArgs eventData)
protected override void OnEventWritten( EventWrittenEventArgs eventData )
{
var sb = new StringBuilder().Append($"{eventData.TimeStamp:HH:mm:ss.fffffff} {eventData.EventSource.Name}.{eventData.EventName}(");
for (int i = 0; i < eventData.Payload?.Count; i++)
var sb = new StringBuilder().Append( $"{eventData.TimeStamp:HH:mm:ss.fffffff} {eventData.EventSource.Name}.{eventData.EventName}(" );
for ( int i = 0; i < eventData.Payload?.Count; i++ )
{
sb.Append(eventData.PayloadNames?[i]).Append(": ").Append(eventData.Payload[i]);
if (i < eventData.Payload?.Count - 1)
sb.Append( eventData.PayloadNames?[ i ] ).Append( ": " ).Append( eventData.Payload[ i ] );
if ( i < eventData.Payload?.Count - 1 )
{
sb.Append(", ");
sb.Append( ", " );
}
}
sb.Append(")");
Console.WriteLine(sb.ToString());
sb.Append( ")" );
Console.WriteLine( sb.ToString() );
}
}
}

47
DepotDownloader/Program.cs
Unescape Escape View File

@ -34,7 +34,7 @@ namespace DepotDownloader
DebugLog.AddListener( ( category, message ) =>
{
Console.WriteLine( "[{0}] {1}", category, message );
});
} );
var httpEventListener = new HttpDiagnosticEventListener();
}
@ -65,7 +65,7 @@ namespace DepotDownloader
ContentDownloader.Config.UsingFileList = true;
ContentDownloader.Config.FilesToDownload = new HashSet<string>( StringComparer.OrdinalIgnoreCase );
ContentDownloader.Config.FilesToDownloadRegex = new List<Regex>();
foreach ( var fileEntry in files )
{
if ( fileEntry.StartsWith( "regex:" ) )
@ -81,7 +81,7 @@ namespace DepotDownloader
Console.WriteLine( "Using filelist: '{0}'.", fileList );
}
catch (Exception ex)
catch ( Exception ex )
{
Console.WriteLine( "Warning: Unable to load filelist: {0}", ex.ToString() );
}
@ -93,7 +93,7 @@ namespace DepotDownloader
ContentDownloader.Config.MaxServers = GetParameter<int>( args, "-max-servers", 20 );
ContentDownloader.Config.MaxDownloads = GetParameter<int>( args, "-max-downloads", 8 );
ContentDownloader.Config.MaxServers = Math.Max( ContentDownloader.Config.MaxServers, ContentDownloader.Config.MaxDownloads );
ContentDownloader.Config.LoginID = HasParameter( args, "-loginid" ) ? (uint?)GetParameter<uint>( args, "-loginid" ) : null;
ContentDownloader.Config.LoginID = HasParameter( args, "-loginid" ) ? ( uint? )GetParameter<uint>( args, "-loginid" ) : null;
#endregion
@ -188,7 +188,7 @@ namespace DepotDownloader
if ( ContentDownloader.Config.DownloadAllPlatforms && !String.IsNullOrEmpty( os ) )
{
Console.WriteLine("Error: Cannot specify -os when -all-platforms is specified.");
Console.WriteLine( "Error: Cannot specify -os when -all-platforms is specified." );
return 1;
}
@ -257,7 +257,7 @@ namespace DepotDownloader
#endregion
}
return 0;
}
@ -277,6 +277,7 @@ namespace DepotDownloader
// Avoid console echoing of password
password = Util.ReadPassword();
}
Console.WriteLine();
} while ( String.Empty == password );
}
@ -298,14 +299,16 @@ namespace DepotDownloader
if ( args[ x ].Equals( param, StringComparison.OrdinalIgnoreCase ) )
return x;
}
return -1;
}
static bool HasParameter( string[] args, string param )
{
return IndexOfParam( args, param ) > -1;
}
static T GetParameter<T>( string[] args, string param, T defaultValue = default( T ) )
static T GetParameter<T>( string[] args, string param, T defaultValue = default(T) )
{
int index = IndexOfParam( args, param );
@ -314,35 +317,35 @@ namespace DepotDownloader
string strParam = args[ index + 1 ];
var converter = TypeDescriptor.GetConverter( typeof( T ) );
var converter = TypeDescriptor.GetConverter( typeof(T) );
if ( converter != null )
{
return ( T )converter.ConvertFromString( strParam );
}
return default( T );
return default(T);
}
static List<T> GetParameterList<T>(string[] args, string param)
static List<T> GetParameterList<T>( string[] args, string param )
{
List<T> list = new List<T>();
int index = IndexOfParam(args, param);
int index = IndexOfParam( args, param );
if (index == -1 || index == (args.Length - 1))
if ( index == -1 || index == ( args.Length - 1 ) )
return list;
index++;
while (index < args.Length)
while ( index < args.Length )
{
string strParam = args[index];
string strParam = args[ index ];
if (strParam[0] == '-') break;
if ( strParam[ 0 ] == '-' ) break;
var converter = TypeDescriptor.GetConverter(typeof(T));
if (converter != null)
var converter = TypeDescriptor.GetConverter( typeof(T) );
if ( converter != null )
{
list.Add((T)converter.ConvertFromString(strParam));
list.Add( ( T )converter.ConvertFromString( strParam ) );
}
index++;
@ -358,10 +361,10 @@ namespace DepotDownloader
Console.WriteLine( "\tdepotdownloader -app <id> [-depot <id> [-manifest <id>]]" );
Console.WriteLine( "\t\t[-username <username> [-password <password>]] [other options]" );
Console.WriteLine();
Console.WriteLine("Usage - downloading a workshop item using pubfile id");
Console.WriteLine( "Usage - downloading a workshop item using pubfile id" );
Console.WriteLine( "\tdepotdownloader -app <id> -pubfile <id> [-username <username> [-password <password>]]" );
Console.WriteLine("Usage - downloading a workshop item using ugc id");
Console.WriteLine("\tdepotdownloader -app <id> -ugc <id> [-username <username> [-password <password>]]");
Console.WriteLine( "Usage - downloading a workshop item using ugc id" );
Console.WriteLine( "\tdepotdownloader -app <id> -ugc <id> [-username <username> [-password <password>]]" );
Console.WriteLine();
Console.WriteLine( "Parameters:" );
Console.WriteLine( "\t-app <#>\t\t\t\t- the AppID to download." );
@ -379,7 +382,7 @@ namespace DepotDownloader
Console.WriteLine( "\t-ugc <#>\t\t\t\t- the UGC ID to download." );
Console.WriteLine( "\t-pubfile <#>\t\t\t- the PublishedFileId to download. (Will automatically resolve to UGC id)" );
Console.WriteLine();
Console.WriteLine( "\t-username <user>\t\t- the username of the account to login to for restricted content.");
Console.WriteLine( "\t-username <user>\t\t- the username of the account to login to for restricted content." );
Console.WriteLine( "\t-password <pass>\t\t- the password of the account to login to for restricted content." );
Console.WriteLine( "\t-remember-password\t\t- if set, remember the password for subsequent logins of this user." );
Console.WriteLine();

78
DepotDownloader/ProtoManifest.cs
Unescape Escape View File

@ -2,13 +2,12 @@
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using ProtoBuf;
using SteamKit2;
namespace DepotDownloader
{
[ProtoContract()]
[ ProtoContract() ]
class ProtoManifest
{
// Proto ctor
@ -17,14 +16,14 @@ namespace DepotDownloader
Files = new List<FileData>();
}
public ProtoManifest(DepotManifest sourceManifest, ulong id) : this()
public ProtoManifest( DepotManifest sourceManifest, ulong id ) : this()
{
sourceManifest.Files.ForEach(f => Files.Add(new FileData(f)));
sourceManifest.Files.ForEach( f => Files.Add( new FileData( f ) ) );
ID = id;
CreationTime = sourceManifest.CreationTime;
}
[ProtoContract()]
[ ProtoContract() ]
public class FileData
{
// Proto ctor
@ -33,47 +32,47 @@ namespace DepotDownloader
Chunks = new List<ChunkData>();
}
public FileData(DepotManifest.FileData sourceData) : this()
public FileData( DepotManifest.FileData sourceData ) : this()
{
FileName = sourceData.FileName;
sourceData.Chunks.ForEach(c => Chunks.Add(new ChunkData(c)));
sourceData.Chunks.ForEach( c => Chunks.Add( new ChunkData( c ) ) );
Flags = sourceData.Flags;
TotalSize = sourceData.TotalSize;
FileHash = sourceData.FileHash;
}
[ProtoMember(1)]
[ ProtoMember( 1 ) ]
public string FileName { get; internal set; }
/// <summary>
/// Gets the chunks that this file is composed of.
/// </summary>
[ProtoMember(2)]
[ ProtoMember( 2 ) ]
public List<ChunkData> Chunks { get; private set; }
/// <summary>
/// Gets the file flags
/// </summary>
[ProtoMember(3)]
[ ProtoMember( 3 ) ]
public EDepotFileFlag Flags { get; private set; }
/// <summary>
/// Gets the total size of this file.
/// </summary>
[ProtoMember(4)]
[ ProtoMember( 4 ) ]
public ulong TotalSize { get; private set; }
/// <summary>
/// Gets the hash of this file.
/// </summary>
[ProtoMember(5)]
[ ProtoMember( 5 ) ]
public byte[] FileHash { get; private set; }
}
[ProtoContract(SkipConstructor = true)]
[ ProtoContract( SkipConstructor = true ) ]
public class ChunkData
{
public ChunkData(DepotManifest.ChunkData sourceChunk)
public ChunkData( DepotManifest.ChunkData sourceChunk )
{
ChunkID = sourceChunk.ChunkID;
Checksum = sourceChunk.Checksum;
@ -85,78 +84,77 @@ namespace DepotDownloader
/// <summary>
/// Gets the SHA-1 hash chunk id.
/// </summary>
[ProtoMember(1)]
[ ProtoMember( 1 ) ]
public byte[] ChunkID { get; private set; }
/// <summary>
/// Gets the expected Adler32 checksum of this chunk.
/// </summary>
[ProtoMember(2)]
[ ProtoMember( 2 ) ]
public byte[] Checksum { get; private set; }
/// <summary>
/// Gets the chunk offset.
/// </summary>
[ProtoMember(3)]
[ ProtoMember( 3 ) ]
public ulong Offset { get; private set; }
/// <summary>
/// Gets the compressed length of this chunk.
/// </summary>
[ProtoMember(4)]
[ ProtoMember( 4 ) ]
public uint CompressedLength { get; private set; }
/// <summary>
/// Gets the decompressed length of this chunk.
/// </summary>
[ProtoMember(5)]
[ ProtoMember( 5 ) ]
public uint UncompressedLength { get; private set; }
}
[ProtoMember(1)]
[ ProtoMember( 1 ) ]
public List<FileData> Files { get; private set; }
[ProtoMember(2)]
[ ProtoMember( 2 ) ]
public ulong ID { get; private set; }
[ProtoMember(3)]
[ ProtoMember( 3 ) ]
public DateTime CreationTime { get; private set; }
public static ProtoManifest LoadFromFile(string filename, out byte[] checksum)
public static ProtoManifest LoadFromFile( string filename, out byte[] checksum )
{
if (!File.Exists(filename))
if ( !File.Exists( filename ) )
{
checksum = null;
return null;
}
using (MemoryStream ms = new MemoryStream())
using ( MemoryStream ms = new MemoryStream() )
{
using (FileStream fs = File.Open(filename, FileMode.Open))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress))
ds.CopyTo(ms);
using ( FileStream fs = File.Open( filename, FileMode.Open ) )
using ( DeflateStream ds = new DeflateStream( fs, CompressionMode.Decompress ) )
ds.CopyTo( ms );
checksum = Util.SHAHash(ms.ToArray());
checksum = Util.SHAHash( ms.ToArray() );
ms.Seek(0, SeekOrigin.Begin);
return ProtoBuf.Serializer.Deserialize<ProtoManifest>(ms);
ms.Seek( 0, SeekOrigin.Begin );
return ProtoBuf.Serializer.Deserialize<ProtoManifest>( ms );
}
}
public void SaveToFile(string filename, out byte[] checksum)
public void SaveToFile( string filename, out byte[] checksum )
{
using (MemoryStream ms = new MemoryStream())
using ( MemoryStream ms = new MemoryStream() )
{
ProtoBuf.Serializer.Serialize<ProtoManifest>(ms, this);
ProtoBuf.Serializer.Serialize<ProtoManifest>( ms, this );
checksum = Util.SHAHash(ms.ToArray());
checksum = Util.SHAHash( ms.ToArray() );
ms.Seek(0, SeekOrigin.Begin);
ms.Seek( 0, SeekOrigin.Begin );
using (FileStream fs = File.Open(filename, FileMode.Create))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Compress))
ms.CopyTo(ds);
using ( FileStream fs = File.Open( filename, FileMode.Create ) )
using ( DeflateStream ds = new DeflateStream( fs, CompressionMode.Compress ) )
ms.CopyTo( ds );
}
}
}

72
DepotDownloader/Steam3Session.cs
Unescape Escape View File

@ -11,7 +11,6 @@ using System.Threading.Tasks;
namespace DepotDownloader
{
class Steam3Session
{
public class Credentials
@ -90,12 +89,12 @@ namespace DepotDownloader
this.PackageInfo = new Dictionary<uint, SteamApps.PICSProductInfoCallback.PICSProductInfo>();
this.AppBetaPasswords = new Dictionary<string, byte[]>();
var clientConfiguration = SteamConfiguration.Create(config =>
var clientConfiguration = SteamConfiguration.Create( config =>
config
.WithHttpClientFactory(HttpClientFactory.CreateHttpClient)
.WithHttpClientFactory( HttpClientFactory.CreateHttpClient )
);
this.steamClient = new SteamClient(clientConfiguration);
this.steamClient = new SteamClient( clientConfiguration );
this.steamUser = this.steamClient.GetHandler<SteamUser>();
this.steamApps = this.steamClient.GetHandler<SteamApps>();
@ -135,13 +134,14 @@ namespace DepotDownloader
}
public delegate bool WaitCondition();
private object steamLock = new object();
public bool WaitUntilCallback( Action submitter, WaitCondition waiter )
{
while ( !bAborted && !waiter() )
{
lock (steamLock)
lock ( steamLock )
{
submitter();
}
@ -149,12 +149,11 @@ namespace DepotDownloader
int seq = this.seq;
do
{
lock (steamLock)
lock ( steamLock )
{
WaitForCallbacks();
}
}
while ( !bAborted && this.seq == seq && !waiter() );
} while ( !bAborted && this.seq == seq && !waiter() );
}
return bAborted;
@ -248,7 +247,7 @@ namespace DepotDownloader
foreach ( var package in packageInfo.UnknownPackages )
{
PackageInfo[package] = null;
PackageInfo[ package ] = null;
}
};
@ -318,14 +317,14 @@ namespace DepotDownloader
}, () => { return completed; } );
}
public string ResolveCDNTopLevelHost(string host)
public string ResolveCDNTopLevelHost( string host )
{
// SteamPipe CDN shares tokens with all hosts
if (host.EndsWith( ".steampipe.steamcontent.com" ) )
if ( host.EndsWith( ".steampipe.steamcontent.com" ) )
{
return "steampipe.steamcontent.com";
}
else if (host.EndsWith(".steamcontent.com"))
else if ( host.EndsWith( ".steamcontent.com" ) )
{
return "steamcontent.com";
}
@ -354,7 +353,7 @@ namespace DepotDownloader
return;
}
CDNAuthTokens[cdnKey].TrySetResult( cdnAuth );
CDNAuthTokens[ cdnKey ].TrySetResult( cdnAuth );
};
WaitUntilCallback( () =>
@ -383,8 +382,8 @@ namespace DepotDownloader
callbacks.Subscribe( steamApps.CheckAppBetaPassword( appid, password ), cbMethod );
}, () => { return completed; } );
}
public PublishedFileDetails GetPublishedFileDetails(uint appId, PublishedFileID pubFile)
public PublishedFileDetails GetPublishedFileDetails( uint appId, PublishedFileID pubFile )
{
var pubFileRequest = new CPublishedFile_GetDetails_Request() { appid = appId };
pubFileRequest.publishedfileids.Add( pubFile );
@ -395,27 +394,27 @@ namespace DepotDownloader
Action<SteamUnifiedMessages.ServiceMethodResponse> cbMethod = callback =>
{
completed = true;
if (callback.Result == EResult.OK)
if ( callback.Result == EResult.OK )
{
var response = callback.GetDeserializedResponse<CPublishedFile_GetDetails_Response>();
details = response.publishedfiledetails.FirstOrDefault();
}
else
{
throw new Exception($"EResult {(int)callback.Result} ({callback.Result}) while retrieving file details for pubfile {pubFile}.");
throw new Exception( $"EResult {( int )callback.Result} ({callback.Result}) while retrieving file details for pubfile {pubFile}." );
}
};
WaitUntilCallback(() =>
WaitUntilCallback( () =>
{
callbacks.Subscribe(steamPublishedFile.SendMessage(api => api.GetDetails(pubFileRequest)), cbMethod);
}, () => { return completed; });
callbacks.Subscribe( steamPublishedFile.SendMessage( api => api.GetDetails( pubFileRequest ) ), cbMethod );
}, () => { return completed; } );
return details;
}
public SteamCloud.UGCDetailsCallback GetUGCDetails(UGCHandle ugcHandle)
public SteamCloud.UGCDetailsCallback GetUGCDetails( UGCHandle ugcHandle )
{
bool completed = false;
SteamCloud.UGCDetailsCallback details = null;
@ -423,24 +422,24 @@ namespace DepotDownloader
Action<SteamCloud.UGCDetailsCallback> cbMethod = callback =>
{
completed = true;
if (callback.Result == EResult.OK)
if ( callback.Result == EResult.OK )
{
details = callback;
}
else if (callback.Result == EResult.FileNotFound)
else if ( callback.Result == EResult.FileNotFound )
{
details = null;
}
else
{
throw new Exception($"EResult {(int)callback.Result} ({callback.Result}) while retrieving UGC details for {ugcHandle}.");
throw new Exception( $"EResult {( int )callback.Result} ({callback.Result}) while retrieving UGC details for {ugcHandle}." );
}
};
WaitUntilCallback(() =>
WaitUntilCallback( () =>
{
callbacks.Subscribe(steamCloud.RequestUGCDetails(ugcHandle), cbMethod);
}, () => { return completed; });
callbacks.Subscribe( steamCloud.RequestUGCDetails( ugcHandle ), cbMethod );
}, () => { return completed; } );
return details;
}
@ -452,16 +451,16 @@ namespace DepotDownloader
bIsConnectionRecovery = false;
bDidReceiveLoginKey = false;
}
void Connect()
{
bAborted = false;
bConnected = false;
bConnecting = true;
connectionBackoff = 0;
ResetConnectionFlags();
this.connectTime = DateTime.Now;
this.steamClient.Connect();
}
@ -470,19 +469,20 @@ namespace DepotDownloader
{
Disconnect( sendLogOff );
}
public void Disconnect( bool sendLogOff = true )
{
if ( sendLogOff )
{
steamUser.LogOff();
}
bAborted = true;
bConnected = false;
bConnecting = false;
bIsConnectionRecovery = false;
steamClient.Disconnect();
// flush callbacks until our disconnected event
while ( !bDidDisconnect )
{
@ -495,7 +495,7 @@ namespace DepotDownloader
bIsConnectionRecovery = true;
steamClient.Disconnect();
}
public void TryWaitForLoginKey()
{
if ( logonDetails.Username == null || !credentials.LoggedOn || !ContentDownloader.Config.RememberPassword ) return;
@ -548,7 +548,7 @@ namespace DepotDownloader
private void DisconnectedCallback( SteamClient.DisconnectedCallback disconnected )
{
bDidDisconnect = true;
// When recovering the connection, we want to reconnect even if the remote disconnects us
if ( !bIsConnectionRecovery && ( disconnected.UserInitiated || bExpectingDisconnectRemote ) )
{
@ -634,7 +634,7 @@ namespace DepotDownloader
else if ( loggedOn.Result == EResult.TryAnotherCM )
{
Console.Write( "Retrying Steam3 connection (TryAnotherCM)..." );
Reconnect();
return;
@ -734,7 +734,5 @@ namespace DepotDownloader
bDidReceiveLoginKey = true;
}
}
}

75
DepotDownloader/Util.cs
Unescape Escape View File

@ -13,15 +13,15 @@ namespace DepotDownloader
{
public static string GetSteamOS()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
if ( RuntimeInformation.IsOSPlatform( OSPlatform.Windows ) )
{
return "windows";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
else if ( RuntimeInformation.IsOSPlatform( OSPlatform.OSX ) )
{
return "macos";
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
else if ( RuntimeInformation.IsOSPlatform( OSPlatform.Linux ) )
{
return "linux";
}
@ -39,7 +39,7 @@ namespace DepotDownloader
ConsoleKeyInfo keyInfo;
StringBuilder password = new StringBuilder();
do
do
{
keyInfo = Console.ReadKey( true );
@ -50,6 +50,7 @@ namespace DepotDownloader
password.Remove( password.Length - 1, 1 );
Console.Write( "\x1B[1D\x1B[1P" );
}
continue;
}
@ -66,52 +67,53 @@ namespace DepotDownloader
}
// Validate a file against Steam3 Chunk data
public static List<ProtoManifest.ChunkData> ValidateSteam3FileChecksums(FileStream fs, ProtoManifest.ChunkData[] chunkdata)
public static List<ProtoManifest.ChunkData> ValidateSteam3FileChecksums( FileStream fs, ProtoManifest.ChunkData[] chunkdata )
{
var neededChunks = new List<ProtoManifest.ChunkData>();
int read;
foreach (var data in chunkdata)
foreach ( var data in chunkdata )
{
byte[] chunk = new byte[data.UncompressedLength];
fs.Seek((long)data.Offset, SeekOrigin.Begin);
read = fs.Read(chunk, 0, (int)data.UncompressedLength);
byte[] chunk = new byte[ data.UncompressedLength ];
fs.Seek( ( long )data.Offset, SeekOrigin.Begin );
read = fs.Read( chunk, 0, ( int )data.UncompressedLength );
byte[] tempchunk;
if (read < data.UncompressedLength)
if ( read < data.UncompressedLength )
{
tempchunk = new byte[read];
Array.Copy(chunk, 0, tempchunk, 0, read);
tempchunk = new byte[ read ];
Array.Copy( chunk, 0, tempchunk, 0, read );
}
else
{
tempchunk = chunk;
}
byte[] adler = AdlerHash(tempchunk);
if (!adler.SequenceEqual(data.Checksum))
byte[] adler = AdlerHash( tempchunk );
if ( !adler.SequenceEqual( data.Checksum ) )
{
neededChunks.Add(data);
neededChunks.Add( data );
}
}
return neededChunks;
}
public static byte[] AdlerHash(byte[] input)
public static byte[] AdlerHash( byte[] input )
{
uint a = 0, b = 0;
for (int i = 0; i < input.Length; i++)
for ( int i = 0; i < input.Length; i++ )
{
a = (a + input[i]) % 65521;
b = (b + a) % 65521;
a = ( a + input[ i ] ) % 65521;
b = ( b + a ) % 65521;
}
return BitConverter.GetBytes(a | (b << 16));
return BitConverter.GetBytes( a | ( b << 16 ) );
}
public static byte[] SHAHash( byte[] input )
{
using (var sha = SHA1.Create())
using ( var sha = SHA1.Create() )
{
var output = sha.ComputeHash( input );
@ -127,7 +129,7 @@ namespace DepotDownloader
int chars = hex.Length;
byte[] bytes = new byte[ chars / 2 ];
for ( int i = 0 ; i < chars ; i += 2 )
for ( int i = 0; i < chars; i += 2 )
bytes[ i / 2 ] = Convert.ToByte( hex.Substring( i, 2 ), 16 );
return bytes;
@ -137,40 +139,39 @@ namespace DepotDownloader
{
return input.Aggregate( new StringBuilder(),
( sb, v ) => sb.Append( v.ToString( "x2" ) )
).ToString();
).ToString();
}
public static async Task InvokeAsync(IEnumerable<Func<Task>> taskFactories, int maxDegreeOfParallelism)
public static async Task InvokeAsync( IEnumerable<Func<Task>> taskFactories, int maxDegreeOfParallelism )
{
if (taskFactories == null) throw new ArgumentNullException(nameof(taskFactories));
if (maxDegreeOfParallelism <= 0) throw new ArgumentException(nameof(maxDegreeOfParallelism));
if ( taskFactories == null ) throw new ArgumentNullException( nameof(taskFactories) );
if ( maxDegreeOfParallelism <= 0 ) throw new ArgumentException( nameof(maxDegreeOfParallelism) );
Func<Task>[] queue = taskFactories.ToArray();
if (queue.Length == 0)
if ( queue.Length == 0 )
{
return;
}
List<Task> tasksInFlight = new List<Task>(maxDegreeOfParallelism);
List<Task> tasksInFlight = new List<Task>( maxDegreeOfParallelism );
int index = 0;
do
{
while (tasksInFlight.Count < maxDegreeOfParallelism && index < queue.Length)
while ( tasksInFlight.Count < maxDegreeOfParallelism && index < queue.Length )
{
Func<Task> taskFactory = queue[index++];
Func<Task> taskFactory = queue[ index++ ];
tasksInFlight.Add(taskFactory());
tasksInFlight.Add( taskFactory() );
}
Task completedTask = await Task.WhenAny(tasksInFlight).ConfigureAwait(false);
Task completedTask = await Task.WhenAny( tasksInFlight ).ConfigureAwait( false );
await completedTask.ConfigureAwait(false);
await completedTask.ConfigureAwait( false );
tasksInFlight.Remove(completedTask);
}
while (index < queue.Length || tasksInFlight.Count != 0);
tasksInFlight.Remove( completedTask );
} while ( index < queue.Length || tasksInFlight.Count != 0 );
}
}
}

Write Preview
Loading…
Cancel
Save