KnightsOfTheArk
/
DepotDownloader
mirror of https://github.com/SteamRE/DepotDownloader.git
23
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert to .NET Core.

Browse Source 
Works on macOS, haven't tried Windows / .NET Framework yet.
pull/19/head
Netshroud 9 years ago
parent 35aa666ea5
commit c0b6d0ce3c
6 changed files with 268 additions and 409 deletions
Show Stats Download Patch File Download Diff File

35
DepotDownloader/CDNClientPool.cs
Unescape Escape View File

@ -25,7 +25,7 @@ namespace DepotDownloader
private BlockingCollection<CDNClient.Server> availableServerEndpoints; private BlockingCollection<CDNClient.Server> availableServerEndpoints;
private AutoResetEvent populatePoolEvent; private AutoResetEvent populatePoolEvent;
private Thread monitorThread; private Task monitorTask;
public CDNClientPool(Steam3Session steamSession) public CDNClientPool(Steam3Session steamSession)
{ {
@ -37,13 +37,10 @@ namespace DepotDownloader
populatePoolEvent = new AutoResetEvent(true); populatePoolEvent = new AutoResetEvent(true);
monitorThread = new Thread(ConnectionPoolMonitor); monitorTask = Task.Factory.StartNew(ConnectionPoolMonitorAsync).Unwrap();
monitorThread.Name = "CDNClient Pool Monitor";
monitorThread.IsBackground = true;
monitorThread.Start();
} }
private List<CDNClient.Server> FetchBootstrapServerList() private async Task<IList<CDNClient.Server>> FetchBootstrapServerListAsync()
{ {
CDNClient bootstrap = new CDNClient(steamSession.steamClient); CDNClient bootstrap = new CDNClient(steamSession.steamClient);
@ -51,7 +48,7 @@ namespace DepotDownloader
{ {
try try
{ {
var cdnServers = bootstrap.FetchServerList(cellId: (uint)ContentDownloader.Config.CellID); var cdnServers = await bootstrap.FetchServerListAsync(cellId: (uint)ContentDownloader.Config.CellID).ConfigureAwait(false);
if (cdnServers != null) if (cdnServers != null)
{ {
return cdnServers; return cdnServers;
@ -64,7 +61,7 @@ namespace DepotDownloader
} }
} }
private void ConnectionPoolMonitor() private async Task ConnectionPoolMonitorAsync()
{ {
while(true) while(true)
{ {
@ -75,7 +72,7 @@ namespace DepotDownloader
steamSession.steamClient.IsConnected && steamSession.steamClient.IsConnected &&
steamSession.steamClient.GetServersOfType(EServerType.CS).Count > 0) steamSession.steamClient.GetServersOfType(EServerType.CS).Count > 0)
{ {
var servers = FetchBootstrapServerList(); var servers = await FetchBootstrapServerListAsync();
var weightedCdnServers = servers.Select(x => var weightedCdnServers = servers.Select(x =>
{ {
@ -101,7 +98,7 @@ namespace DepotDownloader
activeClientAuthed.TryRemove(client, out authData); activeClientAuthed.TryRemove(client, out authData);
} }
private CDNClient BuildConnection(uint appId, uint depotId, byte[] depotKey, CDNClient.Server serverSeed, CancellationToken token) private async Task<CDNClient> BuildConnectionAsync(uint appId, uint depotId, byte[] depotKey, CDNClient.Server serverSeed, CancellationToken token)
{ {
CDNClient.Server server = null; CDNClient.Server server = null;
CDNClient client = null; CDNClient client = null;
@ -147,8 +144,8 @@ namespace DepotDownloader
} }
} }
client.Connect(server); await client.ConnectAsync(server).ConfigureAwait(false);
client.AuthenticateDepot(depotId, depotKey, cdnAuthToken); await client.AuthenticateDepotAsync(depotId, depotKey, cdnAuthToken).ConfigureAwait(false);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -168,7 +165,7 @@ namespace DepotDownloader
return client; return client;
} }
private bool ReauthConnection(CDNClient client, CDNClient.Server server, uint appId, uint depotId, byte[] depotKey) private async Task<bool> ReauthConnectionAsync(CDNClient client, CDNClient.Server server, uint appId, uint depotId, byte[] depotKey)
{ {
DebugLog.Assert(server.Type == "CDN" || steamSession.AppTickets[depotId] == null, "CDNClientPool", "Re-authing a CDN or anonymous connection"); DebugLog.Assert(server.Type == "CDN" || steamSession.AppTickets[depotId] == null, "CDNClientPool", "Re-authing a CDN or anonymous connection");
@ -193,7 +190,7 @@ namespace DepotDownloader
} }
} }
client.AuthenticateDepot(depotId, depotKey, cdnAuthToken); await client.AuthenticateDepotAsync(depotId, depotKey, cdnAuthToken).ConfigureAwait(false);
activeClientAuthed[client] = Tuple.Create(depotId, server); activeClientAuthed[client] = Tuple.Create(depotId, server);
return true; return true;
} }
@ -205,7 +202,7 @@ namespace DepotDownloader
return false; return false;
} }
public CDNClient GetConnectionForDepot(uint appId, uint depotId, byte[] depotKey, CancellationToken token) public async Task<CDNClient> GetConnectionForDepotAsync(uint appId, uint depotId, byte[] depotKey, CancellationToken token)
{ {
CDNClient client = null; CDNClient client = null;
@ -216,24 +213,24 @@ namespace DepotDownloader
// if we couldn't find a connection, make one now // if we couldn't find a connection, make one now
if (client == null) if (client == null)
{ {
client = BuildConnection(appId, depotId, depotKey, null, token); client = await BuildConnectionAsync(appId, depotId, depotKey, null, token).ConfigureAwait(false);
} }
// if we couldn't find the authorization data or it's not authed to this depotid, re-initialize // if we couldn't find the authorization data or it's not authed to this depotid, re-initialize
if (!activeClientAuthed.TryGetValue(client, out authData) || authData.Item1 != depotId) if (!activeClientAuthed.TryGetValue(client, out authData) || authData.Item1 != depotId)
{ {
if (authData.Item2.Type == "CDN" && ReauthConnection(client, authData.Item2, appId, depotId, depotKey)) if (authData.Item2.Type == "CDN" && await ReauthConnectionAsync(client, authData.Item2, appId, depotId, depotKey).ConfigureAwait(false))
{ {
Console.WriteLine("Re-authed CDN connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId); Console.WriteLine("Re-authed CDN connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId);
} }
else if (authData.Item2.Type == "CS" && steamSession.AppTickets[depotId] == null && ReauthConnection(client, authData.Item2, appId, depotId, depotKey)) else if (authData.Item2.Type == "CS" && steamSession.AppTickets[depotId] == null && await ReauthConnectionAsync(client, authData.Item2, appId, depotId, depotKey).ConfigureAwait(false))
{ {
Console.WriteLine("Re-authed anonymous connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId); Console.WriteLine("Re-authed anonymous connection to content server {0} from {1} to {2}", authData.Item2, authData.Item1, depotId);
} }
else else
{ {
ReleaseConnection(client); ReleaseConnection(client);
client = BuildConnection(appId, depotId, depotKey, authData.Item2, token); client = await BuildConnectionAsync(appId, depotId, depotKey, authData.Item2, token).ConfigureAwait(false);
} }
} }

30
DepotDownloader/ContentDownloader.cs
Unescape Escape View File

@ -362,7 +362,7 @@ namespace DepotDownloader
steam3.Disconnect(); steam3.Disconnect();
} }
public static void DownloadApp(uint appId, uint depotId, string branch, bool forceDepot = false) public static async Task DownloadAppAsync(uint appId, uint depotId, string branch, bool forceDepot = false)
{ {
if(steam3 != null) if(steam3 != null)
steam3.RequestAppInfo(appId); steam3.RequestAppInfo(appId);
@ -451,7 +451,7 @@ namespace DepotDownloader
try try
{ {
DownloadSteam3(appId, infos); await DownloadSteam3Async(appId, infos).ConfigureAwait(false);
} }
catch (OperationCanceledException) catch (OperationCanceledException)
{ {
@ -524,7 +524,7 @@ namespace DepotDownloader
public ProtoManifest.ChunkData NewChunk { get; private set; } public ProtoManifest.ChunkData NewChunk { get; private set; }
} }
private static void DownloadSteam3( uint appId, List<DepotDownloadInfo> depots ) private static async Task DownloadSteam3Async( uint appId, List<DepotDownloadInfo> depots )
{ {
ulong TotalBytesCompressed = 0; ulong TotalBytesCompressed = 0;
ulong TotalBytesUncompressed = 0; ulong TotalBytesUncompressed = 0;
@ -583,9 +583,9 @@ namespace DepotDownloader
{ {
CDNClient client = null; CDNClient client = null;
try { try {
client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, CancellationToken.None); client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, CancellationToken.None).ConfigureAwait(false);
depotManifest = client.DownloadManifest(depot.id, depot.manifestId); depotManifest = await client.DownloadManifestAsync(depot.id, depot.manifestId).ConfigureAwait(false);
cdnPool.ReturnConnection(client); cdnPool.ReturnConnection(client);
} }
@ -677,10 +677,15 @@ namespace DepotDownloader
} }
}); });
var semaphore = new SemaphoreSlim(Config.MaxDownloads);
filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)) filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory))
.AsParallel().WithCancellation(cts.Token).WithDegreeOfParallelism(Config.MaxDownloads) .AsParallel().WithCancellation(cts.Token).WithDegreeOfParallelism(Config.MaxDownloads)
.ForAll(file => .ForAll(async file =>
{ {
try
{
await semaphore.WaitAsync().ConfigureAwait(false);
string fileFinalPath = Path.Combine(depot.installDir, file.FileName); string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName); string fileStagingPath = Path.Combine(stagingDir, file.FileName);
@ -780,7 +785,7 @@ namespace DepotDownloader
size_downloaded += file.TotalSize; size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath); Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath);
if (fs != null) if (fs != null)
fs.Close(); fs.Dispose();
return; return;
} }
else else
@ -801,7 +806,7 @@ namespace DepotDownloader
CDNClient client; CDNClient client;
try try
{ {
client = cdnPool.GetConnectionForDepot(appId, depot.id, depot.depotKey, cts.Token); client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false);
} }
catch (OperationCanceledException) catch (OperationCanceledException)
{ {
@ -817,7 +822,7 @@ namespace DepotDownloader
try try
{ {
chunkData = client.DownloadDepotChunk(depot.id, data); chunkData = await client.DownloadDepotChunkAsync(depot.id, data).ConfigureAwait(false);
cdnPool.ReturnConnection(client); cdnPool.ReturnConnection(client);
break; break;
} }
@ -868,9 +873,14 @@ namespace DepotDownloader
size_downloaded += chunk.UncompressedLength; size_downloaded += chunk.UncompressedLength;
} }
fs.Close(); fs.Dispose();
Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath); Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath);
}
finally
{
semaphore.Release();
}
}); });
ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId; ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;

122
DepotDownloader/DepotDownloader.csproj
Unescape Escape View File

@ -1,124 +1,14 @@
<?xml version="1.0" encoding="utf-8"?> <Project Sdk="Microsoft.NET.Sdk">
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProductVersion>9.0.30729</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{39159C47-ACD3-449F-96CA-4F30C8ED147A}</ProjectGuid>
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <TargetFrameworks>netcoreapp1.1;net46</TargetFrameworks>
<RootNamespace>DepotDownloader</RootNamespace> <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<AssemblyName>DepotDownloader</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<FileUpgradeFlags>
</FileUpgradeFlags>
<OldToolsVersion>3.5</OldToolsVersion>
<UpgradeBackupLocation />
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
<RestorePackages>true</RestorePackages>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
<PlatformTarget>x86</PlatformTarget>
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
<PlatformTarget>AnyCPU</PlatformTarget>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<ItemGroup>
<Reference Include="protobuf-net, Version=2.0.0.668, Culture=neutral, PublicKeyToken=257b51d87d2e4d67, processorArchitecture=MSIL">
<SpecificVersion>True</SpecificVersion>
<HintPath>..\packages\protobuf-net.2.0.0.668\lib\net40\protobuf-net.dll</HintPath>
</Reference>
<Reference Include="SteamKit2, Version=1.8.1.0, Culture=neutral, PublicKeyToken=ed3ce47ed5aad940, processorArchitecture=MSIL">
<HintPath>..\packages\SteamKit2.1.8.1\lib\net45\SteamKit2.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Xml.Linq">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.DataSetExtensions">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="CDNClientPool.cs" />
<Compile Include="ContentDownloader.cs" />
<Compile Include="ConfigStore.cs" />
<Compile Include="DownloadConfig.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="ProtoManifest.cs" />
<Compile Include="Steam3Session.cs" />
<Compile Include="Util.cs" />
</ItemGroup>
<ItemGroup> <ItemGroup>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5"> <ProjectReference Include="..\..\SteamKit\SteamKit2\SteamKit2\SteamKit2.csproj" />
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="app.config" /> <PackageReference Include="protobuf-net" Version="2.1.0" />
<None Include="packages.config" /> <PackageReference Include="system.componentmodel.typeconverter" Version="4.3.0" />
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project> </Project>

8
DepotDownloader/Program.cs
Unescape Escape View File

@ -4,12 +4,16 @@ using System.IO;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using SteamKit2; using SteamKit2;
using System.ComponentModel; using System.ComponentModel;
using System.Threading.Tasks;
namespace DepotDownloader namespace DepotDownloader
{ {
class Program class Program
{ {
static void Main( string[] args ) static void Main( string[] args )
=> MainAsync(args).GetAwaiter().GetResult();
static async Task MainAsync( string[] args )
{ {
if ( args.Length == 0 ) if ( args.Length == 0 )
{ {
@ -19,7 +23,7 @@ namespace DepotDownloader
DebugLog.Enabled = false; DebugLog.Enabled = false;
ConfigStore.LoadFromFile(Path.Combine(Environment.CurrentDirectory, "DepotDownloader.config")); ConfigStore.LoadFromFile(Path.Combine(Directory.GetCurrentDirectory(), "DepotDownloader.config"));
bool bDumpManifest = HasParameter( args, "-manifest-only" ); bool bDumpManifest = HasParameter( args, "-manifest-only" );
uint appId = GetParameter<uint>( args, "-app", ContentDownloader.INVALID_APP_ID ); uint appId = GetParameter<uint>( args, "-app", ContentDownloader.INVALID_APP_ID );
@ -110,7 +114,7 @@ namespace DepotDownloader
if (ContentDownloader.InitializeSteam3(username, password)) if (ContentDownloader.InitializeSteam3(username, password))
{ {
ContentDownloader.DownloadApp(appId, depotId, branch, forceDepot); await ContentDownloader.DownloadAppAsync(appId, depotId, branch, forceDepot).ConfigureAwait(false);
ContentDownloader.ShutdownSteam3(); ContentDownloader.ShutdownSteam3();
} }
} }

2
DepotDownloader/Properties/AssemblyInfo.cs
Unescape Escape View File

@ -31,4 +31,4 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers // You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below: // by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")] // [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("2.2.1.*")] [assembly: AssemblyVersion("2.2.2.0")]

64
DepotDownloader/Util.cs
Unescape Escape View File

@ -11,60 +11,19 @@ namespace DepotDownloader
{ {
static class Util static class Util
{ {
[DllImport( "libc" )] public static string GetSteamOS()
static extern int uname( IntPtr buf );
static int _isMacOSX = -1;
// Environment.OSVersion.Platform returns PlatformID.Unix under Mono on OS X
// Code adapted from Mono: mcs/class/Managed.Windows.Forms/System.Windows.Forms/XplatUI.cs
private static bool IsMacOSX()
{
if ( _isMacOSX != -1 )
return _isMacOSX == 1;
IntPtr buf = IntPtr.Zero;
try
{
// The size of the utsname struct varies from system to system, but this _seems_ more than enough
buf = Marshal.AllocHGlobal( 4096 );
if ( uname( buf ) == 0 )
{
string sys = Marshal.PtrToStringAnsi( buf );
if ( sys == "Darwin" )
{ {
_isMacOSX = 1; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
return true;
}
}
}
catch
{ {
// Do nothing? return "windows";
} }
finally else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{ {
if ( buf != IntPtr.Zero ) return "macos";
Marshal.FreeHGlobal( buf );
}
_isMacOSX = 0;
return false;
} }
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
public static string GetSteamOS()
{
switch (Environment.OSVersion.Platform)
{ {
case PlatformID.Win32NT: return "linux";
case PlatformID.Win32Windows:
return "windows";
case PlatformID.MacOSX:
return "macos";
case PlatformID.Unix:
return IsMacOSX() ? "macos" : "linux";
} }
return "unknown"; return "unknown";
@ -141,14 +100,13 @@ namespace DepotDownloader
public static byte[] SHAHash( byte[] input ) public static byte[] SHAHash( byte[] input )
{ {
SHA1Managed sha = new SHA1Managed(); using (var sha = SHA1.Create())
{
byte[] output = sha.ComputeHash( input ); var output = sha.ComputeHash( input );
sha.Clear();
return output; return output;
} }
}
public static byte[] DecodeHexString( string hex ) public static byte[] DecodeHexString( string hex )
{ {

Write Preview
Loading…
Cancel
Save