commit bb5c54441499d4cd4bd66904be41887a39037d71 Author: Ryan Stecker Date: Sat Mar 19 09:16:59 2011 +0000 Added DepotDownloader POC project. Move over hldsupdatetool, there's a new kid on the block. --HG-- extra : convert_revision : svn%3A946a0da7-ebce-4904-9acb-2f1e67aed693%40212 diff --git a/DepotDownloader.sln b/DepotDownloader.sln new file mode 100644 index 00000000..af710988 --- /dev/null +++ b/DepotDownloader.sln @@ -0,0 +1,26 @@ + +Microsoft Visual Studio Solution File, Format Version 10.00 +# Visual Studio 2008 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DepotDownloader", "DepotDownloader\DepotDownloader.csproj", "{39159C47-ACD3-449F-96CA-4F30C8ED147A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SteamKit2", "..\..\SteamKit2\SteamKit2\SteamKit2.csproj", "{BEB5BF07-BB56-402A-97A3-A41C6444C6A5}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {39159C47-ACD3-449F-96CA-4F30C8ED147A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {39159C47-ACD3-449F-96CA-4F30C8ED147A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {39159C47-ACD3-449F-96CA-4F30C8ED147A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {39159C47-ACD3-449F-96CA-4F30C8ED147A}.Release|Any CPU.Build.0 = Release|Any CPU + {BEB5BF07-BB56-402A-97A3-A41C6444C6A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BEB5BF07-BB56-402A-97A3-A41C6444C6A5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BEB5BF07-BB56-402A-97A3-A41C6444C6A5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BEB5BF07-BB56-402A-97A3-A41C6444C6A5}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/DepotDownloader/CDRManager.cs b/DepotDownloader/CDRManager.cs new file mode 100644 index 00000000..e5704318 --- /dev/null +++ b/DepotDownloader/CDRManager.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using SteamKit2; +using System.Net; + +namespace DepotDownloader +{ + static class CDRManager + { + const string BLOB_FILENAME = "cdr.blob"; + + static Blob cdrBlob; + + + public static void Update() + { + Console.Write( "Updating CDR..." ); + + byte[] cdr = GetCdr(); + byte[] cdrHash = GetHash( cdr ); + + foreach ( var configServer in ServerCache.ConfigServers ) + { + try + { + ConfigServerClient csClient = new ConfigServerClient(); + csClient.Connect( configServer ); + + byte[] tempCdr = csClient.GetContentDescriptionRecord( cdrHash ); + + if ( tempCdr == null ) + continue; + + if ( tempCdr.Length == 0 ) + break; + + cdr = tempCdr; + File.WriteAllBytes( BLOB_FILENAME, tempCdr ); + + break; + } + catch ( Exception ex ) + { + Console.WriteLine( "Warning: Unable to download CDR from config server {0}", configServer ); + } + } + + if ( cdr == null ) + { + Console.WriteLine( "Error: Unable to download CDR!" ); + return; + } + + cdrBlob = new Blob( cdr ); + Console.WriteLine( " Done!" ); + } + + public static int GetLatestDepotVersion( int depotId ) + { + Blob appsBlob = cdrBlob[ CDRFields.eFieldApplicationsRecord ].GetChildBlob(); + + foreach ( var blobField in appsBlob.Fields ) + { + Blob appBlob = blobField.GetChildBlob(); + int appId = appBlob[ CDRAppRecordFields.eFieldAppId ].GetInt32Data(); + + if ( depotId != appId ) + continue; + + return appBlob[ CDRAppRecordFields.eFieldCurrentVersionId ].GetInt32Data(); + } + + return -1; + } + + static byte[] GetCdr() + { + try + { + return File.ReadAllBytes( BLOB_FILENAME ); + } + catch + { + return null; + } + } + static byte[] GetHash( byte[] cdr ) + { + try + { + if ( cdr == null ) + return null; + + return CryptoHelper.SHAHash( cdr ); + } + catch + { + return null; + } + } + } +} diff --git a/DepotDownloader/ContentDownloader.cs b/DepotDownloader/ContentDownloader.cs new file mode 100644 index 00000000..77bba751 --- /dev/null +++ b/DepotDownloader/ContentDownloader.cs @@ -0,0 +1,209 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using SteamKit2; +using System.Net; +using System.IO; +using System.IO.Compression; +using System.Diagnostics; + +namespace DepotDownloader +{ + static class ContentDownloader + { + const string DOWNLOAD_DIR = "depots"; + + public static void Download( int depotId, int depotVersion, int cellId, string username, string password ) + { + Directory.CreateDirectory( DOWNLOAD_DIR ); + + Console.Write( "Finding content servers..." ); + IPEndPoint contentServer = GetStorageServer( depotId, depotVersion, cellId ); + + if ( contentServer == null ) + { + Console.WriteLine( "\nError: Unable to find any content servers for depot {0}, version {1}", depotId, depotVersion ); + return; + } + + Console.WriteLine( " Done!" ); + + ContentServerClient.Credentials credentials = null; + + if ( username != null ) + { + ServerCache.BuildAuthServers( username ); + credentials = GetCredentials( ( uint )depotId, username, password ); + } + + string depotPath = Path.Combine( DOWNLOAD_DIR, depotId.ToString() ); + Directory.CreateDirectory( depotPath ); + + string downloadDir = Path.Combine( depotPath, depotVersion.ToString() ); + Directory.CreateDirectory( downloadDir ); + + string manifestFile = Path.Combine( downloadDir, string.Format( "manifest.bin", depotId, depotVersion ) ); + + ContentServerClient csClient = new ContentServerClient(); + + csClient.Connect( contentServer ); + csClient.EnterStorageMode( ( uint )cellId ); + + uint storageId = csClient.OpenStorage( ( uint )depotId, ( uint )depotVersion, credentials ); + + if ( storageId == uint.MaxValue ) + { + Console.WriteLine( "This depot requires valid user credentials and a license for this app" ); + return; + } + + Console.Write( "Downloading depot manifest..." ); + + byte[] manifestData = csClient.DownloadManifest( storageId ); + File.WriteAllBytes( manifestFile, manifestData ); + + Console.WriteLine( " Done!" ); + + Manifest manifest = new Manifest( manifestData ); + + for ( int x = 0; x < manifest.DirEntries.Count; ++x ) + { + Manifest.DirectoryEntry dirEntry = manifest.DirEntries[ x ]; + + string downloadPath = Path.Combine( downloadDir, dirEntry.FullName ); + + if ( dirEntry.FileID == -1 ) + { + // this is a directory, so lets just create it + Directory.CreateDirectory( downloadPath ); + continue; + } + + float perc = ( ( float )x / ( float )manifest.DirEntries.Count ) * 100.0f; + Console.WriteLine( " {0:0.00}%\t{1}", perc, dirEntry.FullName ); + + ContentServerClient.File file = csClient.DownloadFile( storageId, dirEntry.FileID ); + + if ( file.FileMode == 1 ) + { + // file is compressed + using ( MemoryStream ms = new MemoryStream( file.Data ) ) + using ( DeflateStream ds = new DeflateStream( ms, CompressionMode.Decompress ) ) + { + // skip zlib header + ms.Seek( 2, SeekOrigin.Begin ); + + byte[] inflated = new byte[ dirEntry.ItemSize ]; + ds.Read( inflated, 0, inflated.Length ); + + file.Data = inflated; + } + } + else + { + Debug.Assert( false, string.Format( + "Got file with unexpected filemode!\n" + + "DepotID: {0}\nVersion: {1}\nFile: {2}\nMode: {3}\n", + depotId, depotVersion, dirEntry.FullName, file.FileMode + ) ); + } + + File.WriteAllBytes( downloadPath, file.Data ); + } + + csClient.CloseStorage( storageId ); + csClient.ExitStorageMode(); + + csClient.Disconnect(); + + } + + static ContentServerClient.Credentials GetCredentials( uint depotId, string username, string password ) + { + IPEndPoint authServer = GetAuthServer(); + if ( authServer == null ) + { + Console.WriteLine( "Error: Unable to get authserver!" ); + return null; + } + + AuthServerClient asClient = new AuthServerClient(); + asClient.Connect( authServer ); + + ClientTGT clientTgt; + byte[] serverTgt; + Blob accountRecord; + + Console.Write( "Logging in '{0}'... ", username ); + AuthServerClient.LoginResult result = asClient.Login( username, password, out clientTgt, out serverTgt, out accountRecord ); + + if ( result != AuthServerClient.LoginResult.LoggedIn ) + { + Console.WriteLine( "Unable to login to Steam2: {0}", result ); + return null; + } + + Steam3Session steam3 = new Steam3Session( + new SteamUser.LogOnDetails() + { + Username = username, + Password = password, + + ClientTGT = clientTgt, + ServerTGT = serverTgt, + AccRecord = accountRecord, + }, + depotId + ); + + var steam3Credentials = steam3.WaitForCredentials(); + + if ( !steam3Credentials.HasSessionToken || steam3Credentials.AppTicket == null ) + { + Console.WriteLine( "Unable to get steam3 credentials." ); + return null; + } + + ContentServerClient.Credentials credentials = new ContentServerClient.Credentials() + { + ServerTGT = serverTgt, + AppTicket = steam3Credentials.AppTicket, + SessionToken = steam3Credentials.SessionToken, + }; + + return credentials; + } + + static IPEndPoint GetStorageServer( int depotId, int depotVersion, int cellId ) + { + foreach ( IPEndPoint csdServer in ServerCache.CSDSServers ) + { + ContentServerDSClient csdsClient = new ContentServerDSClient(); + csdsClient.Connect( csdServer ); + + ContentServer[] servers = csdsClient.GetContentServerList( ( uint )depotId, ( uint )depotVersion, ( uint )cellId ); + + if ( servers == null ) + { + Console.WriteLine( "Warning: CSDS {0} rejected the given depotid or version!", csdServer ); + continue; + } + + if ( servers.Length == 0 ) + continue; + + return servers[ 0 ].StorageServer; + } + + return null; + } + static IPEndPoint GetAuthServer() + { + if ( ServerCache.AuthServers.Count > 0 ) + return ServerCache.AuthServers[ 0 ]; + + return null; + } + } +} diff --git a/DepotDownloader/DepotDownloader.csproj b/DepotDownloader/DepotDownloader.csproj new file mode 100644 index 00000000..f3cd963b --- /dev/null +++ b/DepotDownloader/DepotDownloader.csproj @@ -0,0 +1,69 @@ + + + + Debug + AnyCPU + 9.0.30729 + 2.0 + {39159C47-ACD3-449F-96CA-4F30C8ED147A} + Exe + Properties + DepotDownloader + DepotDownloader + v3.5 + 512 + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + 3.5 + + + 3.5 + + + 3.5 + + + + + + + + + + + + + + + {BEB5BF07-BB56-402A-97A3-A41C6444C6A5} + SteamKit2 + + + + + \ No newline at end of file diff --git a/DepotDownloader/Program.cs b/DepotDownloader/Program.cs new file mode 100644 index 00000000..a565d931 --- /dev/null +++ b/DepotDownloader/Program.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using SteamKit2; + +namespace DepotDownloader +{ + class Program + { + + static void Main( string[] args ) + { + if ( args.Length == 0 ) + { + PrintUsage(); + return; + } + + DebugLog.Enabled = false; + + ServerCache.Build(); + CDRManager.Update(); + + int depotId = GetIntParameter( args, "-depot" ); + int depotVersion = GetIntParameter( args, "-version" ); + + if ( depotVersion == -1 ) + { + int latestVer = CDRManager.GetLatestDepotVersion( depotId ); + + if ( latestVer == -1 ) + { + Console.WriteLine( "Error: Unable to find DepotID {0} in the CDR!", depotId ); + return; + } + + string strVersion = GetStringParameter( args, "-version" ); + if ( strVersion != null && strVersion.Equals( "latest", StringComparison.OrdinalIgnoreCase ) ) + { + Console.WriteLine( "Using latest version: {0}", latestVer ); + depotVersion = latestVer; + } + else + { + Console.WriteLine( "Available depot versions:" ); + Console.WriteLine( " Oldest: 0" ); + Console.WriteLine( " Newest: {0}", latestVer ); + return; + } + } + + int cellId = GetIntParameter( args, "-cellid" ); + + if ( cellId == -1 ) + { + cellId = 0; + Console.WriteLine( + "Warning: Using default CellID of 0! This may lead to slow downloads. " + + "You can specify the CellID using the -cellid parameter" ); + } + + string username = GetStringParameter( args, "-username" ); + string password = GetStringParameter( args, "-password" ); + + ContentDownloader.Download( depotId, depotVersion, cellId, username, password ); + + } + + static int IndexOfParam( string[] args, string param ) + { + for ( int x = 0 ; x < args.Length ; ++x ) + { + if ( args[ x ].Equals( param, StringComparison.OrdinalIgnoreCase ) ) + return x; + } + return -1; + } + static int GetIntParameter( string[] args, string param ) + { + string strParam = GetStringParameter( args, param ); + + if ( strParam == null ) + return -1; + + int intParam = -1; + if ( !int.TryParse( strParam, out intParam ) ) + return -1; + + return intParam; + } + static string GetStringParameter( string[] args, string param ) + { + int index = IndexOfParam( args, param ); + + if ( index == -1 || index == ( args.Length - 1 ) ) + return null; + + return args[ index + 1 ]; + } + + static void PrintUsage() + { + Console.WriteLine( "\nUse: depotdownloader \n" ); + + Console.WriteLine( "Parameters:" ); + Console.WriteLine( "\t-depot #\t\t\t- the DepotID to download." ); + Console.WriteLine( "\t-version [# or \"latest\"]\t- the version of the depot to download.\n" ); + + Console.WriteLine( "Optional Parameters:" ); + Console.WriteLine( "\t-cellid #\t\t\t- the CellID of the content server to download from." ); + Console.WriteLine( "\t-username user\t\t\t- the username of the account to login to for restricted content." ); + Console.WriteLine( "\t-password pass\t\t\t- the password of the account to login to for restricted content.\n" ); + } + } +} diff --git a/DepotDownloader/Properties/AssemblyInfo.cs b/DepotDownloader/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..811f5d1f --- /dev/null +++ b/DepotDownloader/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle( "DepotDownloader" )] +[assembly: AssemblyDescription( "" )] +[assembly: AssemblyConfiguration( "" )] +[assembly: AssemblyCompany( "Microsoft" )] +[assembly: AssemblyProduct( "DepotDownloader" )] +[assembly: AssemblyCopyright( "Copyright © Microsoft 2011" )] +[assembly: AssemblyTrademark( "" )] +[assembly: AssemblyCulture( "" )] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible( false )] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid( "df2ab32a-923c-46e3-a1b4-c901ee92ec94" )] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion( "1.0.0.0" )] +[assembly: AssemblyFileVersion( "1.0.0.0" )] diff --git a/DepotDownloader/ServerCache.cs b/DepotDownloader/ServerCache.cs new file mode 100644 index 00000000..e9abc2fd --- /dev/null +++ b/DepotDownloader/ServerCache.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Net; +using SteamKit2; + +namespace DepotDownloader +{ + static class ServerCache + { + public static ServerList ConfigServers { get; private set; } + public static ServerList CSDSServers { get; private set; } + public static ServerList AuthServers { get; private set; } + + static ServerCache() + { + ConfigServers = new ServerList(); + CSDSServers = new ServerList(); + AuthServers = new ServerList(); + } + + + public static void Build() + { + Console.Write( "\nBuilding Steam2 server cache..." ); + + foreach ( IPEndPoint gdServer in GeneralDSClient.GDServers ) + { + BuildServer( gdServer, ConfigServers, EServerType.ConfigServer ); + BuildServer( gdServer, CSDSServers, EServerType.CSDS ); + } + + Console.WriteLine( " Done!" ); + } + + public static void BuildAuthServers( string username ) + { + foreach ( IPEndPoint gdServer in GeneralDSClient.GDServers ) + { + try + { + GeneralDSClient gdsClient = new GeneralDSClient(); + gdsClient.Connect( gdServer ); + + IPEndPoint[] servers = gdsClient.GetAuthServerList( username ); + AuthServers.AddRange( servers ); + + gdsClient.Disconnect(); + } + catch + { + Console.WriteLine( "Warning: Unable to connect to GDS {0} to get list of auth servers.", gdServer ); + } + } + } + + private static void BuildServer( IPEndPoint gdServer, ServerList list, EServerType type ) + { + try + { + GeneralDSClient gdsClient = new GeneralDSClient(); + gdsClient.Connect( gdServer ); + + IPEndPoint[] servers = gdsClient.GetServerList( type ); + list.AddRange( servers ); + + gdsClient.Disconnect(); + } + catch + { + Console.WriteLine( "Warning: Unable to connect to GDS {0} to get list of {1} servers.", gdServer, type ); + } + } + } + + class ServerList : List + { + public new void AddRange( IEnumerable endPoints ) + { + foreach ( IPEndPoint endPoint in endPoints ) + Add( endPoint ); + } + + public new void Add( IPEndPoint endPoint ) + { + if ( this.HasServer( endPoint ) ) + return; + + base.Add( endPoint ); + } + + public bool HasServer( IPEndPoint endPoint ) + { + foreach ( var server in this ) + { + if ( server.Equals( endPoint ) ) + return true; + } + return false; + } + } +} diff --git a/DepotDownloader/Steam3Session.cs b/DepotDownloader/Steam3Session.cs new file mode 100644 index 00000000..bcfb025d --- /dev/null +++ b/DepotDownloader/Steam3Session.cs @@ -0,0 +1,130 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using SteamKit2; +using System.Threading; + +namespace DepotDownloader +{ + + class Steam3Session + { + public class Credentials + { + public bool HasSessionToken { get; set; } + public ulong SessionToken { get; set; } + + public byte[] AppTicket { get; set; } + } + + SteamClient steamClient; + + SteamUser steamUser; + SteamApps steamApps; + + Thread callbackThread; + + DateTime connectTime; + + // input + uint depotId; + SteamUser.LogOnDetails logonDetails; + + // output + Credentials credentials; + + static readonly TimeSpan STEAM3_TIMEOUT = TimeSpan.FromSeconds( 30 ); + + + public Steam3Session( SteamUser.LogOnDetails details, uint depotId ) + { + this.depotId = depotId; + this.logonDetails = details; + + this.credentials = new Credentials(); + + + this.steamClient = new SteamClient(); + + this.steamUser = this.steamClient.GetHandler( SteamUser.NAME ); + this.steamApps = this.steamClient.GetHandler( SteamApps.NAME ); + + this.callbackThread = new Thread( HandleCallbacks ); + this.callbackThread.Start(); + + this.connectTime = DateTime.Now; + this.steamClient.Connect(); + } + + public Credentials WaitForCredentials() + { + this.callbackThread.Join(); // no timespan as the thread will terminate itself + + return credentials; + } + + void HandleCallbacks() + { + while ( true ) + { + var callback = steamClient.WaitForCallback( true, TimeSpan.FromSeconds( 1 ) ); + + TimeSpan diff = DateTime.Now - connectTime; + + if ( diff > STEAM3_TIMEOUT || ( credentials.HasSessionToken && credentials.AppTicket != null ) ) + break; + + if ( callback == null ) + continue; + + if ( callback.IsType() ) + { + steamUser.LogOn( logonDetails ); + } + + if ( callback.IsType() ) + { + var msg = callback as SteamUser.LogOnCallback; + + if ( msg.Result != EResult.OK ) + { + Console.WriteLine( "Unable to login to Steam3: {0}", msg.Result ); + steamUser.LogOff(); + break; + } + + steamApps.GetAppOwnershipTicket( depotId ); + } + + if ( callback.IsType() ) + { + var msg = callback as SteamApps.AppOwnershipTicketCallback; + + if ( msg.AppID != depotId ) + continue; + + if ( msg.Result != EResult.OK ) + { + Console.WriteLine( "Unable to get appticket for {0}: {1}", depotId, msg.Result ); + steamUser.LogOff(); + break; + } + + credentials.AppTicket = msg.Ticket; + + } + + if ( callback.IsType() ) + { + var msg = callback as SteamUser.SessionTokenCallback; + + credentials.SessionToken = msg.SessionToken; + credentials.HasSessionToken = true; + } + } + + steamClient.Disconnect(); + } + } +}