KnightsOfTheArk
/
DepotDownloader
mirror of https://github.com/SteamRE/DepotDownloader.git
23
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Save manifests to optimize needed chunks check. Add -validate option. Concurrent donwloads.

Browse Source
pull/8/head
Nicholas Hastings 12 years ago
parent 43ecfcffc7
commit 2e5ebc8851
6 changed files with 283 additions and 50 deletions
Show Stats Download Patch File Download Diff File

192
DepotDownloader/ContentDownloader.cs
Unescape Escape View File

@ -13,7 +13,6 @@ namespace DepotDownloader
{
static class ContentDownloader
{
private const string DEFAULT_DIR = "depots";
public const uint INVALID_APP_ID = uint.MaxValue;
public const uint INVALID_DEPOT_ID = uint.MaxValue;
public const ulong INVALID_MANIFEST_ID = ulong.MaxValue;
@ -23,6 +22,10 @@ namespace DepotDownloader
private static Steam3Session steam3;
private static Steam3Session.Credentials steam3Credentials;
private const string DEFAULT_DOWNLOAD_DIR = "depots";
private const string CONFIG_DIR = ".DepotDownloader";
private static readonly string STAGING_DIR = Path.Combine(CONFIG_DIR, "staging");
private sealed class DepotDownloadInfo
{
public uint id { get; private set; }
@ -46,21 +49,27 @@ namespace DepotDownloader
installDir = null;
try
{
if (ContentDownloader.Config.InstallDirectory == null || ContentDownloader.Config.InstallDirectory == "")
if (string.IsNullOrWhiteSpace(ContentDownloader.Config.InstallDirectory))
{
Directory.CreateDirectory( DEFAULT_DIR );
Directory.CreateDirectory( DEFAULT_DOWNLOAD_DIR );
string depotPath = Path.Combine( DEFAULT_DIR, depotId.ToString() );
string depotPath = Path.Combine( DEFAULT_DOWNLOAD_DIR, depotId.ToString() );
Directory.CreateDirectory( depotPath );
installDir = Path.Combine(depotPath, depotVersion.ToString());
Directory.CreateDirectory(installDir);
Directory.CreateDirectory(Path.Combine(installDir, CONFIG_DIR));
Directory.CreateDirectory(Path.Combine(installDir, STAGING_DIR));
}
else
{
Directory.CreateDirectory(ContentDownloader.Config.InstallDirectory);
installDir = ContentDownloader.Config.InstallDirectory;
Directory.CreateDirectory(Path.Combine(installDir, CONFIG_DIR));
Directory.CreateDirectory(Path.Combine(installDir, STAGING_DIR));
}
}
catch
@ -361,8 +370,7 @@ namespace DepotDownloader
}
}
if( infos.Count() > 0 )
DownloadSteam3( infos );
DownloadSteam3( infos );
}
static DepotDownloadInfo GetDepotInfo(uint depotId, uint appId, string branch)
@ -412,6 +420,17 @@ namespace DepotDownloader
return info;
}
private class ChunkMatch
{
public ChunkMatch(ProtoManifest.ChunkData oldChunk, DepotManifest.ChunkData newChunk)
{
OldChunk = oldChunk;
NewChunk = newChunk;
}
public ProtoManifest.ChunkData OldChunk { get; private set; }
public DepotManifest.ChunkData NewChunk { get; private set; }
}
private static void DownloadSteam3( List<DepotDownloadInfo> depots )
{
ulong TotalBytesCompressed = 0;
@ -421,20 +440,16 @@ namespace DepotDownloader
{
ulong DepotBytesCompressed = 0;
ulong DepotBytesUncompressed = 0;
uint depotId = depot.id;
ulong depot_manifest = depot.manifestId;
byte[] depotKey = depot.depotKey;
string installDir = depot.installDir;
Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
Console.Write("Finding content servers...");
CDNClient client = new CDNClient(steam3.steamClient, depotId, steam3.AppTickets[depotId], depotKey);
CDNClient client = new CDNClient(steam3.steamClient, depot.id, steam3.AppTickets[depot.id], depot.depotKey);
var cdnServers = client.FetchServerList(cellId: (uint)Config.CellID);
if (cdnServers.Count == 0)
{
Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depotId, depot.contentName);
Console.WriteLine("\nUnable to find any content servers for depot {0} - {1}", depot.id, depot.contentName);
return;
}
@ -455,26 +470,15 @@ namespace DepotDownloader
}
}
DepotManifest depotManifest = client.DownloadManifest(depot_manifest);
DepotManifest depotManifest = client.DownloadManifest(depot.manifestId);
if ( depotManifest == null )
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot_manifest, depotId);
return;
}
if (!depotManifest.DecryptFilenames(depotKey))
{
Console.WriteLine("\nUnable to decrypt manifest for depot {0}", depotId);
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.manifestId, depot.id);
return;
}
Console.WriteLine(" Done!");
ulong complete_download_size = 0;
ulong size_downloaded = 0;
depotManifest.Files.Sort((x, y) => { return x.FileName.CompareTo(y.FileName); });
if (Config.DownloadManifestOnly)
@ -494,56 +498,151 @@ namespace DepotDownloader
continue;
}
ulong complete_download_size = 0;
ulong size_downloaded = 0;
string configDir = Path.Combine(depot.installDir, CONFIG_DIR);
string stagingDir = Path.Combine(depot.installDir, STAGING_DIR);
ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
{
var oldManifestFileName = Directory.GetFiles(configDir, string.Format("{0}.bin", depot.id)).OrderByDescending(x => File.GetLastWriteTimeUtc(x)).FirstOrDefault();
if (oldManifestFileName != null)
{
if (!Config.VerifyAll)
{
oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName);
}
// Delete this regardless. If we finish successfully, we'll write the new one.
File.Delete(oldManifestFileName);
}
}
depotManifest.Files.RemoveAll((x) => !TestIsFileIncluded(x.FileName));
newProtoManifest = new ProtoManifest(depotManifest, depot.manifestId);
foreach (var file in depotManifest.Files)
{
complete_download_size += file.TotalSize;
}
foreach (var file in depotManifest.Files)
depotManifest.Files.AsParallel().WithDegreeOfParallelism(4).ForAll(file =>
{
string download_path = Path.Combine(installDir, file.FileName);
string download_path = Path.Combine(depot.installDir, file.FileName);
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
if (file.Flags.HasFlag(EDepotFileFlag.Directory))
{
if (!Directory.Exists(download_path))
Directory.CreateDirectory(download_path);
continue;
if (!Directory.Exists(fileStagingPath))
Directory.CreateDirectory(fileStagingPath);
return;
}
// TODO: review this section compared to above
string dir_path = Path.GetDirectoryName(download_path);
string dir_staging_path = Path.GetDirectoryName(fileStagingPath);
if (!Directory.Exists(dir_path))
Directory.CreateDirectory(dir_path);
if (!Directory.Exists(Path.Combine(stagingDir, dir_path)))
Directory.CreateDirectory(dir_staging_path);
////
FileStream fs;
DepotManifest.ChunkData[] neededChunks;
FileStream fs = null;
List<DepotManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(download_path);
if (!fi.Exists)
{
// create new file. need all chunks
fs = File.Create(download_path);
neededChunks = file.Chunks.ToArray();
neededChunks = new List<DepotManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
fs = File.Open(download_path, FileMode.Open);
if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
if (oldManifestFile != null)
{
neededChunks = new List<DepotManifest.ChunkData>();
// we have a version of this file. it may or may not match what we want
var matchingChunks = new List<ChunkMatch>();
foreach (var chunk in file.Chunks)
{
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
// can't use FileHash here. It doesn't seem to actually change...
if (matchingChunks.Count != file.Chunks.Count)
{
if (File.Exists(download_path))
{
File.Move(download_path, fileStagingPath);
fs = File.Open(download_path, FileMode.Create);
fs.SetLength((long)file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
{
foreach (var match in matchingChunks)
{
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
fs.Write(tmp, 0, tmp.Length);
}
}
File.Delete(fileStagingPath);
}
else
{
fs = File.Open(download_path, FileMode.Create);
fs.SetLength((long)file.TotalSize);
neededChunks = new List<DepotManifest.ChunkData>(file.Chunks);
}
}
}
else
{
fs = File.Open(download_path, FileMode.Open);
if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
}
// find which chunks we need, in order so that we aren't seeking every which way
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
}
// find which chunks we need, in order so that we aren't seeking every which way
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
if (neededChunks.Count() == 0)
{
size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path);
fs.Close();
continue;
if (fs != null)
fs.Close();
return;
}
else
{
@ -551,8 +650,6 @@ namespace DepotDownloader
}
}
Console.Write("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path);
foreach (var chunk in neededChunks)
{
string chunkID = Util.EncodeHexString(chunk.ChunkID);
@ -567,17 +664,16 @@ namespace DepotDownloader
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength;
Console.CursorLeft = 0;
Console.Write("{0,6:#00.00}%", ((float)size_downloaded / (float)complete_download_size) * 100.0f);
}
fs.Close();
Console.WriteLine();
}
Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path);
});
newProtoManifest.SaveToFile(Path.Combine(configDir, string.Format("{0}.bin", depot.id)));
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depotId, DepotBytesCompressed, DepotBytesUncompressed);
Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.id, DepotBytesCompressed, DepotBytesUncompressed);
}
Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots", TotalBytesCompressed, TotalBytesUncompressed, depots.Count);

1
DepotDownloader/DepotDownloader.csproj
Unescape Escape View File

@ -84,6 +84,7 @@
<Compile Include="DownloadConfig.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="ProtoManifest.cs" />
<Compile Include="Steam3Session.cs" />
<Compile Include="Util.cs" />
</ItemGroup>

2
DepotDownloader/DownloadConfig.cs
Unescape Escape View File

@ -19,5 +19,7 @@ namespace DepotDownloader
public string BetaPassword { get; set; }
public ulong ManifestId { get; set; }
public bool VerifyAll { get; set; }
}
}

1
DepotDownloader/Program.cs
Unescape Escape View File

@ -87,6 +87,7 @@ namespace DepotDownloader
string password = GetParameter<string>(args, "-password");
ContentDownloader.Config.InstallDirectory = GetParameter<string>(args, "-dir");
ContentDownloader.Config.DownloadAllPlatforms = HasParameter(args, "-all-platforms");
ContentDownloader.Config.VerifyAll = HasParameter(args, "-verify-all") || HasParameter(args, "-verify_all") || HasParameter(args, "-validate");
string branch = GetParameter<string>(args, "-branch") ?? GetParameter<string>(args, "-beta") ?? "Public";
if (username != null && password == null)

133
DepotDownloader/ProtoManifest.cs
Unescape Escape View File

@ -0,0 +1,133 @@
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using ProtoBuf;
using SteamKit2;
namespace DepotDownloader
{
[ProtoContract()]
class ProtoManifest
{
// Proto ctor
private ProtoManifest() { }
public ProtoManifest(DepotManifest sourceManifest, ulong id)
{
Files = new List<FileData>();
sourceManifest.Files.ForEach(f => Files.Add(new FileData(f)));
ID = id;
}
[ProtoContract()]
public class FileData
{
// Proto ctor
private FileData() { }
public FileData(DepotManifest.FileData sourceData)
{
Chunks = new List<ChunkData>();
FileName = sourceData.FileName;
sourceData.Chunks.ForEach(c => Chunks.Add(new ChunkData(c)));
Flags = sourceData.Flags;
TotalSize = sourceData.TotalSize;
FileHash = sourceData.FileHash;
}
[ProtoMember(1)]
public string FileName { get; internal set; }
/// <summary>
/// Gets the chunks that this file is composed of.
/// </summary>
[ProtoMember(2)]
public List<ChunkData> Chunks { get; private set; }
/// <summary>
/// Gets the file flags
/// </summary>
[ProtoMember(3)]
public EDepotFileFlag Flags { get; private set; }
/// <summary>
/// Gets the total size of this file.
/// </summary>
[ProtoMember(4)]
public ulong TotalSize { get; private set; }
/// <summary>
/// Gets the hash of this file.
/// </summary>
[ProtoMember(5)]
public byte[] FileHash { get; private set; }
}
[ProtoContract(SkipConstructor = true)]
public class ChunkData
{
public ChunkData(DepotManifest.ChunkData sourceChunk)
{
ChunkID = sourceChunk.ChunkID;
Checksum = sourceChunk.Checksum;
Offset = sourceChunk.Offset;
CompressedLength = sourceChunk.CompressedLength;
UncompressedLength = sourceChunk.UncompressedLength;
}
/// <summary>
/// Gets the SHA-1 hash chunk id.
/// </summary>
[ProtoMember(1)]
public byte[] ChunkID { get; private set; }
/// <summary>
/// Gets the expected Adler32 checksum of this chunk.
/// </summary>
[ProtoMember(2)]
public byte[] Checksum { get; private set; }
/// <summary>
/// Gets the chunk offset.
/// </summary>
[ProtoMember(3)]
public ulong Offset { get; private set; }
/// <summary>
/// Gets the compressed length of this chunk.
/// </summary>
[ProtoMember(4)]
public uint CompressedLength { get; private set; }
/// <summary>
/// Gets the decompressed length of this chunk.
/// </summary>
[ProtoMember(5)]
public uint UncompressedLength { get; private set; }
}
[ProtoMember(1)]
public List<FileData> Files { get; private set; }
[ProtoMember(2)]
public ulong ID { get; private set; }
public static ProtoManifest LoadFromFile(string filename)
{
if (!File.Exists(filename))
return null;
using (FileStream fs = File.Open(filename, FileMode.Open))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress))
return ProtoBuf.Serializer.Deserialize<ProtoManifest>(ds);
}
public void SaveToFile(string filename)
{
using (FileStream fs = File.Open(filename, FileMode.Create))
using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Compress))
ProtoBuf.Serializer.Serialize<ProtoManifest>(ds, this);
}
}
}

4
DepotDownloader/Util.cs
Unescape Escape View File

@ -96,7 +96,7 @@ namespace DepotDownloader
}
// Validate a file against Steam3 Chunk data
public static DepotManifest.ChunkData[] ValidateSteam3FileChecksums(FileStream fs, DepotManifest.ChunkData[] chunkdata)
public static List<DepotManifest.ChunkData> ValidateSteam3FileChecksums(FileStream fs, DepotManifest.ChunkData[] chunkdata)
{
var neededChunks = new List<DepotManifest.ChunkData>();
int read;
@ -125,7 +125,7 @@ namespace DepotDownloader
}
}
return neededChunks.ToArray();
return neededChunks;
}
public static byte[] AdlerHash(byte[] input)

Write Preview
Loading…
Cancel
Save