KnightsOfTheArk
/
DepotDownloader
mirror of https://github.com/SteamRE/DepotDownloader.git
23
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix bug where DD would exit prematurely

Browse Source
pull/19/head
Netshroud 9 years ago
parent ab9ca2a7da
commit b17e343d0e
1 changed files with 160 additions and 152 deletions
Show Stats Download Patch File Download Diff File

312
DepotDownloader/ContentDownloader.cs
Unescape Escape View File

@ -3,12 +3,10 @@ using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Net; using System.Net;
using System.Net.Sockets;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading; using System.Threading;
using SteamKit2; using SteamKit2;
using System.Collections.Concurrent;
using System.Threading.Tasks; using System.Threading.Tasks;
namespace DepotDownloader namespace DepotDownloader
@ -442,7 +440,7 @@ namespace DepotDownloader
foreach (var depot in depotIDs) foreach (var depot in depotIDs)
{ {
DepotDownloadInfo info = GetDepotInfo(depot, appId, branch); var info = GetDepotInfo(depot, appId, branch);
if (info != null) if (info != null)
{ {
infos.Add(info); infos.Add(info);
@ -678,210 +676,220 @@ namespace DepotDownloader
}); });
var semaphore = new SemaphoreSlim(Config.MaxDownloads); var semaphore = new SemaphoreSlim(Config.MaxDownloads);
filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)) var files = filesAfterExclusions.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
.AsParallel().WithCancellation(cts.Token).WithDegreeOfParallelism(Config.MaxDownloads) var tasks = new Task[files.Length];
.ForAll(async file => for (var i = 0; i < files.Length; i++)
{ {
try var file = files[i];
var task = Task.Run(async () =>
{ {
await semaphore.WaitAsync().ConfigureAwait(false); cts.Token.ThrowIfCancellationRequested();
string fileFinalPath = Path.Combine(depot.installDir, file.FileName); try
string fileStagingPath = Path.Combine(stagingDir, file.FileName);
// This may still exist if the previous run exited before cleanup
if (File.Exists(fileStagingPath))
{ {
File.Delete(fileStagingPath); await semaphore.WaitAsync().ConfigureAwait(false);
}
FileStream fs = null; string fileFinalPath = Path.Combine(depot.installDir, file.FileName);
List<ProtoManifest.ChunkData> neededChunks; string fileStagingPath = Path.Combine(stagingDir, file.FileName);
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists) // This may still exist if the previous run exited before cleanup
{ if (File.Exists(fileStagingPath))
// create new file. need all chunks
fs = File.Create(fileFinalPath);
fs.SetLength((long)file.TotalSize);
neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{ {
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName); File.Delete(fileStagingPath);
} }
if (oldManifestFile != null) FileStream fs = null;
List<ProtoManifest.ChunkData> neededChunks;
FileInfo fi = new FileInfo(fileFinalPath);
if (!fi.Exists)
{ {
neededChunks = new List<ProtoManifest.ChunkData>(); // create new file. need all chunks
fs = File.Create(fileFinalPath);
if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash)) fs.SetLength((long)file.TotalSize);
neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
}
else
{
// open existing
ProtoManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{ {
// we have a version of this file, but it doesn't fully match what we want oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
}
var matchingChunks = new List<ChunkMatch>(); if (oldManifestFile != null)
{
neededChunks = new List<ProtoManifest.ChunkData>();
foreach (var chunk in file.Chunks) if (Config.VerifyAll || !oldManifestFile.FileHash.SequenceEqual(file.FileHash))
{ {
var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID)); // we have a version of this file, but it doesn't fully match what we want
if (oldChunk != null)
{
matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
}
else
{
neededChunks.Add(chunk);
}
}
File.Move(fileFinalPath, fileStagingPath);
fs = File.Open(fileFinalPath, FileMode.Create); var matchingChunks = new List<ChunkMatch>();
fs.SetLength((long)file.TotalSize);
using (var fsOld = File.Open(fileStagingPath, FileMode.Open)) foreach (var chunk in file.Chunks)
{
foreach (var match in matchingChunks)
{ {
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin); var oldChunk = oldManifestFile.Chunks.FirstOrDefault(c => c.ChunkID.SequenceEqual(chunk.ChunkID));
if (oldChunk != null)
byte[] tmp = new byte[match.OldChunk.UncompressedLength];
fsOld.Read(tmp, 0, tmp.Length);
byte[] adler = Util.AdlerHash(tmp);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
{ {
neededChunks.Add(match.NewChunk); matchingChunks.Add(new ChunkMatch(oldChunk, chunk));
} }
else else
{ {
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin); neededChunks.Add(chunk);
fs.Write(tmp, 0, tmp.Length);
} }
} }
}
File.Delete(fileStagingPath); File.Move(fileFinalPath, fileStagingPath);
}
}
else
{
// No old manifest or file not in old manifest. We must validate.
fs = File.Open(fileFinalPath, FileMode.Open); fs = File.Open(fileFinalPath, FileMode.Create);
if ((ulong)fi.Length != file.TotalSize) fs.SetLength((long)file.TotalSize);
{
fs.SetLength((long)file.TotalSize);
}
neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray()); using (var fsOld = File.Open(fileStagingPath, FileMode.Open))
} {
foreach (var match in matchingChunks)
if (neededChunks.Count() == 0) {
{ fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);
size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath); byte[] tmp = new byte[match.OldChunk.UncompressedLength];
if (fs != null) fsOld.Read(tmp, 0, tmp.Length);
fs.Dispose();
return; byte[] adler = Util.AdlerHash(tmp);
} if (!adler.SequenceEqual(match.OldChunk.Checksum))
else {
{ neededChunks.Add(match.NewChunk);
size_downloaded += (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum()); }
} else
} {
fs.Seek((long)match.NewChunk.Offset, SeekOrigin.Begin);
fs.Write(tmp, 0, tmp.Length);
}
}
}
foreach (var chunk in neededChunks) File.Delete(fileStagingPath);
{ }
if (cts.IsCancellationRequested) break; }
else
{
// No old manifest or file not in old manifest. We must validate.
string chunkID = Util.EncodeHexString(chunk.ChunkID); fs = File.Open(fileFinalPath, FileMode.Open);
CDNClient.DepotChunk chunkData = null; if ((ulong)fi.Length != file.TotalSize)
{
fs.SetLength((long)file.TotalSize);
}
while (!cts.IsCancellationRequested) neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
{ }
CDNClient client;
try if (neededChunks.Count() == 0)
{ {
client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false); size_downloaded += file.TotalSize;
Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath);
if (fs != null)
fs.Dispose();
return;
} }
catch (OperationCanceledException) else
{ {
break; size_downloaded += (file.TotalSize - (ulong)neededChunks.Select(x => (long)x.UncompressedLength).Sum());
} }
}
foreach (var chunk in neededChunks)
{
if (cts.IsCancellationRequested) break;
DepotManifest.ChunkData data = new DepotManifest.ChunkData(); string chunkID = Util.EncodeHexString(chunk.ChunkID);
data.ChunkID = chunk.ChunkID; CDNClient.DepotChunk chunkData = null;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
try while (!cts.IsCancellationRequested)
{
chunkData = await client.DownloadDepotChunkAsync(depot.id, data).ConfigureAwait(false);
cdnPool.ReturnConnection(client);
break;
}
catch (WebException e)
{ {
cdnPool.ReturnBrokenConnection(client); CDNClient client;
try
{
client = await cdnPool.GetConnectionForDepotAsync(appId, depot.id, depot.depotKey, cts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
break;
}
DepotManifest.ChunkData data = new DepotManifest.ChunkData();
data.ChunkID = chunk.ChunkID;
data.Checksum = chunk.Checksum;
data.Offset = chunk.Offset;
data.CompressedLength = chunk.CompressedLength;
data.UncompressedLength = chunk.UncompressedLength;
if (e.Status == WebExceptionStatus.ProtocolError) try
{ {
var response = e.Response as HttpWebResponse; chunkData = await client.DownloadDepotChunkAsync(depot.id, data).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden) cdnPool.ReturnConnection(client);
break;
}
catch (WebException e)
{
cdnPool.ReturnBrokenConnection(client);
if (e.Status == WebExceptionStatus.ProtocolError)
{ {
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID); var response = e.Response as HttpWebResponse;
cts.Cancel(); if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
break; {
Console.WriteLine("Encountered 401 for chunk {0}. Aborting.", chunkID);
cts.Cancel();
break;
}
else
{
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode);
}
} }
else else
{ {
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode); Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.Status);
} }
} }
else catch (Exception e)
{ {
Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.Status); cdnPool.ReturnBrokenConnection(client);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
} }
} }
catch (Exception e)
if (chunkData == null)
{ {
cdnPool.ReturnBrokenConnection(client); Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id);
Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message); return;
} }
}
if (chunkData == null) TotalBytesCompressed += chunk.CompressedLength;
{ DepotBytesCompressed += chunk.CompressedLength;
Console.WriteLine("Failed to find any server with chunk {0} for depot {1}. Aborting.", chunkID, depot.id); TotalBytesUncompressed += chunk.UncompressedLength;
return; DepotBytesUncompressed += chunk.UncompressedLength;
}
fs.Seek((long)chunk.Offset, SeekOrigin.Begin);
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
TotalBytesCompressed += chunk.CompressedLength; size_downloaded += chunk.UncompressedLength;
DepotBytesCompressed += chunk.CompressedLength; }
TotalBytesUncompressed += chunk.UncompressedLength;
DepotBytesUncompressed += chunk.UncompressedLength;
fs.Seek((long)chunk.Offset, SeekOrigin.Begin); fs.Dispose();
fs.Write(chunkData.Data, 0, chunkData.Data.Length);
size_downloaded += chunk.UncompressedLength; Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath);
}
finally
{
semaphore.Release();
} }
});
fs.Dispose(); tasks[i] = task;
}
Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, fileFinalPath); await Task.WhenAll(tasks).ConfigureAwait(false);;
}
finally
{
semaphore.Release();
}
});
ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId; ConfigStore.TheConfig.LastManifests[depot.id] = depot.manifestId;
ConfigStore.Save(); ConfigStore.Save();

Write Preview
Loading…
Cancel
Save