DepotDownloader: Added support for Steam3 checksums and delta downloads.

DepotDownloader/ContentDownloader.cs

@@ -597,21 +597,42 @@ namespace DepotDownloader
                 if (!Directory.Exists(dir_path))
                     Directory.CreateDirectory(dir_path);
 
-                // TODO: non-checksum validation
+                FileStream fs;
+                DepotManifest.ChunkData[] neededChunks;
                 FileInfo fi = new FileInfo(download_path);
-                if (fi.Exists && (ulong)fi.Length == file.TotalSize)
+                if (!fi.Exists)
+                {
+                    // create new file. need all chunks
+                    fs = File.Create(download_path);
+                    neededChunks = file.Chunks.ToArray();
+                }
+                else
+                {
+                    // open existing
+                    fs = File.Open(download_path, FileMode.Open);
+                    if ((ulong)fi.Length != file.TotalSize)
+                    {                    
+                        fs.SetLength((long)file.TotalSize);
+                    }
+
+                    // find which chunks we need, in order so that we aren't seeking every which way
+                    neededChunks = Util.ValidateSteam3FileChecksums(fs, file.Chunks.OrderBy(x => x.Offset).ToArray());
+
+                    if (neededChunks.Count() == 0)
                     {
                         size_downloaded += file.TotalSize;
                         Console.WriteLine("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path);
                         continue;
                     }
+                    else
+                    {
+                        size_downloaded += (file.TotalSize - (ulong)neededChunks.Select(x => (int)x.UncompressedLength).Sum());
+                    }
+                }
 
                 Console.Write("{0,6:#00.00}% {1}", ((float)size_downloaded / (float)complete_download_size) * 100.0f, download_path);
 
-                FileStream fs = File.Create(download_path);
+                foreach (var chunk in neededChunks)
-                fs.SetLength((long)file.TotalSize);
-
-                foreach (var chunk in file.Chunks)
                 {
                     string chunkID = EncodeHexString(chunk.ChunkID);
 
@@ -781,7 +802,7 @@ namespace DepotDownloader
                     Console.WriteLine("{0,6:#00.00}%\t{1}", perc, downloadPath);
                     // Similar file, let's check checksums
                     if(fi.Length == dirEntry.SizeOrCount && 
-                        Util.ValidateFileChecksums(fi, checksums.GetFileChecksums(dirEntry.FileID)))
+                        Util.ValidateSteam2FileChecksums(fi, checksums.GetFileChecksums(dirEntry.FileID)))
                     {
                         // checksums OK
                         continue;

DepotDownloader/Util.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 using System.Text;
@@ -77,17 +79,20 @@ namespace DepotDownloader
             return password.ToString();
         }
 
-        // Validate a file against Steam2 Checksums
+        // Validate a file against Steam3 Chunk data
-        public static bool ValidateFileChecksums( FileInfo file, int [] checksums )
+        public static DepotManifest.ChunkData[] ValidateSteam3FileChecksums(FileStream fs, DepotManifest.ChunkData[] chunkdata)
         {
-            byte[] chunk = new byte[0x8000]; // checksums are for 32KB at a time
+            var neededChunks = new List<DepotManifest.ChunkData>();
+            int read;
 
-            FileStream fs = file.OpenRead();
+            foreach (DepotManifest.ChunkData data in chunkdata)
-            int read, cnt=0;
-            while ((read = fs.Read(chunk, 0, 0x8000)) > 0)
             {
+                byte[] chunk = new byte[data.UncompressedLength];
+                fs.Seek((long)data.Offset, SeekOrigin.Begin);
+                read = fs.Read(chunk, 0, (int)data.UncompressedLength);
+
                 byte[] tempchunk;
-                if(read < 0x8000)
+                if (read < data.UncompressedLength)
                 {
                     tempchunk = new byte[read];
                     Array.Copy(chunk, 0, tempchunk, 0, read);
@@ -96,30 +101,30 @@ namespace DepotDownloader
                 {
                     tempchunk = chunk;
                 }
-                int adler = BitConverter.ToInt32(AdlerHash(tempchunk), 0);
+
-                int crc32 = BitConverter.ToInt32(CryptoHelper.CRCHash(tempchunk), 0);
+                byte[] adler = AdlerHash(tempchunk);
-                if((adler ^ crc32) != checksums[cnt])
+                if (adler.SequenceEqual(data.Checksum))
                 {
-                    fs.Close();
+                    neededChunks.Add(data);
-                    return false;
                 }
-                ++cnt;   
             }
-            fs.Close();
+
-            return (cnt == checksums.Length);
+            return neededChunks.ToArray();
         }
 
-        // Generate Steam2 Checksums for a file
+        const int STEAM2_CHUCK_SIZE = 0x8000;
-        public static int [] CalculateFileChecksums( FileInfo file )
+        
+        // Validate a file against Steam2 Checksums
+        public static bool ValidateSteam2FileChecksums( FileInfo file, int [] checksums )
         {
-            byte[] chunk = new byte[0x8000]; // checksums are for 32KB at a time
+            byte[] chunk = new byte[STEAM2_CHUCK_SIZE]; // checksums are for 32KB at a time
-            int [] checksums = new int[((file.Length-1)/0x8000)+1];
+
             FileStream fs = file.OpenRead();
             int read, cnt=0;
-            while ((read = fs.Read(chunk, 0, 0x8000)) > 0)
+            while ((read = fs.Read(chunk, 0, STEAM2_CHUCK_SIZE)) > 0)
             {
                 byte[] tempchunk;
-                if(read < 0x8000)
+                if (read < STEAM2_CHUCK_SIZE)
                 {
                     tempchunk = new byte[read];
                     Array.Copy(chunk, 0, tempchunk, 0, read);
@@ -130,10 +135,15 @@ namespace DepotDownloader
                 }
                 int adler = BitConverter.ToInt32(AdlerHash(tempchunk), 0);
                 int crc32 = BitConverter.ToInt32(CryptoHelper.CRCHash(tempchunk), 0);
-                checksums[cnt++] = adler ^ crc32;
+                if((adler ^ crc32) != checksums[cnt])
+                {
+                    fs.Close();
+                    return false;
+                }
+                ++cnt;   
             }
             fs.Close();
-            return checksums;
+            return (cnt == checksums.Length);
         }
 
         public static byte[] AdlerHash(byte[] input)