DepotDownloader: Cache server lists and CDR

DepotDownloader/CDRManager.cs

@@ -4,81 +4,105 @@ using System.IO;
 using System.Security.Cryptography;
 using SteamKit2;
 using SteamKit2.Blob;
+using ProtoBuf;
+using System.IO.Compression;
 
 // TODO: make these properties tofix this
 #pragma warning disable 649
 
 namespace DepotDownloader
 {
+    [ProtoContract]
     class CDR
     {
         [BlobField(1)]
+        [ProtoMember(1)]
         public List<App> Apps;
 
         [BlobField(2)]
+        [ProtoMember(2)]
         public List<Sub> Subs;
     }
 
+    [ProtoContract]
     class App
     {
         [BlobField(2)]
+        [ProtoMember(1)]
         public string Name;
 
         [BlobField(1)]
+        [ProtoMember(2)]
         public int AppID;
 
         [BlobField(11)]
+        [ProtoMember(3)]
         public int CurrentVersion;
         
         [BlobField(10)]
+        [ProtoMember(4)]
         public List<AppVersion> Versions;
         
         [BlobField(12)]
+        [ProtoMember(5)]
         public List<FileSystem> FileSystems;
 
         [BlobField(14)]
+        [ProtoMember(6)]
         public Dictionary<string, string> UserDefined;
 
         [BlobField(16)]
+        [ProtoMember(7)]
         public int BetaVersion;
     }
 
+    [ProtoContract]
     class Sub
     {
         [BlobField(1)]
+        [ProtoMember(1)]
         public int SubID;
  
         [BlobField(6)]
+        [ProtoMember(2)]
         public List<int> AppIDs;
     }
 
+    [ProtoContract]
     class AppVersion
     {
         [BlobField(2)]
+        [ProtoMember(1)]
         public uint VersionID;
 
         [BlobField(5)]
+        [ProtoMember(2)]
         public string DepotEncryptionKey;
 
         [BlobField(6)]
+        [ProtoMember(3)]
         public bool IsEncryptionKeyAvailable;
     }
 
+    [ProtoContract]
     class FileSystem
     {
         [BlobField(1)]
+        [ProtoMember(1)]
         public int AppID;
 
         [BlobField(2)]
+        [ProtoMember(2)]
         public string Name;
 
         [BlobField(4)]
+        [ProtoMember(3)]
         public string Platform;
     }
 
     static class CDRManager
     {
-        const string BLOB_FILENAME = "cdr.blob";
+        const string CDR_FILENAME = "cdr.proto";
 
         static CDR cdrObj;
 
@@ -86,48 +110,61 @@ namespace DepotDownloader
         {
             Console.Write( "Updating CDR..." );
 
-            byte[] cdr = GetCdr();
+            if (DateTime.Now > ConfigCache.Instance.CDRCacheTime)
-            byte[] cdrHash = GetHash( cdr );
-
-            foreach ( var configServer in ServerCache.ConfigServers )
             {
-                try
+                byte[] cdrHash = ConfigCache.Instance.CDRHash;
+
+                foreach (var configServer in ServerCache.ConfigServers)
                 {
-                    ConfigServerClient csClient = new ConfigServerClient();
+                    try
-                    csClient.Connect( configServer );
+                    {
+                        ConfigServerClient csClient = new ConfigServerClient();
+                        csClient.Connect(configServer);
 
-                    byte[] tempCdr = csClient.GetContentDescriptionRecord( cdrHash );
+                        byte[] tempCdr = csClient.GetContentDescriptionRecord(cdrHash);
 
-                    if ( tempCdr == null )
+                        if (tempCdr == null)
-                        continue;
+                            continue;
 
-                    if ( tempCdr.Length == 0 )
+                        if (tempCdr.Length == 0)
-                        break;
+                            break;
 
-                    cdr = tempCdr;
+                        using (MemoryStream ms = new MemoryStream(tempCdr))
-                    File.WriteAllBytes( BLOB_FILENAME, tempCdr );
+                        using (BlobReader reader = BlobReader.CreateFrom(ms))
+                            cdrObj = (CDR)BlobTypedReader.Deserialize(reader, typeof(CDR));
 
-                    break;
+                        using (FileStream fs = File.Open(CDR_FILENAME, FileMode.Create))
+                        using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Compress))
+                            ProtoBuf.Serializer.Serialize<CDR>(ds, cdrObj);
+
+                        ConfigCache.Instance.CDRHash = SHAHash(tempCdr);
+                        ConfigCache.Instance.CDRCacheTime = DateTime.Now.AddMinutes(30);
+                        ConfigCache.Instance.Save(ConfigCache.CONFIG_FILENAME);
+                        break;
+                    }
+                    catch (Exception e)
+                    {
+                        Console.WriteLine("Warning: Unable to download CDR from config server {0}: {1}", configServer, e.Message);
+                    }
                 }
-                catch ( Exception )
+
+                if(cdrObj != null)
                 {
-                    Console.WriteLine( "Warning: Unable to download CDR from config server {0}", configServer );
+                    Console.WriteLine(" Done!");
+                    return;
+                }
+                else if (!File.Exists(CDR_FILENAME))
+                {
+                    Console.WriteLine("Error: Unable to download CDR!");
+                    return;
                 }
             }
 
-            if ( cdr == null )
+            using (FileStream fs = File.Open(CDR_FILENAME, FileMode.Open))
-            {
+            using (DeflateStream ds = new DeflateStream(fs, CompressionMode.Decompress))
-                Console.WriteLine( "Error: Unable to download CDR!" );
+                cdrObj = ProtoBuf.Serializer.Deserialize<CDR>(ds);
-                return;
-            }
-
-            using(MemoryStream ms = new MemoryStream(cdr))
-            using(BlobReader reader = BlobReader.CreateFrom(ms))
-            {
-                cdrObj = (CDR)BlobTypedReader.Deserialize(reader, typeof(CDR));
-            }
 
-            Console.WriteLine( " Done!" );
+            Console.WriteLine(" Done!");
         }
 
         static App GetAppBlob( int appID )
@@ -399,32 +436,6 @@ namespace DepotDownloader
             return sub.AppIDs.Contains( depotId );
         }
 
-        static byte[] GetCdr()
-        {
-            try
-            {
-                return File.ReadAllBytes( BLOB_FILENAME );
-            }
-            catch
-            {
-                return null;
-            }
-        }
-        static byte[] GetHash( byte[] cdr )
-        {
-            try
-            {
-                if ( cdr == null )
-                    return null;
-
-                return SHAHash( cdr );
-            }
-            catch
-            {
-                return null;
-            }
-        }
-
         static byte[] SHAHash( byte[] data )
         {
             using ( SHA1Managed sha = new SHA1Managed() )

DepotDownloader/DepotDownloader.csproj

@@ -71,6 +71,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="CDRManager.cs" />
+    <Compile Include="ConfigCache.cs" />
     <Compile Include="ContentDownloader.cs" />
     <Compile Include="DownloadConfig.cs" />
     <Compile Include="Program.cs" />

DepotDownloader/ServerCache.cs

@@ -21,12 +21,33 @@ namespace DepotDownloader
         {
             Console.Write( "\nBuilding Steam2 server cache..." );
 
-            foreach ( IPEndPoint gdServer in GeneralDSClient.GDServers )
+            if (DateTime.Now > ConfigCache.Instance.ServerCacheTime)
             {
-                BuildServer( gdServer, ConfigServers, ESteam2ServerType.ConfigServer );
+                foreach (IPEndPoint gdServer in GeneralDSClient.GDServers)
-                BuildServer( gdServer, CSDSServers, ESteam2ServerType.CSDS );
+                {
+                    BuildServer(gdServer, ConfigServers, ESteam2ServerType.ConfigServer);
+                    BuildServer(gdServer, CSDSServers, ESteam2ServerType.CSDS);
+                }
+
+                if (ConfigServers.Count > 0 && CSDSServers.Count > 0)
+                {
+                    ConfigCache.Instance.ConfigServers = ConfigServers;
+                    ConfigCache.Instance.CSDSServers = CSDSServers;
+                    ConfigCache.Instance.ServerCacheTime = DateTime.Now.AddDays(30);
+                    ConfigCache.Instance.Save(ConfigCache.CONFIG_FILENAME);
+
+                    Console.WriteLine(" Done!");
+                    return;
+                } else if(ConfigCache.Instance.CSDSServers == null || ConfigCache.Instance.ConfigServers == null)
+                {
+                    Console.WriteLine(" Unable to get server list");
+                    return;
+                }
             }
 
+            ConfigServers = ConfigCache.Instance.ConfigServers;
+            CSDSServers = ConfigCache.Instance.CSDSServers;
+
             Console.WriteLine( " Done!" );
         }
 
@@ -42,7 +63,7 @@ namespace DepotDownloader
 
                 gdsClient.Disconnect();
             }
-            catch
+            catch(Exception)
             {
                 Console.WriteLine( "Warning: Unable to connect to GDS {0} to get list of {1} servers.", gdServer, type );
             }