Added retry logic to steam requests and made downloader "retry forever" where appropriate.

DepotDownloader/ContentDownloader.cs

@@ -466,9 +466,8 @@ namespace DepotDownloader
             var cdnClients = new BlockingCollection<CDNClient>();
             CDNClient initialClient = new CDNClient( steam3.steamClient, steam3.AppTickets[depot.id] );
             List<CDNClient.Server> cdnServers = null;
-            int tries = 5;
 
-            while ( tries-- > 0 ) 
+            while(true)
             {
                 try
                 {
@@ -477,7 +476,8 @@ namespace DepotDownloader
                 }
                 catch (WebException)
                 {
-                    Console.WriteLine("\nFailed to retrieve content server list. Remaining tries: {0}", tries);
+                    Console.WriteLine("\nFailed to retrieve content server list.");
+                    Thread.Sleep(500);
                 }
             }
 
@@ -553,8 +553,7 @@ namespace DepotDownloader
                 Console.WriteLine("Downloading depot {0} - {1}", depot.id, depot.contentName);
 
                 var cdnClientsLock = new Object();
-                BlockingCollection<CDNClient> cdnClients = null;            
+                BlockingCollection<CDNClient> cdnClients = null;
-                int liveCdnClients = 0;
                 CancellationTokenSource cts = new CancellationTokenSource();
 
                 ProtoManifest oldProtoManifest = null;
@@ -599,7 +598,6 @@ namespace DepotDownloader
                         DepotManifest depotManifest = null;
 
                         cdnClients = CollectCDNClientsForDepot(depot);
-                        liveCdnClients = cdnClients.Count;
 
                         foreach (var c in cdnClients)
                         {
@@ -793,7 +791,6 @@ namespace DepotDownloader
                             if (cdnClients == null)
                             {
                                 cdnClients = CollectCDNClientsForDepot(depot);
-                                liveCdnClients = cdnClients.Count;
                             }
                         }
                     }
@@ -805,7 +802,7 @@ namespace DepotDownloader
                         string chunkID = Util.EncodeHexString(chunk.ChunkID);
                         CDNClient.DepotChunk chunkData = null;
 
-                        while (liveCdnClients > 0 && !cts.IsCancellationRequested)
+                        while (!cts.IsCancellationRequested)
                         {
                             CDNClient client;
                             try
@@ -827,18 +824,39 @@ namespace DepotDownloader
                             try
                             {
                                 chunkData = client.DownloadDepotChunk(depot.id, data);
-                                cdnClients.Add(client);
                                 break;
                             }
-                            catch (Exception e)
+                            catch (WebException e)
                             {
-                                Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, liveCdnClients);
+                                if (e.Status == WebExceptionStatus.ProtocolError)
-                                int liveCount = Interlocked.Decrement(ref liveCdnClients);
+                                {
-                                if (liveCount == 0)
+                                    var response = e.Response as HttpWebResponse;
+                                    if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden)
+                                    {
+                                        Console.WriteLine("Encountered 401 for chunk {0}. Aborting.");
+                                        cts.Cancel();
+                                        break;
+                                    }
+                                    else
+                                    {
+                                        Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, response.StatusCode);
+                                    }
+                                }
+                                else
                                 {
-                                    // we've run out of clients, tell other threads to abort
+                                    Console.WriteLine("Encountered error downloading chunk {0}: {1}", chunkID, e.Status);
-                                    cts.Cancel();
                                 }
+
+                                // let client "cool off" before re-adding it to the queue
+                                Thread.Sleep(500);
+                            }
+                            catch (Exception e)
+                            {
+                                Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
+                            }
+                            finally
+                            {
+                                cdnClients.Add(client);
                             }
                         }
 

DepotDownloader/Steam3Session.cs

@@ -46,6 +46,7 @@ namespace DepotDownloader
         bool bConnected;
         bool bConnecting;
         bool bAborted;
+        int seq; // more hack fixes
         DateTime connectTime;
 
         // input
@@ -66,6 +67,7 @@ namespace DepotDownloader
             this.bConnected = false;
             this.bConnecting = false;
             this.bAborted = false;
+            this.seq = 0;
 
             this.AppTickets = new Dictionary<uint, byte[]>();
             this.AppTokens = new Dictionary<uint, ulong>();
@@ -109,16 +111,30 @@ namespace DepotDownloader
             Connect();
         }
 
+        public delegate bool WaitCondition();
+        public bool WaitUntilCallback(Action submitter, WaitCondition waiter)
+        {
+            while (!bAborted && !waiter())
+            {
+                submitter();
+
+                int seq = this.seq;
+                do
+                {
+                    WaitForCallbacks();
+                }
+                while (!bAborted && this.seq == seq && !waiter());
+            }
+
+            return bAborted;
+        }
+
         public Credentials WaitForCredentials()
         {
             if (credentials.IsValid || bAborted)
                 return credentials;
 
-            do
+            WaitUntilCallback(() => { }, () => { return credentials.IsValid; });
-            {
-                WaitForCallbacks();
-            }
-            while (!bAborted && !credentials.IsValid);
 
             return credentials;
         }
@@ -143,14 +159,9 @@ namespace DepotDownloader
                 }
             };
 
-            using (var appTokensCallback = new Callback<SteamApps.PICSTokensCallback>(cbMethodTokens, callbacks, steamApps.PICSGetAccessTokens(new List<uint>() { appId }, new List<uint>() { })))
+            WaitUntilCallback(() => { 
-            {
+                new Callback<SteamApps.PICSTokensCallback>(cbMethodTokens, callbacks, steamApps.PICSGetAccessTokens(new List<uint>() { appId }, new List<uint>() { }));
-                do
+            }, () => { return completed; });
-                {
-                    WaitForCallbacks();
-                }
-                while (!completed && !bAborted);
-            }
 
             completed = false;
             Action<SteamApps.PICSProductInfoCallback> cbMethod = (appInfo) =>
@@ -178,14 +189,9 @@ namespace DepotDownloader
                 request.Public = false;
             }
 
-            using (var appInfoCallback = new Callback<SteamApps.PICSProductInfoCallback>(cbMethod, callbacks, steamApps.PICSGetProductInfo(new List<SteamApps.PICSRequest>() { request }, new List<SteamApps.PICSRequest>() { })))
+            WaitUntilCallback(() => { 
-            {
+                new Callback<SteamApps.PICSProductInfoCallback>(cbMethod, callbacks, steamApps.PICSGetProductInfo(new List<SteamApps.PICSRequest>() { request }, new List<SteamApps.PICSRequest>() { }));
-                do
+            }, () => { return completed; });
-                {
-                    WaitForCallbacks();
-                }
-                while (!completed && !bAborted);
-            }
         }
 
         public void RequestPackageInfo(IEnumerable<uint> packageIds)
@@ -213,14 +219,9 @@ namespace DepotDownloader
                 }
             };
 
-            using (var packageInfoCallback = new Callback<SteamApps.PICSProductInfoCallback>(cbMethod, callbacks, steamApps.PICSGetProductInfo(new List<uint>(), packages)))
+            WaitUntilCallback(() => { 
-            {
+                new Callback<SteamApps.PICSProductInfoCallback>(cbMethod, callbacks, steamApps.PICSGetProductInfo(new List<uint>(), packages));
-                do
+            }, () => { return completed; });
-                {
-                    WaitForCallbacks();
-                }
-                while (!completed && !bAborted);
-            }
         }
 
         public void RequestAppTicket(uint appId)
@@ -252,14 +253,9 @@ namespace DepotDownloader
                 }
             };
 
-            using (var appTicketCallback = new Callback<SteamApps.AppOwnershipTicketCallback>(cbMethod, callbacks, steamApps.GetAppOwnershipTicket(appId)))
+            WaitUntilCallback(() => { 
-            {
+                new Callback<SteamApps.AppOwnershipTicketCallback>(cbMethod, callbacks, steamApps.GetAppOwnershipTicket(appId));
-                do
+            }, () => { return completed; });
-                {
-                    WaitForCallbacks();
-                }
-                while (!completed && !bAborted);
-            }
         }
 
         public void RequestDepotKey(uint depotId, uint appid = 0)
@@ -283,14 +279,10 @@ namespace DepotDownloader
                 DepotKeys[depotKey.DepotID] = depotKey.DepotKey;
             };
 
-            using ( var depotKeyCallback = new Callback<SteamApps.DepotKeyCallback>( cbMethod, callbacks, steamApps.GetDepotDecryptionKey( depotId, appid ) ) )
+            WaitUntilCallback(() =>
             {
-                do
+                new Callback<SteamApps.DepotKeyCallback>(cbMethod, callbacks, steamApps.GetDepotDecryptionKey(depotId, appid));
-                {
+            }, () => { return completed; });
-                    WaitForCallbacks();
-                }
-                while ( !completed && !bAborted );
-            }
         }
 
         public void RequestCDNAuthToken(uint depotid, string host)
@@ -313,14 +305,10 @@ namespace DepotDownloader
                 CDNAuthTokens[Tuple.Create(depotid, host)] = cdnAuth;
             };
 
-            using (var cdnAuthCallback = new Callback<SteamApps.CDNAuthTokenCallback>(cbMethod, callbacks, steamApps.GetCDNAuthToken(depotid, host)))
+            WaitUntilCallback(() =>
             {
-                do
+                new Callback<SteamApps.CDNAuthTokenCallback>(cbMethod, callbacks, steamApps.GetCDNAuthToken(depotid, host));
-                {
+            }, () => { return completed; });
-                    WaitForCallbacks();
-                }
-                while (!completed && !bAborted);
-            }
         }
 
         void Connect()
@@ -424,6 +412,7 @@ namespace DepotDownloader
 
             Console.WriteLine(" Done!");
 
+            this.seq++;
             credentials.LoggedOn = true;
 
             if (ContentDownloader.Config.CellID == 0)