Add exponential backoff to refresh_channels

In practice with the patch I usually see backoff to 2 hours when blocked, so it should improve recovery time. The lim_thread is to work with multi-threading, not sure if it's the best way to do it.

1 files changed, 14 insertions, 2 deletions

diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr
index 4594c1e0..43334718 100644
--- a/src/invidious/helpers/jobs.cr
+++ b/src/invidious/helpers/jobs.cr
@@ -3,15 +3,17 @@ def refresh_channels(db, logger, config)
 
   spawn do
     max_threads = max_channel.receive
+    lim_threads = max_threads
     active_threads = 0
     active_channel = Channel(Bool).new
+    backoff = 1.seconds
 
     loop do
       db.query("SELECT id FROM channels ORDER BY updated") do |rs|
         rs.each do
           id = rs.read(String)
 
-          if active_threads >= max_threads
+          if active_threads >= lim_threads
             if active_channel.receive
               active_threads -= 1
             end
@@ -22,12 +24,22 @@ def refresh_channels(db, logger, config)
             begin
               channel = fetch_channel(id, db, config.full_refresh)
 
+              lim_threads = max_threads
               db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id)
             rescue ex
+              logger.puts("#{id} : #{ex.message}")
               if ex.message == "Deleted or invalid channel"
                 db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id)
+              else
+                lim_threads = 1
+	              logger.puts("#{id} : backing off for #{backoff}s")
+                sleep backoff
+                if backoff < 1.days
+                  backoff += backoff
+                else
+                  backoff = 1.days
+                end
               end
-              logger.puts("#{id} : #{ex.message}")
             end
 
             active_channel.send(true)