Merge branch 'master' into master

2 files changed, 32 insertions, 20 deletions

diff --git a/src/invidious.cr b/src/invidious.cr
index 7f77e317..f3404d23 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -108,17 +108,17 @@ LOCALES = {
 
 crawl_threads.times do
   spawn do
-    crawl_videos(PG_DB)
+    crawl_videos(PG_DB, logger)
   end
 end
 
-refresh_channels(PG_DB, channel_threads, CONFIG.full_refresh)
+refresh_channels(PG_DB, logger, channel_threads, CONFIG.full_refresh)
 
-refresh_feeds(PG_DB, feed_threads)
+refresh_feeds(PG_DB, logger, feed_threads)
 
 video_threads.times do |i|
   spawn do
-    refresh_videos(PG_DB)
+    refresh_videos(PG_DB, logger)
   end
 end
 
@@ -126,6 +126,8 @@ top_videos = [] of Video
 spawn do
   pull_top_videos(CONFIG, PG_DB) do |videos|
     top_videos = videos
+    sleep 1.minutes
+    Fiber.yield
   end
 end
 
@@ -133,6 +135,8 @@ popular_videos = [] of ChannelVideo
 spawn do
   pull_popular_videos(PG_DB) do |videos|
     popular_videos = videos
+    sleep 1.minutes
+    Fiber.yield
   end
 end
 
@@ -3644,9 +3648,23 @@ get "/videoplayback" do |env|
   host = "https://r#{fvip}---#{mn}.googlevideo.com"
   url = "/videoplayback?#{query_params.to_s}"
 
+  headers = env.request.headers
+  headers.delete("Host")
+  headers.delete("Cookie")
+  headers.delete("User-Agent")
+  headers.delete("Referer")
+
   region = query_params["region"]?
-  client = make_client(URI.parse(host), proxies, region)
-  response = client.head(url)
+
+  response = HTTP::Client::Response.new(403)
+  loop do
+    begin
+      client = make_client(URI.parse(host), proxies, region)
+      response = client.head(url, headers)
+      break
+    rescue ex
+    end
+  end
 
   if response.headers["Location"]?
     url = URI.parse(response.headers["Location"])
@@ -3664,12 +3682,6 @@ get "/videoplayback" do |env|
     halt env, status_code: 403
   end
 
-  headers = env.request.headers
-  headers.delete("Host")
-  headers.delete("Cookie")
-  headers.delete("User-Agent")
-  headers.delete("Referer")
-
   client = make_client(URI.parse(host), proxies, region)
   client.get(url, headers) do |response|
     env.response.status_code = response.status_code
diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr
index df02c7fb..f6e2d8fe 100644
--- a/src/invidious/jobs.cr
+++ b/src/invidious/jobs.cr
@@ -1,4 +1,4 @@
-def crawl_videos(db)
+def crawl_videos(db, logger)
   ids = Deque(String).new
   random = Random.new
 
@@ -21,7 +21,7 @@ def crawl_videos(db)
       id = ids[0]
       video = get_video(id, db)
     rescue ex
-      STDOUT << id << " : " << ex.message << "\n"
+      logger.write("#{id} : #{ex.message}\n")
       next
     ensure
       ids.delete(id)
@@ -46,7 +46,7 @@ def crawl_videos(db)
   end
 end
 
-def refresh_channels(db, max_threads = 1, full_refresh = false)
+def refresh_channels(db, logger, max_threads = 1, full_refresh = false)
   max_channel = Channel(Int32).new
 
   spawn do
@@ -73,7 +73,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false)
 
               db.exec("UPDATE channels SET updated = $1, author = $2 WHERE id = $3", Time.now, channel.author, id)
             rescue ex
-              STDOUT << id << " : " << ex.message << "\n"
+              logger.write("#{id} : #{ex.message}\n")
             end
 
             active_channel.send(true)
@@ -86,7 +86,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false)
   max_channel.send(max_threads)
 end
 
-def refresh_videos(db)
+def refresh_videos(db, logger)
   loop do
     db.query("SELECT id FROM videos ORDER BY updated") do |rs|
       rs.each do
@@ -94,7 +94,7 @@ def refresh_videos(db)
           id = rs.read(String)
           video = get_video(id, db)
         rescue ex
-          STDOUT << id << " : " << ex.message << "\n"
+          logger.write("#{id} : #{ex.message}\n")
           next
         end
       end
@@ -104,7 +104,7 @@ def refresh_videos(db)
   end
 end
 
-def refresh_feeds(db, max_threads = 1)
+def refresh_feeds(db, logger, max_threads = 1)
   max_channel = Channel(Int32).new
 
   spawn do
@@ -129,7 +129,7 @@ def refresh_feeds(db, max_threads = 1)
             begin
               db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
             rescue ex
-              STDOUT << "REFRESH " << email << " : " << ex.message << "\n"
+              logger.write("REFRESH #{email} : #{ex.message}\n")
             end
 
             active_channel.send(true)