Remove client pool

3 files changed, 17 insertions, 62 deletions

diff --git a/config/config.yml b/config/config.yml
index 0dc93f86..3891c228 100644
--- a/config/config.yml
+++ b/config/config.yml
@@ -1,4 +1,3 @@
-pool_size: 20
 channel_threads: 5
 threads: 5
 db:
diff --git a/src/helpers.cr b/src/helpers.cr
index 20477241..1506a3dd 100644
--- a/src/helpers.cr
+++ b/src/helpers.cr
@@ -15,7 +15,6 @@ end
 
 class Config
   YAML.mapping({
-    pool_size:       Int32,
     threads:         Int32,
     channel_threads: Int32,
     db:              NamedTuple(
@@ -178,14 +177,6 @@ def elapsed_text(elapsed)
   "#{(millis * 1000).round(2)}µs"
 end
 
-def get_client(pool)
-  while pool.empty?
-    sleep rand(0..10).milliseconds
-  end
-
-  return pool.shift
-end
-
 def fetch_video(id, client)
   info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en").body
   html = client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}").body
@@ -312,7 +303,7 @@ def decrypt_signature(a)
   return a.join("")
 end
 
-def rank_videos(db, n, pool, filter)
+def rank_videos(db, n, filter, url)
   top = [] of {Float64, String}
 
   db.query("SELECT id, wilson_score, published FROM videos WHERE views > 5000 ORDER BY published DESC LIMIT 1000") do |rs|
@@ -339,15 +330,13 @@ def rank_videos(db, n, pool, filter)
       if language_list.size == n
         break
       else
-        client = get_client(pool)
+        client = make_client(url)
         begin
           video = get_video(id, client, db)
         rescue ex
           next
         end
 
-        pool << client
-
         if video.language
           language = video.language
         else
diff --git a/src/invidious.cr b/src/invidious.cr
index a2afcd36..125659ff 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -25,20 +25,11 @@ require "./cookie_fix"
 
 CONFIG = Config.from_yaml(File.read("config/config.yml"))
 
-pool_size = CONFIG.pool_size
 threads = CONFIG.threads
 channel_threads = CONFIG.channel_threads
 
 Kemal.config.extra_options do |parser|
   parser.banner = "Usage: invidious [arguments]"
-  parser.on("-z SIZE", "--youtube-pool=SIZE", "Number of clients in youtube pool (default: #{pool_size})") do |number|
-    begin
-      pool_size = number.to_i
-    rescue ex
-      puts "SIZE must be integer"
-      exit
-    end
-  end
   parser.on("-t THREADS", "--youtube-threads=THREADS", "Number of threads for crawling (default: #{threads})") do |number|
     begin
       threads = number.to_i
@@ -73,25 +64,17 @@ YT_URL     = URI.parse("https://www.youtube.com")
 REDDIT_URL = URI.parse("https://api.reddit.com")
 LOGIN_URL  = URI.parse("https://accounts.google.com")
 
-youtube_pool = Deque.new(pool_size) do
-  make_client(YT_URL)
-end
-
-# Refresh youtube_pool by crawling YT
 threads.times do
   spawn do
     ids = Deque(String).new
     random = Random.new
 
-    client = get_client(youtube_pool)
+    client = make_client(YT_URL)
     search(random.base64(3), client) do |id|
       ids << id
     end
-    youtube_pool << client
 
     loop do
-      client = get_client(youtube_pool)
-
       if ids.empty?
         search(random.base64(3), client) do |id|
           ids << id
@@ -103,7 +86,7 @@ threads.times do
         video = get_video(id, client, PG_DB)
       rescue ex
         STDOUT << id << " : " << ex.message << "\n"
-        youtube_pool << make_client(YT_URL)
+        client = make_client(YT_URL)
         next
       ensure
         ids.delete(id)
@@ -125,8 +108,6 @@ threads.times do
           end
         end
       end
-
-      youtube_pool << client
     end
   end
 end
@@ -139,16 +120,16 @@ channel_threads.times do |i|
       OFFSET (SELECT count(*)*$1/$2 FROM channels)"
       PG_DB.query(query, i, channel_threads) do |rs|
         rs.each do
-          client = get_client(youtube_pool)
+          client = make_client(YT_URL)
+
           begin
             id = rs.read(String)
             channel = get_channel(id, client, PG_DB)
           rescue ex
             STDOUT << id << " : " << ex.message << "\n"
-            youtube_pool << make_client(YT_URL)
+            client = make_client(YT_URL)
             next
           end
-          youtube_pool << client
         end
       end
     end
@@ -169,7 +150,7 @@ spawn do
 
   loop do
     begin
-      top = rank_videos(PG_DB, 40, youtube_pool, filter)
+      top = rank_videos(PG_DB, 40, filter, YT_URL)
     rescue ex
       next
     end
@@ -183,13 +164,12 @@ spawn do
     videos = [] of Video
 
     top.each do |id|
-      client = get_client(youtube_pool)
+      client = make_client(YT_URL)
       begin
         videos << get_video(id, client, PG_DB)
       rescue ex
         next
       end
-      youtube_pool << client
     end
 
     top_videos = videos
@@ -237,8 +217,6 @@ get "/watch" do |env|
     env.params.query.delete_all("listen")
   end
 
-  client = get_client(youtube_pool)
-
   authorized = env.get? "authorized"
   if authorized
     sid = env.get("sid").as(String)
@@ -248,13 +226,12 @@ get "/watch" do |env|
 
   subscriptions ||= [] of String
 
+  client = make_client(YT_URL)
   begin
     video = get_video(id, client, PG_DB)
   rescue ex
     error_message = ex.message
     next templated "error"
-  ensure
-    youtube_pool << client
   end
 
   fmt_stream = [] of HTTP::Params
@@ -364,13 +341,11 @@ get "/search" do |env|
   page = env.params.query["page"]?.try &.to_i
   page ||= 1
 
-  client = get_client(youtube_pool)
+  client = make_client(YT_URL)
 
   html = client.get("/results?q=#{URI.escape(query)}&page=#{page}&sp=EgIQAVAU").body
   html = XML.parse_html(html)
 
-  youtube_pool << client
-
   videos = Array(Hash(String, String)).new
 
   html.xpath_nodes(%q(//ol[@class="item-section"]/li)).each do |item|
@@ -499,9 +474,8 @@ post "/login" do |env|
 
     sid = login.cookies["SID"].value
 
-    client = get_client(youtube_pool)
+    client = make_client(YT_URL)
     user = get_user(sid, client, headers, PG_DB)
-    youtube_pool << client
 
     # We are now logged in
 
@@ -552,13 +526,11 @@ get "/api/manifest/dash/id/:id" do |env|
   local = env.params.query["local"]?.try &.== "true"
   id = env.params.url["id"]
 
-  yt_client = get_client(youtube_pool)
+  client = make_client(YT_URL)
   begin
-    video = get_video(id, yt_client, PG_DB)
+    video = get_video(id, client, PG_DB)
   rescue ex
     halt env, status_code: 403
-  ensure
-    youtube_pool << yt_client
   end
 
   adaptive_fmts = [] of HTTP::Params
@@ -670,9 +642,8 @@ get "/feed/subscriptions" do |env|
 
     sid = env.get("sid").as(String)
 
-    client = get_client(youtube_pool)
+    client = make_client(YT_URL)
     user = get_user(sid, client, headers, PG_DB)
-    youtube_pool << client
 
     args = arg_array(user.subscriptions, 3)
     videos = PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid IN (#{args}) \
@@ -716,7 +687,7 @@ get "/modify_notifications" do |env|
     headers = HTTP::Headers.new
     headers["Cookie"] = env.request.headers["Cookie"]
 
-    client = get_client(youtube_pool)
+    client = make_client(YT_URL)
     subs = client.get("/subscription_manager?disable_polymer=1", headers)
     headers["Cookie"] += "; " + subs.cookies.add_request_headers(headers)["Cookie"]
     match = subs.body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/)
@@ -737,8 +708,6 @@ get "/modify_notifications" do |env|
 
       client.post("/subscription_ajax?action_update_subscription_preferences=1", headers, HTTP::Params.encode(channel_req)).body
     end
-
-    youtube_pool << client
   end
 
   env.redirect referer
@@ -764,7 +733,7 @@ get "/subscription_ajax" do |env|
     headers = HTTP::Headers.new
     headers["Cookie"] = env.request.headers["Cookie"]
 
-    client = get_client(youtube_pool)
+    client = make_client(YT_URL)
     subs = client.get("/subscription_manager?disable_polymer=1", headers)
 
     headers["Cookie"] += "; " + subs.cookies.add_request_headers(headers)["Cookie"]
@@ -795,8 +764,6 @@ get "/subscription_ajax" do |env|
         PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE id = $2", channel_id, sid)
       end
     end
-
-    youtube_pool << client
   end
 
   env.redirect referer