Add wait time option and add handling for reddit_pool

2 files changed, 20 insertions, 4 deletions

diff --git a/src/helpers.cr b/src/helpers.cr
index d0b6feec..eeda239a 100644
--- a/src/helpers.cr
+++ b/src/helpers.cr
@@ -252,8 +252,8 @@ end
 
 def make_client(url, context)
   client = HTTP::Client.new(url, context)
-  client.read_timeout = 10.seconds
-  client.connect_timeout = 10.seconds
+  client.read_timeout = 30.seconds
+  client.connect_timeout = 30.seconds
   return client
 end
 
diff --git a/src/invidious.cr b/src/invidious.cr
index 7d870756..a95e1fce 100644
--- a/src/invidious.cr
+++ b/src/invidious.cr
@@ -21,7 +21,7 @@ require "xml"
 require "./helpers"
 
 threads = 10
-
+sleep_time = 1.0
 Kemal.config.extra_options do |parser|
   parser.banner = "Usage: invidious [arguments]"
   parser.on("-t THREADS", "--threads=THREADS", "Number of threads for crawling (default: 10)") do |number|
@@ -32,6 +32,14 @@ Kemal.config.extra_options do |parser|
       exit
     end
   end
+  parser.on("-w TIME", "--wait=TIME", "Time to wait between server requests in seconds (default: 1 second)") do |number|
+    begin
+      sleep_time = number.to_f64
+    rescue ex
+      puts "TIME must be integer or float"
+      exit
+    end
+  end
 end
 
 Kemal::CLI.new
@@ -108,6 +116,8 @@ threads.times do
         end
       end
 
+      sleep sleep_time.seconds
+
       youtube_pool << client
     end
   end
@@ -118,8 +128,14 @@ threads.times do
     loop do
       client = get_client(reddit_pool)
 
+      begin
       client.get("/")
-      sleep 10.seconds
+      rescue ex
+        reddit_pool << make_client(URI.parse("https://api.reddit.com"), CONTEXT)
+        next
+      end
+
+      sleep sleep_time.seconds
 
       reddit_pool << client
     end