diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/helpers.cr | 66 | ||||
| -rw-r--r-- | src/invidious.cr | 84 |
2 files changed, 76 insertions, 74 deletions
diff --git a/src/helpers.cr b/src/helpers.cr new file mode 100644 index 00000000..d2186d6c --- /dev/null +++ b/src/helpers.cr @@ -0,0 +1,66 @@ +# See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html +def ci_lower_bound(pos, n) + if n == 0 + return 0 + end + + # z value here represents a confidence level of 0.95 + z = 1.96 + phat = 1.0*pos/n + + return (phat + z*z/(2*n) - z * Math.sqrt((phat*(1 - phat) + z*z/(4*n))/n))/(1 + z*z/n) +end + +def elapsed_text(elapsed) + millis = elapsed.total_milliseconds + return "#{millis.round(2)}ms" if millis >= 1 + + "#{(millis * 1000).round(2)}µs" +end + +def get_client + while POOL.empty? + sleep rand(0..10).milliseconds + end + + return POOL.shift +end + +def fetch_video(id) + # Grab connection from pool + client = get_client + + info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en").body + info = HTTP::Params.parse(info) + + html = client.get("/watch?v=#{id}").body + html = XML.parse_html(html) + + if info["reason"]? + raise info["reason"] + end + + # Return connection to pool + POOL << client + + video = Video.new(id, info, html, Time.now) + + return video +end + +def get_video(id, refresh = true) + if PG_DB.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool) + video = PG_DB.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video) + + # If record was last updated more than 5 hours ago, refresh (expire param in response lasts for 6 hours) + if refresh && Time.now - video.updated > Time::Span.new(0, 5, 0, 0) + video = fetch_video(id) + PG_DB.exec("UPDATE videos SET info = $2, html = $3, updated = $4 WHERE id = $1", video.to_a) + end + else + video = fetch_video(id) + PG_DB.exec("INSERT INTO videos VALUES ($1, $2, $3, $4)", video.to_a) + end + + return video +end diff --git a/src/invidious.cr b/src/invidious.cr index 0481b026..8591c9d7 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -4,6 +4,7 @@ require "kemal" require "pg" require "time" require "xml" +require "./helpers" PG_DB = DB.open "postgres://kemal:kemal@localhost:5432/invidious" URL = URI.parse("https://www.youtube.com") @@ -24,9 +25,9 @@ end spawn do # Arbitrary start value id = Deque.new(50, "0xjKNDMgE54") - while true + loop do client = get_client - if rand(50) < 1 + if rand(100) < 1 client = HTTP::Client.new(URL, CONTEXT) client.connect_timeout = Time::Span.new(0, 0, 0, 5) end @@ -36,24 +37,26 @@ spawn do video = get_video(id[rand(id.size)], false) rescue ex puts ex + POOL << client next end - rvs = [] of Hash(String, String) + rvs = [] of Hash(String, String) if video.info.has_key?("rvs") video.info["rvs"].split(",").each do |rv| rvs << HTTP::Params.parse(rv).to_h end end - rvs.each do |rv| + rvs.each do |rv| if rv.has_key?("id") - id << rv["id"] + id << rv["id"] + id.shift end end - POOL << client - puts "#{Time.now} 200 GET #{elapsed_text(Time.now - time)}" + POOL << client + puts "#{Time.now} 200 GET youtube.com/watch?v=#{video.id} #{elapsed_text(Time.now - time)}" end end @@ -101,73 +104,6 @@ class Video }) end -# See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html -def ci_lower_bound(pos, n) - if n == 0 - return 0 - end - - # z value here represents a confidence level of 0.95 - z = 1.96 - phat = 1.0*pos/n - - return (phat + z*z/(2*n) - z * Math.sqrt((phat*(1 - phat) + z*z/(4*n))/n))/(1 + z*z/n) -end - -def elapsed_text(elapsed) - millis = elapsed.total_milliseconds - return "#{millis.round(2)}ms" if millis >= 1 - - "#{(millis * 1000).round(2)}µs" -end - -def get_client - while POOL.empty? - sleep rand(0..10).milliseconds - end - - return POOL.shift -end - -def fetch_video(id) - # Grab connection from pool - client = get_client - - info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en").body - info = HTTP::Params.parse(info) - - html = client.get("/watch?v=#{id}").body - html = XML.parse_html(html) - - if info["reason"]? - raise info["reason"] - end - - # Return connection to pool - POOL << client - - video = Video.new(id, info, html, Time.now) - - return video -end - -def get_video(id, refresh = true) - if PG_DB.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool) - video = PG_DB.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video) - - # If record was last updated more than 5 hours ago, refresh (expire param in response lasts for 6 hours) - if refresh && Time.now - video.updated > Time::Span.new(0, 5, 0, 0) - video = fetch_video(id) - PG_DB.exec("UPDATE videos SET info = $2, html = $3, updated = $4 WHERE id = $1", video.to_a) - end - else - video = fetch_video(id) - PG_DB.exec("INSERT INTO videos VALUES ($1, $2, $3, $4)", video.to_a) - end - - return video -end - get "/" do |env| templated "index" end |