diff options
| author | Omar Roth <omarroth@hotmail.com> | 2018-07-26 08:41:23 -0500 |
|---|---|---|
| committer | Omar Roth <omarroth@hotmail.com> | 2018-07-26 08:41:23 -0500 |
| commit | 0fba601e91b6c02d456c5a0710d445d7dfcebfe0 (patch) | |
| tree | 6c8846516dd48640da5dba3703b77ebfee546e1a | |
| parent | 3f577650bc5b613e1bebd29743a56015bb73fbf7 (diff) | |
| download | invidious-0fba601e91b6c02d456c5a0710d445d7dfcebfe0.tar.gz invidious-0fba601e91b6c02d456c5a0710d445d7dfcebfe0.tar.bz2 invidious-0fba601e91b6c02d456c5a0710d445d7dfcebfe0.zip | |
Fix crash in video crawler
| -rw-r--r-- | src/invidious.cr | 2 | ||||
| -rw-r--r-- | src/invidious/helpers.cr | 5 |
2 files changed, 3 insertions, 4 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 72259f87..2cc34d61 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -86,6 +86,7 @@ crawl_threads.times do end loop do + client = make_client(YT_URL) if ids.empty? search(random.base64(3), client) do |id| ids << id @@ -97,7 +98,6 @@ crawl_threads.times do video = get_video(id, client, PG_DB) rescue ex STDOUT << id << " : " << ex.message << "\n" - client = make_client(YT_URL) next ensure ids.delete(id) diff --git a/src/invidious/helpers.cr b/src/invidious/helpers.cr index 71b462d6..3448245b 100644 --- a/src/invidious/helpers.cr +++ b/src/invidious/helpers.cr @@ -307,9 +307,8 @@ def get_video(id, client, db, refresh = true) return video end -def search(query, client) - html = client.get("https://www.youtube.com/results?q=#{query}&sp=EgIQAVAU&disable_polymer=1").body - +def search(query, client, &block) + html = client.get("/results?q=#{query}&sp=EgIQAVAU&disable_polymer=1").body html = XML.parse_html(html) html.xpath_nodes(%q(//ol[@class="item-section"]/li)).each do |item| |