diff options
| -rw-r--r-- | src/invidious.cr | 37 | ||||
| -rw-r--r-- | src/invidious/channels.cr | 27 | ||||
| -rw-r--r-- | src/invidious/users.cr | 17 |
3 files changed, 39 insertions, 42 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 58dcd81f..4be9ee7e 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1355,7 +1355,7 @@ get "/subscription_manager" do |env| subscriptions = [] of InvidiousChannel user.subscriptions.each do |ucid| begin - subscriptions << get_channel(ucid, PG_DB, false) + subscriptions << get_channel(ucid, PG_DB, false, false) rescue ex next end @@ -1475,14 +1475,7 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions.select! do |ucid| - begin - get_channel(ucid, PG_DB, false, false) - true - rescue ex - false - end - end + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_freetube" @@ -1491,14 +1484,7 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions.select! do |ucid| - begin - get_channel(ucid, PG_DB, false, false) - true - rescue ex - false - end - end + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_newpipe_subscriptions" @@ -1508,13 +1494,7 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions.each do |ucid| - begin - get_channel(ucid, PG_DB, false, false) - rescue ex - next - end - end + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_newpipe" @@ -1533,14 +1513,7 @@ post "/data_control" do |env| user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map { |url| url.lchop("https://www.youtube.com/channel/") } user.subscriptions.uniq! - user.subscriptions.select! do |ucid| - begin - get_channel(ucid, PG_DB, false, false) - true - rescue ex - false - end - end + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index ba17ff07..c1915a10 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -21,6 +21,33 @@ class ChannelVideo }) end +def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) + active_threads = 0 + active_channel = Channel(String | Nil).new + + final = [] of String + channels.map do |ucid| + if active_threads >= max_threads + if response = active_channel.receive + active_threads -= 1 + final << response + end + end + + active_threads += 1 + spawn do + begin + get_channel(ucid, db, refresh, pull_all_videos) + active_channel.send(ucid) + rescue ex + active_channel.send(nil) + end + end + end + + return final +end + def get_channel(id, db, refresh = true, pull_all_videos = true) client = make_client(YT_URL) diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 38799a74..dbbe87f8 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -177,19 +177,16 @@ def fetch_user(sid, headers, db) feed = XML.parse_html(feed.body) channels = [] of String - feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).each do |channel| - if !{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"] - channel_id = channel["href"].lstrip("/channel/") - - begin - channel = get_channel(channel_id, db, false, false) - channels << channel.id - rescue ex - next - end + channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel| + if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"] + nil + else + channel["href"].lstrip("/channel/") end end + channels = get_batch_channels(channels, db, false, false) + email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"])) if email email = email.content.strip |