diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/invidious/channels.cr | 47 | ||||
| -rw-r--r-- | src/invidious/helpers/youtube_api.cr | 31 | ||||
| -rw-r--r-- | src/invidious/playlists.cr | 95 | ||||
| -rw-r--r-- | src/invidious/routes/playlists.cr | 9 | ||||
| -rw-r--r-- | src/invidious/views/playlist.ecr | 2 |
5 files changed, 114 insertions, 70 deletions
diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index b9808d98..9a129e1e 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -229,18 +229,18 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) page = 1 LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") - response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) + response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated) videos = [] of SearchVideo begin - initial_data = JSON.parse(response.body) + initial_data = JSON.parse(response_body) raise InfoException.new("Could not extract channel JSON") if !initial_data LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data") videos = extract_videos(initial_data.as_h, author, ucid) rescue ex - if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") || - response.body.includes?("https://www.google.com/sorry/index") + if response_body.includes?("To continue with your YouTube experience, please fill out the form below.") || + response_body.includes?("https://www.google.com/sorry/index") raise InfoException.new("Could not extract channel info. Instance is likely blocked.") end raise ex @@ -304,8 +304,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) ids = [] of String loop do - response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - initial_data = JSON.parse(response.body) + response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated) + initial_data = JSON.parse(response_body) raise InfoException.new("Could not extract channel JSON") if !initial_data videos = extract_videos(initial_data.as_h, author, ucid) @@ -447,6 +447,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end +# Used in bypass_captcha_job.cr def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" @@ -937,35 +938,19 @@ def get_about_info(ucid, locale) }) end -def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest", youtubei_browse = true) - if youtubei_browse - continuation = produce_channel_videos_continuation(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) - data = { - "context": { - "client": { - "clientName": "WEB", - "clientVersion": "2.20201021.03.00", - }, - }, - "continuation": continuation, - }.to_json - return YT_POOL.client &.post( - "/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", - headers: HTTP::Headers{"content-type" => "application/json"}, - body: data - ) - else - url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) - return YT_POOL.client &.get(url) - end +def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") + continuation = produce_channel_videos_continuation(ucid, page, + auto_generated: auto_generated, sort_by: sort_by, v2: true) + + return request_youtube_api_browse(continuation) end def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") videos = [] of SearchVideo 2.times do |i| - response = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) - initial_data = JSON.parse(response.body) + response_json = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) + initial_data = JSON.parse(response_json) break if !initial_data videos.concat extract_videos(initial_data.as_h, author, ucid) end @@ -974,8 +959,8 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") end def get_latest_videos(ucid) - response = get_channel_videos_response(ucid) - initial_data = JSON.parse(response.body) + response_json = get_channel_videos_response(ucid) + initial_data = JSON.parse(response_json) return [] of SearchVideo if !initial_data author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s items = extract_videos(initial_data.as_h, author, ucid) diff --git a/src/invidious/helpers/youtube_api.cr b/src/invidious/helpers/youtube_api.cr new file mode 100644 index 00000000..30413532 --- /dev/null +++ b/src/invidious/helpers/youtube_api.cr @@ -0,0 +1,31 @@ +# +# This file contains youtube API wrappers +# + +# Hard-coded constants required by the API +HARDCODED_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" +HARDCODED_CLIENT_VERS = "2.20210318.08.00" + +def request_youtube_api_browse(continuation) + # JSON Request data, required by the API + data = { + "context": { + "client": { + "hl": "en", + "gl": "US", + "clientName": "WEB", + "clientVersion": HARDCODED_CLIENT_VERS, + }, + }, + "continuation": continuation, + } + + # Send the POST request and return result + response = YT_POOL.client &.post( + "/youtubei/v1/browse?key=#{HARDCODED_API_KEY}", + headers: HTTP::Headers{"content-type" => "application/json"}, + body: data.to_json + ) + + return response.body +end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 0251a69c..71f6a9b8 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -307,23 +307,32 @@ def subscribe_playlist(db, user, playlist) return playlist end -def produce_playlist_url(id, index) +def produce_playlist_continuation(id, index) if id.starts_with? "UC" id = "UU" + id.lchop("UC") end plid = "VL" + id + # Emulate a "request counter" increment, to make perfectly valid + # ctokens, even if at the time of writing, it's ignored by youtube. + request_count = (index / 100).to_i64 || 1_i64 + data = {"1:varint" => index.to_i64} .try { |i| Protodec::Any.cast_json(i) } .try { |i| Protodec::Any.from_json(i) } .try { |i| Base64.urlsafe_encode(i, padding: false) } + data_wrapper = {"1:varint" => request_count, "15:string" => "PT:#{data}"} + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + object = { "80226972:embedded" => { - "2:string" => plid, - "3:base64" => { - "15:string" => "PT:#{data}", - }, + "2:string" => plid, + "3:string" => data_wrapper, + "35:string" => id, }, } @@ -332,7 +341,7 @@ def produce_playlist_url(id, index) .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" + return continuation end def get_playlist(db, plid, locale, refresh = true, force_refresh = false) @@ -427,47 +436,59 @@ def fetch_playlist(plid, locale) end def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) - if playlist.is_a? InvidiousPlaylist - db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", playlist.id, playlist.index, offset, as: PlaylistVideo) - else - fetch_playlist_videos(playlist.id, playlist.video_count, offset, locale, continuation) - end -end - -def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil) - if continuation - response = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en") - initial_data = extract_initial_data(response.body) - offset = initial_data["currentVideoEndpoint"]?.try &.["watchEndpoint"]?.try &.["index"]?.try &.as_i64 || offset - end - - if video_count > 100 - url = produce_playlist_url(plid, offset) - - response = YT_POOL.client &.get(url) - initial_data = JSON.parse(response.body).as_a.find(&.as_h.["response"]?).try &.as_h - elsif offset > 100 + # Show empy playlist if requested page is out of range + if offset >= playlist.video_count return [] of PlaylistVideo - else # Extract first page of videos - response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en") - initial_data = extract_initial_data(response.body) end - return [] of PlaylistVideo if !initial_data - videos = extract_playlist_videos(initial_data) + if playlist.is_a? InvidiousPlaylist + db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", + playlist.id, playlist.index, offset, as: PlaylistVideo) + else + if offset >= 100 + # Normalize offset to match youtube's behavior (100 videos chunck per request) + offset = (offset / 100).to_i64 * 100_i64 + + ctoken = produce_playlist_continuation(playlist.id, offset) + initial_data = JSON.parse(request_youtube_api_browse(ctoken)).as_h + else + response = YT_POOL.client &.get("/playlist?list=#{playlist.id}&gl=US&hl=en") + initial_data = extract_initial_data(response.body) + end - until videos.empty? || videos[0].index == offset - videos.shift + if initial_data + return extract_playlist_videos(initial_data) + else + return [] of PlaylistVideo + end end - - return videos end def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) videos = [] of PlaylistVideo - (initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"].as_a || - initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a).try &.each do |item| + if initial_data["contents"]? + tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] + tabs_renderer = tabs.as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"] + + # Watch out the two versions, with and without "s" + if tabs_renderer["contents"]? || tabs_renderer["content"]? + # Initial playlist data + tabs_contents = tabs_renderer.["contents"]? || tabs_renderer.["content"] + + list_renderer = tabs_contents.["sectionListRenderer"]["contents"][0] + item_renderer = list_renderer.["itemSectionRenderer"]["contents"][0] + contents = item_renderer.["playlistVideoListRenderer"]["contents"].as_a + else + # Continuation data + contents = initial_data["onResponseReceivedActions"][0]? + .try &.["appendContinuationItemsAction"]["continuationItems"].as_a + end + else + contents = initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a + end + + contents.try &.each do |item| if i = item["playlistVideoRenderer"]? video_id = i["navigationEndpoint"]["watchEndpoint"]["videoId"].as_s plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index c5023c08..73c14155 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -433,6 +433,13 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute return error_template(500, ex) end + page_count = (playlist.video_count / 100).to_i + page_count = 1 if page_count == 0 + + if page > page_count + return env.redirect "/playlist?list=#{plid}&page=#{page_count}" + end + if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email return error_template(403, "This playlist is private.") end @@ -440,7 +447,7 @@ class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute begin videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) rescue ex - videos = [] of PlaylistVideo + return error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}") end if playlist.author == user.try &.email diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index bb721c3a..91156028 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -103,7 +103,7 @@ </div> <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if videos.size == 100 %> + <% if page_count != 1 && page < page_count %> <a href="/playlist?list=<%= playlist.id %>&page=<%= page + 1 %>"> <%= translate(locale, "Next page") %> </a> |