diff options
Diffstat (limited to 'src')
25 files changed, 433 insertions, 195 deletions
diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 13af2d8b..ad786f3a 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -1,3 +1,5 @@ +private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000} + # TODO: Add "sort_by" def fetch_channel_community(ucid, continuation, locale, format, thin_mode) response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en") @@ -29,18 +31,16 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) session_token: session_token, } - response = YT_POOL.client &.post("/comment_service_ajax?action_get_comments=1&ctoken=#{continuation}&continuation=#{continuation}&hl=en&gl=US", headers, form: post_req) - body = JSON.parse(response.body) + body = YoutubeAPI.browse(continuation) - body = body["response"]["continuationContents"]["itemSectionContinuation"]? || - body["response"]["continuationContents"]["backstageCommentsContinuation"]? + body = body.dig?("continuationContents", "itemSectionContinuation") || + body.dig?("continuationContents", "backstageCommentsContinuation") if !body raise InfoException.new("Could not extract continuation.") end end - continuation = body["continuations"]?.try &.[0]["nextContinuationData"]["continuation"].as_s posts = body["contents"].as_a if message = posts[0]["messageRenderer"]? @@ -108,6 +108,8 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"] .try &.as_s.gsub(/\D/, "").to_i? || 0 + reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0") + json.field "content", html_to_content(content_html) json.field "contentHtml", content_html @@ -115,6 +117,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) json.field "likeCount", like_count + json.field "replyCount", reply_count json.field "commentId", post["postId"]? || post["commentId"]? || "" json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid @@ -174,9 +177,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) aspect_ratio = (width.to_f / height.to_f) url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") - qualities = {320, 560, 640, 1280, 2000} - - qualities.each do |quality| + IMAGE_QUALITIES.each do |quality| json.object do json.field "url", url.gsub(/=s\d+/, "=s#{quality}") json.field "width", quality @@ -185,10 +186,39 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) end end end - # TODO - # when .has_key?("pollRenderer") - # attachment = attachment["pollRenderer"] - # json.field "type", "poll" + when .has_key?("pollRenderer") + attachment = attachment["pollRenderer"] + json.field "type", "poll" + json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0]) + json.field "choices" do + json.array do + attachment["choices"].as_a.each do |choice| + json.object do + json.field "text", choice.dig("text", "runs", 0, "text").as_s + # A choice can have an image associated with it. + # Ex post: https://www.youtube.com/post/UgkxD4XavXUD4NQiddJXXdohbwOwcVqrH9Re + if choice["image"]? + thumbnail = choice["image"]["thumbnails"][0].as_h + width = thumbnail["width"].as_i + height = thumbnail["height"].as_i + aspect_ratio = (width.to_f / height.to_f) + url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") + json.field "image" do + json.array do + IMAGE_QUALITIES.each do |quality| + json.object do + json.field "url", url.gsub(/=s\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", (quality / aspect_ratio).ceil.to_i + end + end + end + end + end + end + end + end + end when .has_key?("postMultiImageRenderer") attachment = attachment["postMultiImageRenderer"] json.field "type", "multiImage" @@ -202,9 +232,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) aspect_ratio = (width.to_f / height.to_f) url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") - qualities = {320, 560, 640, 1280, 2000} - - qualities.each do |quality| + IMAGE_QUALITIES.each do |quality| json.object do json.field "url", url.gsub(/=s\d+/, "=s#{quality}") json.field "width", quality @@ -240,10 +268,8 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) end end end - - if body["continuations"]? - continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s - json.field "continuation", extract_channel_community_cursor(continuation) + if cont = posts.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") + json.field "continuation", extract_channel_community_cursor(cont.as_s) end end end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index befec03d..12ed4a7d 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -30,7 +30,9 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so "15:embedded" => { "1:embedded" => { "1:string" => object_inner_2_encoded, - "2:string" => "00000000-0000-0000-0000-000000000000", + }, + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", }, "3:varint" => sort_by_numerical, }, @@ -127,38 +129,15 @@ module Invidious::Channel::Tabs # Shorts # ------------------- - private def fetch_shorts_data(ucid : String, continuation : String? = nil) + def get_shorts(channel : AboutChannel, continuation : String? = nil) if continuation.nil? # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts" # TODO: try to extract the continuation tokens that allows other sorting options - return YoutubeAPI.browse(ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") + initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") else - return YoutubeAPI.browse(continuation: continuation) - end - end - - def get_shorts(channel : AboutChannel, continuation : String? = nil) - initial_data = self.fetch_shorts_data(channel.ucid, continuation) - - begin - # Try to parse the initial data fetched above - return extract_items(initial_data, channel.author, channel.ucid) - rescue ex : RetryOnceException - # Sometimes, for a completely unknown reason, the "reelItemRenderer" - # object is missing some critical information (it happens once in about - # 20 subsequent requests). Refreshing the page is required to properly - # show the "shorts" tab. - # - # In order to make the experience smoother for the user, we simulate - # said page refresh by fetching again the JSON. If that still doesn't - # work, we raise a BrokenTubeException, as something is really broken. - begin - initial_data = self.fetch_shorts_data(channel.ucid, continuation) - return extract_items(initial_data, channel.author, channel.ucid) - rescue ex : RetryOnceException - raise BrokenTubeException.new "reelPlayerHeaderSupportedRenderers" - end + initial_data = YoutubeAPI.browse(continuation: continuation) end + return extract_items(initial_data, channel.author, channel.ucid) end # ------------------- diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 357a461c..ec4449f0 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -182,7 +182,11 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b json.field "contentHtml", content_html json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil) - + json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil) + if node_comment["sponsorCommentBadge"]? + # Sponsor icon thumbnails always have one object and there's only ever the url property in it + json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s + end json.field "published", published.to_unix json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) @@ -324,21 +328,32 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false) end author_name = HTML.escape(child["author"].as_s) + sponsor_icon = "" if child["verified"]?.try &.as_bool && child["authorIsChannelOwner"]?.try &.as_bool author_name += " <i class=\"icon ion ion-md-checkmark-circle\"></i>" elsif child["verified"]?.try &.as_bool author_name += " <i class=\"icon ion ion-md-checkmark\"></i>" end + + if child["isSponsor"]?.try &.as_bool + sponsor_icon = String.build do |str| + str << %(<img alt="" ) + str << %(src="/ggpht) << URI.parse(child["sponsorIconUrl"].as_s).request_target << "\" " + str << %(title=") << translate(locale, "Channel Sponsor") << "\" " + str << %(width="16" height="16" />) + end + end html << <<-END_HTML <div class="pure-g" style="width:100%"> <div class="channel-profile pure-u-4-24 pure-u-md-2-24"> - <img loading="lazy" style="margin-right:1em;margin-top:1em;width:90%" src="#{author_thumbnail}"> + <img loading="lazy" style="margin-right:1em;margin-top:1em;width:90%" src="#{author_thumbnail}" alt="" /> </div> <div class="pure-u-20-24 pure-u-md-22-24"> <p> <b> <a class="#{child["authorIsChannelOwner"] == true ? "channel-owner" : ""}" href="#{child["authorUrl"]}">#{author_name}</a> </b> + #{sponsor_icon} <p style="white-space:pre-wrap">#{child["contentHtml"]}</p> END_HTML @@ -352,7 +367,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false) html << <<-END_HTML <div class="pure-g"> <div class="pure-u-1 pure-u-md-1-2"> - <img loading="lazy" style="width:100%" src="/ggpht#{URI.parse(attachment["url"].as_s).request_target}"> + <img loading="lazy" style="width:100%" src="/ggpht#{URI.parse(attachment["url"].as_s).request_target}" alt="" /> </div> </div> END_HTML @@ -413,7 +428,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false) html << <<-END_HTML <span class="creator-heart-container" title="#{translate(locale, "`x` marked it with a ❤", child["creatorHeart"]["creatorName"].as_s)}"> <div class="creator-heart"> - <img loading="lazy" class="creator-heart-background-hearted" src="#{creator_thumbnail}"></img> + <img loading="lazy" class="creator-heart-background-hearted" src="#{creator_thumbnail}" alt="" /> <div class="creator-heart-small-hearted"> <div class="icon ion-ios-heart creator-heart-small-container"></div> </div> @@ -589,7 +604,7 @@ def text_to_parsed_content(text : String) : JSON::Any currentNode = {"text" => urlMatch[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => urlMatch[0]}}} currentNodes << (JSON.parse(currentNode.to_json)) # If text remain after match create new simple node with text after match - afterNode = {"text" => splittedLastNode.size > 0 ? splittedLastNode[1] : ""} + afterNode = {"text" => splittedLastNode.size > 1 ? splittedLastNode[1] : ""} currentNodes << (JSON.parse(afterNode.to_json)) end @@ -620,61 +635,35 @@ def content_to_comment_html(content, video_id : String? = "") text = HTML.escape(run["text"].as_s) - if run["navigationEndpoint"]? - if url = run["navigationEndpoint"]["urlEndpoint"]?.try &.["url"].as_s - url = URI.parse(url) - displayed_url = text - - if url.host == "youtu.be" - url = "/watch?v=#{url.request_target.lstrip('/')}" - elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") - if url.path == "/redirect" - # Sometimes, links can be corrupted (why?) so make sure to fallback - # nicely. See https://github.com/iv-org/invidious/issues/2682 - url = url.query_params["q"]? || "" - displayed_url = url - else - url = url.request_target - displayed_url = "youtube.com#{url}" - end - end - - text = %(<a href="#{url}">#{reduce_uri(displayed_url)}</a>) - elsif watch_endpoint = run["navigationEndpoint"]["watchEndpoint"]? - start_time = watch_endpoint["startTimeSeconds"]?.try &.as_i - link_video_id = watch_endpoint["videoId"].as_s - - url = "/watch?v=#{link_video_id}" - url += "&t=#{start_time}" if !start_time.nil? - - # If the current video ID (passed through from the caller function) - # is the same as the video ID in the link, add HTML attributes for - # the JS handler function that bypasses page reload. - # - # See: https://github.com/iv-org/invidious/issues/3063 - if link_video_id == video_id - start_time ||= 0 - text = %(<a href="#{url}" data-onclick="jump_to_time" data-jump-time="#{start_time}">#{reduce_uri(text)}</a>) - else - text = %(<a href="#{url}">#{text}</a>) - end - elsif url = run.dig?("navigationEndpoint", "commandMetadata", "webCommandMetadata", "url").try &.as_s - if text.starts_with?(/\s?[@#]/) - # Handle "pings" in comments and hasthags differently - # See: - # - https://github.com/iv-org/invidious/issues/3038 - # - https://github.com/iv-org/invidious/issues/3062 - text = %(<a href="#{url}">#{text}</a>) - else - text = %(<a href="#{url}">#{reduce_uri(url)}</a>) - end - end + if navigationEndpoint = run.dig?("navigationEndpoint") + text = parse_link_endpoint(navigationEndpoint, text, video_id) end text = "<b>#{text}</b>" if run["bold"]? text = "<s>#{text}</s>" if run["strikethrough"]? text = "<i>#{text}</i>" if run["italics"]? + # check for custom emojis + if run["emoji"]? + if run["emoji"]["isCustomEmoji"]?.try &.as_bool + if emojiImage = run.dig?("emoji", "image") + emojiAlt = emojiImage.dig?("accessibility", "accessibilityData", "label").try &.as_s || text + emojiThumb = emojiImage["thumbnails"][0] + text = String.build do |str| + str << %(<img alt=") << emojiAlt << "\" " + str << %(src="/ggpht) << URI.parse(emojiThumb["url"].as_s).request_target << "\" " + str << %(title=") << emojiAlt << "\" " + str << %(width=") << emojiThumb["width"] << "\" " + str << %(height=") << emojiThumb["height"] << "\" " + str << %(class="channel-emoji" />) + end + else + # Hide deleted channel emoji + text = "" + end + end + end + text end diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index a9b00860..e3214469 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -20,7 +20,7 @@ module Invidious::Frontend::WatchPage def download_widget(locale : String, video : Video, video_assets : VideoAssets) : String if CONFIG.disabled?("downloads") - return "<p id=\"download\">#{translate(locale, "Download is disabled.")}</p>" + return "<p id=\"download\">#{translate(locale, "Download is disabled")}</p>" end return String.build(4000) do |str| diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 500a2582..bcf7c963 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -389,3 +389,56 @@ def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = " end return str end + +# Get the html link from a NavigationEndpoint or an innertubeCommand +def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String) + if url = endpoint.dig?("urlEndpoint", "url").try &.as_s + url = URI.parse(url) + displayed_url = text + + if url.host == "youtu.be" + url = "/watch?v=#{url.request_target.lstrip('/')}" + elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") + if url.path == "/redirect" + # Sometimes, links can be corrupted (why?) so make sure to fallback + # nicely. See https://github.com/iv-org/invidious/issues/2682 + url = url.query_params["q"]? || "" + displayed_url = url + else + url = url.request_target + displayed_url = "youtube.com#{url}" + end + end + + text = %(<a href="#{url}">#{reduce_uri(displayed_url)}</a>) + elsif watch_endpoint = endpoint.dig?("watchEndpoint") + start_time = watch_endpoint["startTimeSeconds"]?.try &.as_i + link_video_id = watch_endpoint["videoId"].as_s + + url = "/watch?v=#{link_video_id}" + url += "&t=#{start_time}" if !start_time.nil? + + # If the current video ID (passed through from the caller function) + # is the same as the video ID in the link, add HTML attributes for + # the JS handler function that bypasses page reload. + # + # See: https://github.com/iv-org/invidious/issues/3063 + if link_video_id == video_id + start_time ||= 0 + text = %(<a href="#{url}" data-onclick="jump_to_time" data-jump-time="#{start_time}">#{reduce_uri(text)}</a>) + else + text = %(<a href="#{url}">#{text}</a>) + end + elsif url = endpoint.dig?("commandMetadata", "webCommandMetadata", "url").try &.as_s + if text.starts_with?(/\s?[@#]/) + # Handle "pings" in comments and hasthags differently + # See: + # - https://github.com/iv-org/invidious/issues/3038 + # - https://github.com/iv-org/invidious/issues/3062 + text = %(<a href="#{url}">#{text}</a>) + else + text = %(<a href="#{url}">#{reduce_uri(url)}</a>) + end + end + return text +end diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index a2b1a35c..fe4b5223 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -197,6 +197,21 @@ module Invidious::JSONify::APIv1 end end + if !video.music.empty? + json.field "musicTracks" do + json.array do + video.music.each do |music| + json.object do + json.field "song", music.song + json.field "artist", music.artist + json.field "album", music.album + json.field "license", music.license + end + end + end + end + end + json.field "recommendedVideos" do json.array do video.related_videos.each do |rv| diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 3f342b92..823ca85b 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -97,7 +97,7 @@ def template_mix(mix) <li class="pure-menu-item"> <a href="/watch?v=#{video["videoId"]}&list=#{mix["mixId"]}"> <div class="thumbnail"> - <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg"> + <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" /> <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p> </div> <p style="width:100%">#{video["title"]}</p> diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 57f1f53e..013be268 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -507,7 +507,7 @@ def template_playlist(playlist) <li class="pure-menu-item" id="#{video["videoId"]}"> <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}"> <div class="thumbnail"> - <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg"> + <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" /> <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p> </div> <p style="width:100%">#{video["title"]}</p> diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index 8f69df94..5aa4452c 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -262,7 +262,7 @@ module Invidious::Routes::Account end query["token"] = access_token - query["username"] = user.email + query["username"] = URI.encode_path_segment(user.email) url.query = query.to_s env.redirect url.to_s diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 6b935312..ce2ee812 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -54,6 +54,65 @@ module Invidious::Routes::API::V1::Authenticated env.response.status_code = 204 end + def self.get_history(env) + env.response.content_type = "application/json" + user = env.get("user").as(User) + + page = env.params.query["page"]?.try &.to_i?.try &.clamp(0, Int32::MAX) + page ||= 1 + + max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) + max_results ||= user.preferences.max_results + max_results ||= CONFIG.default_user_preferences.max_results + + start_index = (page - 1) * max_results + if user.watched[start_index]? + watched = user.watched.reverse[start_index, max_results] + end + watched ||= [] of String + + return watched.to_json + end + + def self.mark_watched(env) + user = env.get("user").as(User) + + if !user.preferences.watch_history + return error_json(409, "Watch history is disabled in preferences.") + end + + id = env.params.url["id"] + if !id.match(/^[a-zA-Z0-9_-]{11}$/) + return error_json(400, "Invalid video id.") + end + + Invidious::Database::Users.mark_watched(user, id) + env.response.status_code = 204 + end + + def self.mark_unwatched(env) + user = env.get("user").as(User) + + if !user.preferences.watch_history + return error_json(409, "Watch history is disabled in preferences.") + end + + id = env.params.url["id"] + if !id.match(/^[a-zA-Z0-9_-]{11}$/) + return error_json(400, "Invalid video id.") + end + + Invidious::Database::Users.mark_unwatched(user, id) + env.response.status_code = 204 + end + + def self.clear_history(env) + user = env.get("user").as(User) + + Invidious::Database::Users.clear_watch_history(user) + env.response.status_code = 204 + end + def self.feed(env) env.response.content_type = "application/json" diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 1e932d11..9641e01a 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -256,7 +256,7 @@ module Invidious::Routes::VideoPlayback return error_template(400, "Invalid video ID") end - if itag.nil? || itag <= 0 || itag >= 1000 + if !itag.nil? && (itag <= 0 || itag >= 1000) return error_template(400, "Invalid itag") end @@ -277,7 +277,11 @@ module Invidious::Routes::VideoPlayback return error_template(500, ex) end - fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } + if itag.nil? + fmt = video.fmt_stream[-1]? + else + fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } + end url = fmt.try &.["url"]?.try &.as_s if !url diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index dca2f117..9e2ade3d 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -257,6 +257,11 @@ module Invidious::Routing get "/api/v1/auth/export/invidious", {{namespace}}::Authenticated, :export_invidious post "/api/v1/auth/import/invidious", {{namespace}}::Authenticated, :import_invidious + get "/api/v1/auth/history", {{namespace}}::Authenticated, :get_history + post "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_watched + delete "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_unwatched + delete "/api/v1/auth/history", {{namespace}}::Authenticated, :clear_history + get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr index 7e909590..25edb936 100644 --- a/src/invidious/search/processors.cr +++ b/src/invidious/search/processors.cr @@ -10,7 +10,7 @@ module Invidious::Search initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) items, _ = extract_items(initial_data) - return items + return items.reject!(Category) end # Search a youtube channel @@ -32,7 +32,7 @@ module Invidious::Search response_json = YoutubeAPI.browse(continuation) items, _ = extract_items(response_json, "", ucid) - return items + return items.reject!(Category) end # Search inside of user subscriptions diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index 24e79609..e38845d9 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -113,7 +113,7 @@ module Invidious::Search case @type when .regular?, .playlist? - items = unnest_items(Processors.regular(self)) + items = Processors.regular(self) # when .channel? items = Processors.channel(self) @@ -136,26 +136,5 @@ module Invidious::Search return params end - - # TODO: clean code - private def unnest_items(all_items) : Array(SearchItem) - items = [] of SearchItem - - # Light processing to flatten search results out of Categories. - # They should ideally be supported in the future. - all_items.each do |i| - if i.is_a? Category - i.contents.each do |nest_i| - if !nest_i.is_a? Video - items << nest_i - end - end - else - items << i - end - end - - return items - end end end diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 134eb437..2d9f8a83 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -17,7 +17,24 @@ def fetch_trending(trending_type, region, locale) client_config = YoutubeAPI::ClientConfig.new(region: region) initial_data = YoutubeAPI.browse("FEtrending", params: params, client_config: client_config) - trending = extract_videos(initial_data) - return {trending, plid} + items, _ = extract_items(initial_data) + + extracted = [] of SearchItem + + items.each do |itm| + if itm.is_a?(Category) + # Ignore the smaller categories, as they generally contain a sponsored + # channel, which brings a lot of noise on the trending page. + # See: https://github.com/iv-org/invidious/issues/2989 + next if itm.contents.size < 24 + + extracted.concat extract_category(itm) + else + extracted << itm + end + end + + # Deduplicate items before returning results + return extracted.select(SearchVideo).uniq!(&.id), plid end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 436ac82d..0038a97a 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -249,7 +249,12 @@ struct Video def music : Array(VideoMusic) info["music"].as_a.map { |music_json| - VideoMusic.new(music_json["album"].as_s, music_json["artist"].as_s, music_json["license"].as_s) + VideoMusic.new( + music_json["song"].as_s, + music_json["album"].as_s, + music_json["artist"].as_s, + music_json["license"].as_s + ) } end diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr new file mode 100644 index 00000000..542cb416 --- /dev/null +++ b/src/invidious/videos/description.cr @@ -0,0 +1,64 @@ +require "json" +require "uri" + +private def copy_string(str : String::Builder, iter : Iterator, count : Int) : Int + copied = 0 + while copied < count + cp = iter.next + break if cp.is_a?(Iterator::Stop) + + str << cp.chr + + # A codepoint from the SMP counts twice + copied += 1 if cp > 0xFFFF + copied += 1 + end + + return copied +end + +def parse_description(desc, video_id : String) : String? + return "" if desc.nil? + + content = desc["content"].as_s + return "" if content.empty? + + commands = desc["commandRuns"]?.try &.as_a + return content if commands.nil? + + # Not everything is stored in UTF-8 on youtube's side. The SMP codepoints + # (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are + # automatically decoded by the JSON parser. It means that we need to count + # copied byte in a special manner, preventing the use of regular string copy. + iter = content.each_codepoint + + index = 0 + + return String.build do |str| + commands.each do |command| + cmd_start = command["startIndex"].as_i + cmd_length = command["length"].as_i + + # Copy the text chunk between this command and the previous if needed. + length = cmd_start - index + index += copy_string(str, iter, length) + + # We need to copy the command's text using the iterator + # and the special function defined above. + cmd_content = String.build(cmd_length) do |str2| + copy_string(str2, iter, cmd_length) + end + + link = cmd_content + if on_tap = command.dig?("onTap", "innertubeCommand") + link = parse_link_endpoint(on_tap, cmd_content, video_id) + end + str << link + index += cmd_length + end + + # Copy the end of the string (past the last command). + remaining_length = content.size - index + copy_string(str, iter, remaining_length) if remaining_length > 0 + end +end diff --git a/src/invidious/videos/music.cr b/src/invidious/videos/music.cr index 402ae46f..08d88a3e 100644 --- a/src/invidious/videos/music.cr +++ b/src/invidious/videos/music.cr @@ -3,10 +3,11 @@ require "json" struct VideoMusic include JSON::Serializable + property song : String property album : String property artist : String property license : String - def initialize(@album : String, @artist : String, @license : String) + def initialize(@song : String, @album : String, @artist : String, @license : String) end end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index cf43f1be..2e8eecc3 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -185,10 +185,12 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # We have to try to extract viewCount from videoPrimaryInfoRenderer first, # then from videoDetails, as the latter is "0" for livestreams (we want # to get the amount of viewers watching). - views_txt = video_primary_renderer - .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text") - views_txt ||= video_details["viewCount"]? - views = views_txt.try &.as_s.gsub(/\D/, "").to_i64? + views_txt = extract_text( + video_primary_renderer + .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount") + ) + views_txt ||= video_details["viewCount"]?.try &.as_s || "" + views = views_txt.gsub(/\D/, "").to_i64? length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) .try &.as_s.to_i64 @@ -282,8 +284,10 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any description = microformat.dig?("description", "simpleText").try &.as_s || "" short_description = player_response.dig?("videoDetails", "shortDescription") - description_html = video_secondary_renderer.try &.dig?("description", "runs") - .try &.as_a.try { |t| content_to_comment_html(t, video_id) } + # description_html = video_secondary_renderer.try &.dig?("description", "runs") + # .try &.as_a.try { |t| content_to_comment_html(t, video_id) } + + description_html = parse_description(video_secondary_renderer.try &.dig?("attributedDescription"), video_id) # Video metadata @@ -325,17 +329,28 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any album = nil music_license = nil + # Used when the video has multiple songs + if song_title = music_desc.dig?("carouselLockupRenderer", "videoLockup", "compactVideoRenderer", "title") + # "simpleText" for plain text / "runs" when song has a link + song = song_title["simpleText"]? || song_title.dig?("runs", 0, "text") + + # some videos can have empty tracks. See: https://www.youtube.com/watch?v=eBGIQ7ZuuiU + next if !song + end + music_desc.dig?("carouselLockupRenderer", "infoRows").try &.as_a.each do |desc| desc_title = extract_text(desc.dig?("infoRowRenderer", "title")) if desc_title == "ARTIST" artist = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata")) + elsif desc_title == "SONG" + song = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata")) elsif desc_title == "ALBUM" album = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata")) elsif desc_title == "LICENSES" music_license = extract_text(desc.dig?("infoRowRenderer", "expandedMetadata")) end end - music_list << VideoMusic.new(album.to_s, artist.to_s, music_license.to_s) + music_list << VideoMusic.new(song.to_s, album.to_s, artist.to_s, music_license.to_s) end # Author infos diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr index f216359f..59888760 100644 --- a/src/invidious/views/components/channel_info.ecr +++ b/src/invidious/views/components/channel_info.ecr @@ -1,6 +1,6 @@ <% if channel.banner %> <div class="h-box"> - <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>"> + <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>" alt="" /> </div> <div class="h-box"> @@ -11,7 +11,7 @@ <div class="pure-g h-box"> <div class="pure-u-2-3"> <div class="channel-profile"> - <img src="/ggpht<%= channel_profile_pic %>"> + <img src="/ggpht<%= channel_profile_pic %>" alt="" /> <span><%= author %></span><% if !channel.verified.nil? && channel.verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %> </div> </div> diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index fa12374f..7cfd38db 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -7,7 +7,7 @@ <a href="/channel/<%= item.ucid %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <center> - <img loading="lazy" tabindex="-1" style="width:56.25%" src="/ggpht<%= URI.parse(item.author_thumbnail).request_target.gsub(/=s\d+/, "=s176") %>"/> + <img loading="lazy" tabindex="-1" style="width:56.25%" src="/ggpht<%= URI.parse(item.author_thumbnail).request_target.gsub(/=s\d+/, "=s176") %>" alt="" /> </center> <% end %> <p dir="auto"><%= HTML.escape(item.author) %><% if !item.author_verified.nil? && item.author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></p> @@ -25,7 +25,7 @@ <a style="width:100%" href="<%= url %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img loading="lazy" tabindex="-1" class="thumbnail" src="<%= URI.parse(item.thumbnail || "/").request_target %>"/> + <img loading="lazy" tabindex="-1" class="thumbnail" src="<%= URI.parse(item.thumbnail || "/").request_target %>" alt="" /> <p class="length"><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %></p> </div> <% end %> @@ -38,7 +38,7 @@ <a href="/watch?v=<%= item.id %>&list=<%= item.rdid %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img loading="lazy" tabindex="-1" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> + <img loading="lazy" tabindex="-1" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg" alt="" /> <% if item.length_seconds != 0 %> <p class="length"><%= recode_length_seconds(item.length_seconds) %></p> <% end %> @@ -58,7 +58,7 @@ <a style="width:100%" href="/watch?v=<%= item.id %>&list=<%= item.plid %>&index=<%= item.index %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img loading="lazy" tabindex="-1" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> + <img loading="lazy" tabindex="-1" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg" alt="" /> <% if plid_form = env.get?("remove_playlist_items") %> <form data-onsubmit="return_false" action="/playlist_ajax?action_remove_video=1&set_video_id=<%= item.index %>&playlist_id=<%= plid_form %>&referer=<%= env.get("current_page") %>" method="post"> @@ -112,7 +112,7 @@ <a style="width:100%" href="/watch?v=<%= item.id %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img loading="lazy" tabindex="-1" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> + <img loading="lazy" tabindex="-1" class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg" alt="" /> <% if env.get? "show_watched" %> <form data-onsubmit="return_false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>"> diff --git a/src/invidious/views/feeds/history.ecr b/src/invidious/views/feeds/history.ecr index 471d21db..2234b297 100644 --- a/src/invidious/views/feeds/history.ecr +++ b/src/invidious/views/feeds/history.ecr @@ -34,7 +34,7 @@ <a style="width:100%" href="/watch?v=<%= item %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img class="thumbnail" src="/vi/<%= item %>/mqdefault.jpg"/> + <img class="thumbnail" src="/vi/<%= item %>/mqdefault.jpg" alt="" /> <form data-onsubmit="return_false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <p class="watched"> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 666eb3b0..5b3190f3 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -181,7 +181,11 @@ we're going to need to do it here in order to allow for translations. <% end %> </p> <% if video.license %> - <p id="license"><%= translate(locale, "License: ") %><%= video.license %></p> + <% if video.license.empty? %> + <p id="license"><%= translate(locale, "License: ") %><%= translate(locale, "Standard YouTube license") %></p> + <% else %> + <p id="license"><%= translate(locale, "License: ") %><%= video.license %></p> + <% end %> <% end %> <p id="family_friendly"><%= translate(locale, "Family friendly? ") %><%= translate_bool(locale, video.is_family_friendly) %></p> <p id="wilson" style="display: none; visibility: hidden;"></p> @@ -204,7 +208,7 @@ we're going to need to do it here in order to allow for translations. <a href="/channel/<%= video.ucid %>" style="display:block;width:fit-content;width:-moz-fit-content"> <div class="channel-profile"> <% if !video.author_thumbnail.empty? %> - <img src="/ggpht<%= URI.parse(video.author_thumbnail).request_target %>"> + <img src="/ggpht<%= URI.parse(video.author_thumbnail).request_target %>" alt="" /> <% end %> <span id="channel-name"><%= author %><% if !video.author_verified.nil? && video.author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></span> </div> @@ -248,9 +252,9 @@ we're going to need to do it here in order to allow for translations. <div id="music-description-box"> <% video.music.each do |music| %> <div class="music-item"> - <p id="music-artist"><%= translate(locale, "Artist: ") %><%= music.artist %></p> - <p id="music-album"><%= translate(locale, "Album: ") %><%= music.album %></p> - <p id="music-license"><%= translate(locale, "License: ") %><%= music.license %></p> + <p class="music-song"><%= translate(locale, "Song: ") %><%= music.song %></p> + <p class="music-artist"><%= translate(locale, "Artist: ") %><%= music.artist %></p> + <p class="music-album"><%= translate(locale, "Album: ") %><%= music.album %></p> </div> <% end %> </div> @@ -294,7 +298,7 @@ we're going to need to do it here in order to allow for translations. <a href="/watch?v=<%= rv["id"] %>&listen=<%= params.listen %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> - <img loading="lazy" class="thumbnail" src="/vi/<%= rv["id"] %>/mqdefault.jpg"> + <img loading="lazy" class="thumbnail" src="/vi/<%= rv["id"] %>/mqdefault.jpg" alt="" /> <p class="length"><%= recode_length_seconds(rv["length_seconds"]?.try &.to_i? || 0) %></p> </div> <% end %> diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index b14ad7b9..9c041361 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -18,6 +18,7 @@ private ITEM_PARSERS = { Parsers::CategoryRendererParser, Parsers::RichItemRendererParser, Parsers::ReelItemRendererParser, + Parsers::ItemSectionRendererParser, Parsers::ContinuationItemRendererParser, } @@ -377,6 +378,30 @@ private module Parsers end end + # Parses an InnerTube itemSectionRenderer into a SearchVideo. + # Returns nil when the given object isn't a ItemSectionRenderer + # + # A itemSectionRenderer seems to be a simple wrapper for a videoRenderer, used + # by the result page for channel searches. It is located inside a continuationItems + # container.It is very similar to RichItemRendererParser + # + module ItemSectionRendererParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item.dig?("itemSectionRenderer", "contents", 0) + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + child = VideoRendererParser.process(item_contents, author_fallback) + return child + end + + def self.parser_name + return {{@type.name}} + end + end + # Parses an InnerTube richItemRenderer into a SearchVideo. # Returns nil when the given object isn't a RichItemRenderer # @@ -423,44 +448,43 @@ private module Parsers "overlay", "reelPlayerOverlayRenderer" ) - # Sometimes, the "reelPlayerOverlayRenderer" object is missing the - # important part of the response. We use this exception to tell - # the calling function to fetch the content again. - if !reel_player_overlay.as_h.has_key?("reelPlayerHeaderSupportedRenderers") - raise RetryOnceException.new - end - - video_details_container = reel_player_overlay.dig( - "reelPlayerHeaderSupportedRenderers", - "reelPlayerHeaderRenderer" - ) - - # Author infos + if video_details_container = reel_player_overlay.dig?( + "reelPlayerHeaderSupportedRenderers", + "reelPlayerHeaderRenderer" + ) + # Author infos - author = video_details_container - .dig?("channelTitleText", "runs", 0, "text") - .try &.as_s || author_fallback.name + author = video_details_container + .dig?("channelTitleText", "runs", 0, "text") + .try &.as_s || author_fallback.name - ucid = video_details_container - .dig?("channelNavigationEndpoint", "browseEndpoint", "browseId") - .try &.as_s || author_fallback.id + ucid = video_details_container + .dig?("channelNavigationEndpoint", "browseEndpoint", "browseId") + .try &.as_s || author_fallback.id - # Title & publication date + # Title & publication date - title = video_details_container.dig?("reelTitleText") - .try { |t| extract_text(t) } || "" + title = video_details_container.dig?("reelTitleText") + .try { |t| extract_text(t) } || "" - published = video_details_container - .dig?("timestampText", "simpleText") - .try { |t| decode_date(t.as_s) } || Time.utc + published = video_details_container + .dig?("timestampText", "simpleText") + .try { |t| decode_date(t.as_s) } || Time.utc + # View count + view_count_text = video_details_container.dig?("viewCountText", "simpleText") + else + author = author_fallback.name + ucid = author_fallback.id + published = Time.utc + title = item_contents.dig?("headline", "simpleText").try &.as_s || "" + end # View count # View count used to be in the reelWatchEndpoint, but that changed? - view_count_text = item_contents.dig?("viewCountText", "simpleText") - view_count_text ||= video_details_container.dig?("viewCountText", "simpleText") + view_count_text ||= item_contents.dig?("viewCountText", "simpleText") - view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 + view_count = short_text_to_number(view_count_text.try &.as_s || "0") # Duration @@ -773,6 +797,7 @@ end def extract_items(initial_data : InitialData, &block) if unpackaged_data = initial_data["contents"]?.try &.as_h elsif unpackaged_data = initial_data["response"]?.try &.as_h + elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 1).try &.as_h elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h else unpackaged_data = initial_data diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr index 0cb3c079..11d95958 100644 --- a/src/invidious/yt_backend/extractors_utils.cr +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -68,19 +68,17 @@ rescue ex return false end -def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) : Array(SearchVideo) - extracted, _ = extract_items(initial_data, author_fallback, author_id_fallback) +# This function extracts SearchVideo items from a Category. +# Categories are commonly returned in search results and trending pages. +def extract_category(category : Category) : Array(SearchVideo) + return category.contents.select(SearchVideo) +end - target = [] of (SearchItem | Continuation) - extracted.each do |i| - if i.is_a?(Category) - i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video } - else - target << i - end +# :ditto: +def extract_category(category : Category, &) + category.contents.select(SearchVideo).each do |item| + yield item end - - return target.select(SearchVideo) end def extract_selected_tab(tabs) |