diff options
| -rw-r--r-- | assets/css/default.css | 19 | ||||
| -rw-r--r-- | src/invidious/channels/about.cr | 166 | ||||
| -rw-r--r-- | src/invidious/helpers/serialized_yt_data.cr | 4 | ||||
| -rw-r--r-- | src/invidious/jsonify/api_v1/video_json.cr | 12 | ||||
| -rw-r--r-- | src/invidious/playlists.cr | 7 | ||||
| -rw-r--r-- | src/invidious/routes/api/v1/channels.cr | 89 | ||||
| -rw-r--r-- | src/invidious/routes/api/v1/videos.cr | 4 | ||||
| -rw-r--r-- | src/invidious/routes/channels.cr | 74 | ||||
| -rw-r--r-- | src/invidious/routes/search.cr | 6 | ||||
| -rw-r--r-- | src/invidious/routes/video_playback.cr | 2 | ||||
| -rw-r--r-- | src/invidious/search/query.cr | 38 | ||||
| -rw-r--r-- | src/invidious/user/imports.cr | 6 | ||||
| -rw-r--r-- | src/invidious/videos.cr | 20 | ||||
| -rw-r--r-- | src/invidious/videos/transcript.cr | 4 | ||||
| -rw-r--r-- | src/invidious/views/components/search_box.ecr | 3 | ||||
| -rw-r--r-- | src/invidious/views/watch.ecr | 2 | ||||
| -rw-r--r-- | src/invidious/yt_backend/url_sanitizer.cr | 121 |
17 files changed, 438 insertions, 139 deletions
diff --git a/assets/css/default.css b/assets/css/default.css index a47762ec..2cedcf0c 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -278,7 +278,14 @@ div.thumbnail > .bottom-right-overlay { display: inline; } -.searchbar .pure-form fieldset { padding: 0; } +.searchbar .pure-form { + display: flex; +} + +.searchbar .pure-form fieldset { + padding: 0; + flex: 1; +} .searchbar input[type="search"] { width: 100%; @@ -310,6 +317,16 @@ input[type="search"]::-webkit-search-cancel-button { background-size: 14px; } +.searchbar #searchbutton { + border: none; + background: none; + margin-top: 0; +} + +.searchbar #searchbutton:hover { + color: rgb(0, 182, 240); +} + .user-field { display: flex; flex-direction: row; diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index edaf5c12..13909527 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -15,7 +15,8 @@ record AboutChannel, allowed_regions : Array(String), tabs : Array(String), tags : Array(String), - verified : Bool + verified : Bool, + is_age_gated : Bool def get_about_info(ucid, locale) : AboutChannel begin @@ -45,45 +46,101 @@ def get_about_info(ucid, locale) : AboutChannel end tags = [] of String + tab_names = [] of String + total_views = 0_i64 + joined = Time.unix(0) - if auto_generated - author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s - author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s - author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s + if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") + description_node = nil + author = age_gate_renderer["channelTitle"].as_s + ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s + author_url = "https://www.youtube.com/channel/#{ucid}" + author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s + banner = nil + is_family_friendly = false + is_age_gated = true + tab_names = ["videos", "shorts", "streams"] + auto_generated = false + else + if auto_generated + author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s + author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s + author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s + + # Raises a KeyError on failure. + banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banner = banners.try &.[-1]?.try &.["url"].as_s? + + description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + # some channels have the description in a simpleText + # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ + description_node = description_base_node.dig?("simpleText") || description_base_node + + tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") + .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + else + author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s + author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s + author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s + author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) - # Raises a KeyError on failure. - banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? + ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s - description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] - # some channels have the description in a simpleText - # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ - description_node = description_base_node.dig?("simpleText") || description_base_node + # Raises a KeyError on failure. + banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") + banner = banners.try &.[-1]?.try &.["url"].as_s? - tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") - .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String - else - author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s - author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s - author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) + # if banner.includes? "channels/c4/default_banner" + # banner = nil + # end - ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s + description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? + tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String + end - # Raises a KeyError on failure. - banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") - banner = banners.try &.[-1]?.try &.["url"].as_s? + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? + # Get the name of the tabs available on this channel + tab_names = tabs_json.as_a.compact_map do |entry| + name = entry.dig?("tabRenderer", "title").try &.as_s.downcase - # if banner.includes? "channels/c4/default_banner" - # banner = nil - # end + # This is a small fix to not add extra code on the HTML side + # I.e, the URL for the "live" tab is .../streams, so use "streams" + # everywhere for the sake of simplicity + (name == "live") ? "streams" : name + end - description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? - tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String - end + # Get the currently active tab ("About") + about_tab = extract_selected_tab(tabs_json) + + # Try to find the about metadata section + channel_about_meta = about_tab.dig?( + "content", + "sectionListRenderer", "contents", 0, + "itemSectionRenderer", "contents", 0, + "channelAboutFullMetadataRenderer" + ) - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + if !channel_about_meta.nil? + total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 + + # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. + joined = extract_text(channel_about_meta["joinedDateText"]?) + .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) + + # Normal Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + # For auto-generated channels, channel_about_meta only has + # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] + auto_generated = ( + (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ + extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || + channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" + ) + end + end + end allowed_regions = initdata .dig?("microformat", "microformatDataRenderer", "availableCountries") @@ -102,52 +159,6 @@ def get_about_info(ucid, locale) : AboutChannel end end - total_views = 0_i64 - joined = Time.unix(0) - - tab_names = [] of String - - if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? - # Get the name of the tabs available on this channel - tab_names = tabs_json.as_a.compact_map do |entry| - name = entry.dig?("tabRenderer", "title").try &.as_s.downcase - - # This is a small fix to not add extra code on the HTML side - # I.e, the URL for the "live" tab is .../streams, so use "streams" - # everywhere for the sake of simplicity - (name == "live") ? "streams" : name - end - - # Get the currently active tab ("About") - about_tab = extract_selected_tab(tabs_json) - - # Try to find the about metadata section - channel_about_meta = about_tab.dig?( - "content", - "sectionListRenderer", "contents", 0, - "itemSectionRenderer", "contents", 0, - "channelAboutFullMetadataRenderer" - ) - - if !channel_about_meta.nil? - total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 - - # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = extract_text(channel_about_meta["joinedDateText"]?) - .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - - # Normal Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - # For auto-generated channels, channel_about_meta only has - # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] - auto_generated = ( - (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ - extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || - channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" - ) - end - end - sub_count = 0 if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a) @@ -177,6 +188,7 @@ def get_about_info(ucid, locale) : AboutChannel tabs: tab_names, tags: tags, verified: author_verified || false, + is_age_gated: is_age_gated || false, ) end diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 31a3cf44..463d5557 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -90,7 +90,7 @@ struct SearchVideo json.field "lengthSeconds", self.length_seconds json.field "liveNow", self.live_now json.field "premium", self.premium - json.field "isUpcoming", self.is_upcoming + json.field "isUpcoming", self.upcoming? if self.premiere_timestamp json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix @@ -109,7 +109,7 @@ struct SearchVideo to_json(nil, json) end - def is_upcoming + def upcoming? premiere_timestamp ? true : false end end diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 4d12a072..08cd533f 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -63,7 +63,7 @@ module Invidious::JSONify::APIv1 json.field "isListed", video.is_listed json.field "liveNow", video.live_now json.field "isPostLiveDvr", video.post_live_dvr - json.field "isUpcoming", video.is_upcoming + json.field "isUpcoming", video.upcoming? if video.premiere_timestamp json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix @@ -109,7 +109,7 @@ module Invidious::JSONify::APIv1 # On livestreams, it's not present, so always fall back to the # current unix timestamp (up to mS precision) for compatibility. last_modified = fmt["lastModified"]? - last_modified ||= "#{Time.utc.to_unix_ms.to_s}000" + last_modified ||= "#{Time.utc.to_unix_ms}000" json.field "lmt", last_modified json.field "projectionType", fmt["projectionType"] @@ -162,7 +162,13 @@ module Invidious::JSONify::APIv1 json.array do video.fmt_stream.each do |fmt| json.object do - json.field "url", fmt["url"] + if proxy + json.field "url", Invidious::HttpServer::Utils.proxy_video_url( + fmt["url"].to_s, absolute: true + ) + else + json.field "url", fmt["url"] + end json.field "itag", fmt["itag"].as_i.to_s json.field "type", fmt["mimeType"] json.field "quality", fmt["quality"] diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index a227f794..3e6eef95 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -46,8 +46,14 @@ struct PlaylistVideo XML.build { |xml| to_xml(xml) } end + def to_json(locale : String?, json : JSON::Builder) + to_json(json) + end + def to_json(json : JSON::Builder, index : Int32? = nil) json.object do + json.field "type", "video" + json.field "title", self.title json.field "videoId", self.id @@ -67,6 +73,7 @@ struct PlaylistVideo end json.field "lengthSeconds", self.length_seconds + json.field "liveNow", self.live_now end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 43a5c35b..2da76134 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -27,10 +27,21 @@ module Invidious::Routes::API::V1::Channels # Retrieve "sort by" setting from URL parameters sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" - begin - videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) + rescue ex + return error_json(500, ex) + end end JSON.build do |json| @@ -84,6 +95,7 @@ module Invidious::Routes::API::V1::Channels json.field "joined", channel.joined.to_unix json.field "autoGenerated", channel.auto_generated + json.field "ageGated", channel.is_age_gated json.field "isFamilyFriendly", channel.is_family_friendly json.field "description", html_to_content(channel.description_html) json.field "descriptionHtml", channel.description_html @@ -142,12 +154,23 @@ module Invidious::Routes::API::V1::Channels sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_60_videos( - channel, continuation: continuation, sort_by: sort_by - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -176,12 +199,23 @@ module Invidious::Routes::API::V1::Channels # Retrieve continuation from URL parameters continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -211,12 +245,23 @@ module Invidious::Routes::API::V1::Channels sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation, sort_by: sort_by - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULV")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index c077b85e..368304ac 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -118,7 +118,7 @@ module Invidious::Routes::API::V1::Videos else caption_xml = XML.parse(caption_xml) - webvtt = WebVTT.build(settings_field) do |webvtt| + webvtt = WebVTT.build(settings_field) do |builder| caption_nodes = caption_xml.xpath_nodes("//transcript/text") caption_nodes.each_with_index do |node, i| start_time = node["start"].to_f.seconds @@ -138,7 +138,7 @@ module Invidious::Routes::API::V1::Videos text = "<v #{md["name"]}>#{md["text"]}</v>" end - webvtt.cue(start_time, end_time, text) + builder.cue(start_time, end_time, text) end end end diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 360af2cd..952098e0 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -36,12 +36,24 @@ module Invidious::Routes::Channels items = items.select(SearchPlaylist) items.each(&.author = "") else - sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_videos( - channel, continuation: continuation, sort_by: (sort_by || "newest") - ) + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_options = {"newest", "oldest", "popular"} + items, next_continuation = Channel::Tabs.get_videos( + channel, continuation: continuation, sort_by: (sort_by || "newest") + ) + end end selected_tab = Frontend::ChannelPage::TabsAvailable::Videos @@ -58,14 +70,27 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for shorts - sort_by = "" - sort_options = [] of String + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + # TODO: support sort option for shorts + sort_by = "" + sort_options = [] of String - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation - ) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation + ) + end selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts templated "channel" @@ -81,13 +106,26 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" - sort_options = {"newest", "oldest", "popular"} + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULV")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation, sort_by: sort_by - ) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation, sort_by: sort_by + ) + end selected_tab = Frontend::ChannelPage::TabsAvailable::Streams templated "channel" diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 5be33533..44970922 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -51,6 +51,12 @@ module Invidious::Routes::Search else user = env.get? "user" + # An URL was copy/pasted in the search box. + # Redirect the user to the appropriate page. + if query.url? + return env.redirect UrlSanitizer.process(query.text).to_s + end + begin items = query.process rescue ex : ChannelSearchException diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index ec18f3b8..24693662 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -131,7 +131,7 @@ module Invidious::Routes::VideoPlayback end # TODO: Record bytes written so we can restart after a chunk fails - while true + loop do if !range_end && content_length range_end = content_length end diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index e38845d9..c8e8cf7f 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -20,6 +20,9 @@ module Invidious::Search property region : String? property channel : String = "" + # Flag that indicates if the smart search features have been disabled. + @inhibit_ssf : Bool = false + # Return true if @raw_query is either `nil` or empty private def empty_raw_query? return @raw_query.empty? @@ -48,10 +51,18 @@ module Invidious::Search ) # Get the raw search query string (common to all search types). In # Regular search mode, also look for the `search_query` URL parameter - if @type.regular? - @raw_query = params["q"]? || params["search_query"]? || "" - else - @raw_query = params["q"]? || "" + _raw_query = params["q"]? + _raw_query ||= params["search_query"]? if @type.regular? + _raw_query ||= "" + + # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs. + @raw_query = _raw_query.strip + + # Check for smart features (ex: URL search) inhibitor (backslash). + # If inhibitor is present, remove it. + if @raw_query.starts_with?('\\') + @inhibit_ssf = true + @raw_query = @raw_query[1..] end # Get the page number (also common to all search types) @@ -85,7 +96,7 @@ module Invidious::Search @filters = Filters.from_iv_params(params) @channel = params["channel"]? || "" - if @filters.default? && @raw_query.includes?(':') + if @filters.default? && @raw_query.index(/\w:\w/) # Parse legacy filters from query @filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query) else @@ -136,5 +147,22 @@ module Invidious::Search return params end + + # Checks if the query is a standalone URL + def url? : Bool + # If the smart features have been inhibited, don't go further. + return false if @inhibit_ssf + + # Only supported in regular search mode + return false if !@type.regular? + + # If filters are present, that's a regular search + return false if !@filters.default? + + # Simple heuristics: domain name + return @raw_query.starts_with?( + /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\// + ) + end end end diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index a70434ca..533c18d9 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -115,7 +115,7 @@ struct Invidious::User playlists.each do |item| title = item["title"]?.try &.as_s?.try &.delete("<>") description = item["description"]?.try &.as_s?.try &.delete("\r") - privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } + privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state } next if !title next if !description @@ -161,7 +161,7 @@ struct Invidious::User # Youtube # ------------------- - private def is_opml?(mimetype : String, extension : String) + private def opml?(mimetype : String, extension : String) opml_mimetypes = [ "application/xml", "text/xml", @@ -179,7 +179,7 @@ struct Invidious::User def from_youtube(user : User, body : String, filename : String, type : String) : Bool extension = filename.split(".").last - if is_opml?(type, extension) + if opml?(type, extension) subscriptions = XML.parse(body) user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0] diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 28cbb311..921132f0 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -223,7 +223,7 @@ struct Video info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil end - def is_vr : Bool? + def vr? : Bool? return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type end @@ -304,6 +304,21 @@ struct Video {% if flag?(:debug_macros) %} {{debug}} {% end %} end + # Macro to generate ? and = accessor methods for attributes in `info` + private macro predicate_bool(method_name, name) + # Return {{name.stringify}} from `info` + def {{method_name.id.underscore}}? : Bool + return info[{{name.stringify}}]?.try &.as_bool || false + end + + # Update {{name.stringify}} into `info` + def {{method_name.id.underscore}}=(value : Bool) + info[{{name.stringify}}] = JSON::Any.new(value) + end + + {% if flag?(:debug_macros) %} {{debug}} {% end %} + end + # Method definitions, using the macros above getset_string author @@ -325,11 +340,12 @@ struct Video getset_i64 likes getset_i64 views + # TODO: Make predicate_bool the default as to adhere to Crystal conventions getset_bool allowRatings getset_bool authorVerified getset_bool isFamilyFriendly getset_bool isListed - getset_bool isUpcoming + predicate_bool upcoming, isUpcoming end def get_video(id, refresh = true, region = nil, force_refresh = false) diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index 9cd064c5..4bd9f820 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -110,13 +110,13 @@ module Invidious::Videos "Language" => @language_code, } - vtt = WebVTT.build(settings_field) do |vtt| + vtt = WebVTT.build(settings_field) do |builder| @lines.each do |line| # Section headers are excluded from the VTT conversion as to # match the regular captions returned from YouTube as much as possible next if line.is_a? HeadingLine - vtt.cue(line.start_ms, line.end_ms, line.line) + builder.cue(line.start_ms, line.end_ms, line.line) end end diff --git a/src/invidious/views/components/search_box.ecr b/src/invidious/views/components/search_box.ecr index a03785d1..29da2c52 100644 --- a/src/invidious/views/components/search_box.ecr +++ b/src/invidious/views/components/search_box.ecr @@ -6,4 +6,7 @@ title="<%= translate(locale, "search") %>" value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> </fieldset> + <button type="submit" id="searchbutton" aria-label="<%= translate(locale, "search") %>"> + <i class="icon ion-ios-search"></i> + </button> </form> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 36679bce..45c58a16 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -62,7 +62,7 @@ we're going to need to do it here in order to allow for translations. "params" => params, "preferences" => preferences, "premiere_timestamp" => video.premiere_timestamp.try &.to_unix, - "vr" => video.is_vr, + "vr" => video.vr?, "projection_type" => video.projection_type, "local_disabled" => CONFIG.disabled?("local"), "support_reddit" => true diff --git a/src/invidious/yt_backend/url_sanitizer.cr b/src/invidious/yt_backend/url_sanitizer.cr new file mode 100644 index 00000000..725382ee --- /dev/null +++ b/src/invidious/yt_backend/url_sanitizer.cr @@ -0,0 +1,121 @@ +require "uri" + +module UrlSanitizer + extend self + + ALLOWED_QUERY_PARAMS = { + channel: ["u", "user", "lb"], + playlist: ["list"], + search: ["q", "search_query", "sp"], + watch: [ + "v", # Video ID + "list", "index", # Playlist-related + "playlist", # Unnamed playlist (id,id,id,...) (embed-only?) + "t", "time_continue", "start", "end", # Timestamp + "lc", # Highlighted comment (watch page only) + ], + } + + # Returns whether the given string is an ASCII word. This is the same as + # running the following regex in US-ASCII locale: /^[\w-]+$/ + private def ascii_word?(str : String) : Bool + return false if str.bytesize != str.size + + str.each_byte do |byte| + next if 'a'.ord <= byte <= 'z'.ord + next if 'A'.ord <= byte <= 'Z'.ord + next if '0'.ord <= byte <= '9'.ord + next if byte == '-'.ord || byte == '_'.ord + + return false + end + + return true + end + + # Return which kind of parameters are allowed based on the + # first path component (breadcrumb 0). + private def determine_allowed(path_root : String) + case path_root + when "watch", "w", "v", "embed", "e", "shorts", "clip" + return :watch + when .starts_with?("@"), "c", "channel", "user", "profile", "attribution_link" + return :channel + when "playlist", "mix" + return :playlist + when "results", "search" + return :search + else # hashtag, post, trending, brand URLs, etc.. + return nil + end + end + + # Create a new URI::Param containing only the allowed parameters + private def copy_params(unsafe_params : URI::Params, allowed_type) : URI::Params + new_params = URI::Params.new + + ALLOWED_QUERY_PARAMS[allowed_type].each do |name| + if unsafe_params[name]? + # Only copy the last parameter, in case there is more than one + new_params[name] = unsafe_params.fetch_all(name)[-1] + end + end + + return new_params + end + + # Transform any user-supplied youtube URL into something we can trust + # and use across the code. + def process(str : String) : URI + # Because URI follows RFC3986 specifications, URL without a scheme + # will be parsed as a relative path. So we have to add a scheme ourselves. + str = "https://#{str}" if !str.starts_with?(/https?:\/\//) + + unsafe_uri = URI.parse(str) + unsafe_host = unsafe_uri.host + unsafe_path = unsafe_uri.path + + new_uri = URI.new(path: "/") + + # Redirect to homepage for bogus URLs + return new_uri if (unsafe_host.nil? || unsafe_path.nil?) + + breadcrumbs = unsafe_path + .split('/', remove_empty: true) + .compact_map do |bc| + # Exclude attempts at path trasversal + next if bc == "." || bc == ".." + + # Non-alnum characters are unlikely in a genuine URL + next if !ascii_word?(bc) + + bc + end + + # If nothing remains, it's either a legit URL to the homepage + # (who does that!?) or because we filtered some junk earlier. + return new_uri if breadcrumbs.empty? + + # Replace the original query parameters with the sanitized ones + case unsafe_host + when .ends_with?("youtube.com") + # Use our sanitized path (not forgetting the leading '/') + new_uri.path = "/#{breadcrumbs.join('/')}" + + # Then determine which params are allowed, and copy them over + if allowed = determine_allowed(breadcrumbs[0]) + new_uri.query_params = copy_params(unsafe_uri.query_params, allowed) + end + when "youtu.be" + # Always redirect to the watch page + new_uri.path = "/watch" + + new_params = copy_params(unsafe_uri.query_params, :watch) + new_params["id"] = breadcrumbs[0] + + new_uri.query_params = new_params + end + + return new_uri + end +end |