diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/invidious.cr | 43 | ||||
| -rw-r--r-- | src/invidious/channels.cr | 40 | ||||
| -rw-r--r-- | src/invidious/helpers/i18n.cr | 2 | ||||
| -rw-r--r-- | src/invidious/helpers/utils.cr | 8 | ||||
| -rw-r--r-- | src/invidious/helpers/youtube_api.cr | 118 | ||||
| -rw-r--r-- | src/invidious/jobs/bypass_captcha_job.cr | 2 | ||||
| -rw-r--r-- | src/invidious/playlists.cr | 22 | ||||
| -rw-r--r-- | src/invidious/routes/login.cr | 4 | ||||
| -rw-r--r-- | src/invidious/routes/preferences.cr | 8 | ||||
| -rw-r--r-- | src/invidious/routes/search.cr | 51 | ||||
| -rw-r--r-- | src/invidious/search.cr | 10 | ||||
| -rw-r--r-- | src/invidious/users.cr | 2 | ||||
| -rw-r--r-- | src/invidious/views/search_homepage.ecr | 2 | ||||
| -rw-r--r-- | src/invidious/views/template.ecr | 2 |
14 files changed, 197 insertions, 117 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index ae20e13e..7037ecfe 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -166,7 +166,7 @@ end before_all do |env| preferences = begin - Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}") + Preferences.from_json(URI.decode_www_form(env.request.cookies["PREFS"]?.try &.value || "{}")) rescue Preferences.from_json("{}") end @@ -174,15 +174,44 @@ before_all do |env| env.set "preferences", preferences env.response.headers["X-XSS-Protection"] = "1; mode=block" env.response.headers["X-Content-Type-Options"] = "nosniff" - extra_media_csp = "" + + # Allow media resources to be loaded from google servers + # TODO: check if *.youtube.com can be removed if CONFIG.disabled?("local") || !preferences.local - extra_media_csp += " https://*.googlevideo.com:443" - extra_media_csp += " https://*.youtube.com:443" + extra_media_csp = " https://*.googlevideo.com:443 https://*.youtube.com:443" + else + extra_media_csp = "" end - # TODO: Remove style-src's 'unsafe-inline', requires to remove all inline styles (<style> [..] </style>, style=" [..] ") - env.response.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self'; manifest-src 'self'; media-src 'self' blob:#{extra_media_csp}; child-src blob:" + + # Only allow the pages at /embed/* to be embedded + if env.request.resource.starts_with?("/embed") + frame_ancestors = "'self' http: https:" + else + frame_ancestors = "none" + end + + # TODO: Remove style-src's 'unsafe-inline', requires to remove all + # inline styles (<style> [..] </style>, style=" [..] ") + env.response.headers["Content-Security-Policy"] = { + "default-src 'none'", + "script-src 'self'", + "style-src 'self' 'unsafe-inline'", + "img-src 'self' data:", + "font-src 'self' data:", + "connect-src 'self'", + "manifest-src 'self'", + "media-src 'self' blob:" + extra_media_csp, + "child-src 'self' blob:", + "frame-src 'self'", + "frame-ancestors " + frame_ancestors, + }.join("; ") + env.response.headers["Referrer-Policy"] = "same-origin" + # Ask the chrom*-based browsers to disable FLoC + # See: https://blog.runcloud.io/google-floc/ + env.response.headers["Permissions-Policy"] = "interest-cohort=()" + if (Kemal.config.ssl || CONFIG.https_only) && CONFIG.hsts env.response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains; preload" end @@ -421,7 +450,7 @@ get "/modify_notifications" do |env| html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) html.cookies.each do |cookie| if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name if cookies[cookie.name]? diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 3109b508..bbef3d4f 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -229,22 +229,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) page = 1 LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") - response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - - videos = [] of SearchVideo - begin - initial_data = JSON.parse(response_body) - raise InfoException.new("Could not extract channel JSON") if !initial_data - - LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data") - videos = extract_videos(initial_data.as_h, author, ucid) - rescue ex - if response_body.includes?("To continue with your YouTube experience, please fill out the form below.") || - response_body.includes?("https://www.google.com/sorry/index") - raise InfoException.new("Could not extract channel info. Instance is likely blocked.") - end - raise ex - end + initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) + videos = extract_videos(initial_data, author, ucid) LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") rss.xpath_nodes("//feed/entry").each do |entry| @@ -304,10 +290,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) ids = [] of String loop do - response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - initial_data = JSON.parse(response_body) - raise InfoException.new("Could not extract channel JSON") if !initial_data - videos = extract_videos(initial_data.as_h, author, ucid) + initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated) + videos = extract_videos(initial_data, author, ucid) count = videos.size videos = videos.map { |video| ChannelVideo.new({ @@ -358,8 +342,7 @@ end def fetch_channel_playlists(ucid, author, continuation, sort_by) if continuation response_json = request_youtube_api_browse(continuation) - result = JSON.parse(response_json) - continuationItems = result["onResponseReceivedActions"]? + continuationItems = response_json["onResponseReceivedActions"]? .try &.[0]["appendContinuationItemsAction"]["continuationItems"] return [] of SearchItem, nil if !continuationItems @@ -964,21 +947,16 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") videos = [] of SearchVideo 2.times do |i| - response_json = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) - initial_data = JSON.parse(response_json) - break if !initial_data - videos.concat extract_videos(initial_data.as_h, author, ucid) + initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) + videos.concat extract_videos(initial_data, author, ucid) end return videos.size, videos end def get_latest_videos(ucid) - response_json = get_channel_videos_response(ucid) - initial_data = JSON.parse(response_json) - return [] of SearchVideo if !initial_data + initial_data = get_channel_videos_response(ucid) author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s - items = extract_videos(initial_data.as_h, author, ucid) - return items + return extract_videos(initial_data, author, ucid) end diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 45a3f1ae..dd46feab 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -21,7 +21,7 @@ LOCALES = { "pt-PT" => load_locale("pt-PT"), "ro" => load_locale("ro"), "ru" => load_locale("ru"), - "sv" => load_locale("sv-SE"), + "sv-SE" => load_locale("sv-SE"), "tr" => load_locale("tr"), "uk" => load_locale("uk"), "zh-CN" => load_locale("zh-CN"), diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 6ce457b9..66ad6961 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,5 +1,5 @@ require "lsquic" -require "pool/connection" +require "db" def add_yt_headers(request) request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" @@ -20,7 +20,7 @@ struct YoutubeConnectionPool property! url : URI property! capacity : Int32 property! timeout : Float64 - property pool : ConnectionPool(QUIC::Client | HTTP::Client) + property pool : DB::Pool(QUIC::Client | HTTP::Client) def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) @url = url @@ -43,7 +43,7 @@ struct YoutubeConnectionPool conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn ensure - pool.checkin(conn) + pool.release(conn) end end @@ -51,7 +51,7 @@ struct YoutubeConnectionPool end private def build_pool(use_quic) - ConnectionPool(QUIC::Client | HTTP::Client).new(capacity: capacity, timeout: timeout) do + DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do if use_quic conn = QUIC::Client.new(url) else diff --git a/src/invidious/helpers/youtube_api.cr b/src/invidious/helpers/youtube_api.cr index 30413532..e27d4088 100644 --- a/src/invidious/helpers/youtube_api.cr +++ b/src/invidious/helpers/youtube_api.cr @@ -4,28 +4,116 @@ # Hard-coded constants required by the API HARDCODED_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" -HARDCODED_CLIENT_VERS = "2.20210318.08.00" +HARDCODED_CLIENT_VERS = "2.20210330.08.00" -def request_youtube_api_browse(continuation) +#################################################################### +# make_youtube_api_context(region) +# +# Return, as a Hash, the "context" data required to request the +# youtube API endpoints. +# +def make_youtube_api_context(region : String | Nil) : Hash + return { + "client" => { + "hl" => "en", + "gl" => region || "US", # Can't be empty! + "clientName" => "WEB", + "clientVersion" => HARDCODED_CLIENT_VERS, + }, + } +end + +#################################################################### +# request_youtube_api_browse(continuation) +# request_youtube_api_browse(browse_id, params) +# +# Requests the youtubei/v1/browse endpoint with the required headers +# and POST data in order to get a JSON reply in english US that can +# be easily parsed. +# +# The requested data can either be: +# +# - A continuation token (ctoken). Depending on this token's +# contents, the returned data can be comments, playlist videos, +# search results, channel community tab, ... +# +# - A playlist ID (parameters MUST be an empty string) +# +def request_youtube_api_browse(continuation : String) # JSON Request data, required by the API data = { - "context": { - "client": { - "hl": "en", - "gl": "US", - "clientName": "WEB", - "clientVersion": HARDCODED_CLIENT_VERS, - }, - }, - "continuation": continuation, + "context" => make_youtube_api_context("US"), + "continuation" => continuation, + } + + return _youtube_api_post_json("/youtubei/v1/browse", data) +end + +def request_youtube_api_browse(browse_id : String, params : String) + # JSON Request data, required by the API + data = { + "browseId" => browse_id, + "context" => make_youtube_api_context("US"), } - # Send the POST request and return result + # Append the additionnal parameters if those were provided + # (this is required for channel info, playlist and community, e.g) + if params != "" + data["params"] = params + end + + return _youtube_api_post_json("/youtubei/v1/browse", data) +end + +#################################################################### +# request_youtube_api_search(search_query, params, region) +# +# Requests the youtubei/v1/search endpoint with the required headers +# and POST data in order to get a JSON reply. As the search results +# vary depending on the region, a region code can be specified in +# order to get non-US results. +# +# The requested data is a search string, with some additional +# paramters, formatted as a base64 string. +# +def request_youtube_api_search(search_query : String, params : String, region = nil) + # JSON Request data, required by the API + data = { + "query" => search_query, + "context" => make_youtube_api_context(region), + "params" => params, + } + + return _youtube_api_post_json("/youtubei/v1/search", data) +end + +#################################################################### +# _youtube_api_post_json(endpoint, data) +# +# Internal function that does the actual request to youtube servers +# and handles errors. +# +# The requested data is an endpoint (URL without the domain part) +# and the data as a Hash object. +# +def _youtube_api_post_json(endpoint, data) + # Send the POST request and parse result response = YT_POOL.client &.post( - "/youtubei/v1/browse?key=#{HARDCODED_API_KEY}", - headers: HTTP::Headers{"content-type" => "application/json"}, + "#{endpoint}?key=#{HARDCODED_API_KEY}", + headers: HTTP::Headers{"content-type" => "application/json; charset=UTF-8"}, body: data.to_json ) - return response.body + initial_data = JSON.parse(response.body).as_h + + # Error handling + if initial_data.has_key?("error") + code = initial_data["error"]["code"] + message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "") + + raise InfoException.new("Could not extract JSON. Youtube API returned \ + error #{code} with message:<br>\"#{message}\"") + end + + return initial_data end diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr index 4269e123..e68b81e6 100644 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ b/src/invidious/jobs/bypass_captcha_job.cr @@ -112,7 +112,7 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob headers = HTTP::Headers{ "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0], } - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) cookies.each { |cookie| CONFIG.cookies << cookie } diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 073a9986..fe7f82f3 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -361,16 +361,7 @@ def fetch_playlist(plid, locale) plid = "UU#{plid.lchop("UC")}" end - response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en") - if response.status_code != 200 - if response.headers["location"]?.try &.includes? "/sorry/index" - raise InfoException.new("Could not extract playlist info. Instance is likely blocked.") - else - raise InfoException.new("Not a playlist.") - end - end - - initial_data = extract_initial_data(response.body) + initial_data = request_youtube_api_browse("VL" + plid, params: "") playlist_sidebar_renderer = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]? raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer @@ -451,17 +442,12 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) offset = (offset / 100).to_i64 * 100_i64 ctoken = produce_playlist_continuation(playlist.id, offset) - initial_data = JSON.parse(request_youtube_api_browse(ctoken)).as_h + initial_data = request_youtube_api_browse(ctoken) else - response = YT_POOL.client &.get("/playlist?list=#{playlist.id}&gl=US&hl=en") - initial_data = extract_initial_data(response.body) + initial_data = request_youtube_api_browse("VL" + playlist.id, params: "") end - if initial_data - return extract_playlist_videos(initial_data) - else - return [] of PlaylistVideo - end + return extract_playlist_videos(initial_data) end end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index ffe5f568..f9e6ea6c 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -238,7 +238,7 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute traceback << "Logging in..." location = URI.parse(challenge_results[0][-1][2].to_s) - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) headers.delete("Content-Type") headers.delete("Google-Accounts-XSRF") @@ -261,7 +261,7 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute location = login.headers["Location"]?.try { |u| URI.parse(u) } end - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) sid = cookies["SID"]?.try &.value if !sid raise "Couldn't get SID." diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index cfdad443..f98c7a5e 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -198,10 +198,10 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute end if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: preferences, expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) end end @@ -250,10 +250,10 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute end if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: preferences, expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) end end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index a993a17a..513904b8 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -20,15 +20,17 @@ class Invidious::Routes::Search < Invidious::Routes::BaseRoute query = env.params.query["search_query"]? query ||= env.params.query["q"]? - query ||= "" - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + page = env.params.query["page"]? - if query - env.redirect "/search?q=#{URI.encode_www_form(query)}&page=#{page}" + if query && !query.empty? + if page && !page.empty? + env.redirect "/search?q=" + URI.encode_www_form(query) + "&page=" + page + else + env.redirect "/search?q=" + URI.encode_www_form(query) + end else - env.redirect "/" + env.redirect "/search" end end @@ -38,28 +40,31 @@ class Invidious::Routes::Search < Invidious::Routes::BaseRoute query = env.params.query["search_query"]? query ||= env.params.query["q"]? - query ||= "" - return env.redirect "/" if query.empty? + if !query || query.empty? + # Display the full page search box implemented in #1977 + env.set "search", "" + templated "search_homepage", navbar_search: false + else + page = env.params.query["page"]?.try &.to_i? + page ||= 1 - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + user = env.get? "user" - user = env.get? "user" + begin + search_query, count, videos, operators = process_search_query(query, page, user, region: region) + rescue ex + return error_template(500, ex) + end - begin - search_query, count, videos, operators = process_search_query(query, page, user, region: nil) - rescue ex - return error_template(500, ex) - end + operator_hash = {} of String => String + operators.each do |operator| + key, value = operator.downcase.split(":") + operator_hash[key] = value + end - operator_hash = {} of String => String - operators.each do |operator| - key, value = operator.downcase.split(":") - operator_hash[key] = value + env.set "search", query + templated "search" end - - env.set "search", query - templated "search" end end diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 4b216613..662173a0 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -246,8 +246,7 @@ def channel_search(query, page, channel) continuation = produce_channel_search_continuation(ucid, query, page) response_json = request_youtube_api_browse(continuation) - result = JSON.parse(response_json) - continuationItems = result["onResponseReceivedActions"]? + continuationItems = response_json["onResponseReceivedActions"]? .try &.[0]["appendContinuationItemsAction"]["continuationItems"] return 0, [] of SearchItem if !continuationItems @@ -264,14 +263,9 @@ end def search(query, search_params = produce_search_params(content_type: "all"), region = nil) return 0, [] of SearchItem if query.empty? - body = YT_POOL.client(region, &.get("/results?search_query=#{URI.encode_www_form(query)}&sp=#{search_params}&hl=en").body) - return 0, [] of SearchItem if body.empty? - - initial_data = extract_initial_data(body) + initial_data = request_youtube_api_search(query, search_params, region) items = extract_items(initial_data) - # initial_data["estimatedResults"]?.try &.as_s.to_i64 - return items.size, items end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 5dfd80bb..d774ee12 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -462,7 +462,7 @@ def subscribe_ajax(channel_id, action, env_headers) html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) html.cookies.each do |cookie| if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name if cookies[cookie.name]? diff --git a/src/invidious/views/search_homepage.ecr b/src/invidious/views/search_homepage.ecr index 8927c3f1..7d2dab83 100644 --- a/src/invidious/views/search_homepage.ecr +++ b/src/invidious/views/search_homepage.ecr @@ -1,7 +1,7 @@ <% content_for "header" do %> <meta name="description" content="<%= translate(locale, "An alternative front-end to YouTube") %>"> <title> - Invidious + Invidious - <%= translate(locale, "search") %> </title> <link rel="stylesheet" href="/css/empty.css?v=<%= ASSET_COMMIT %>"> <% end %> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 5b63bf1f..a13d3928 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -26,7 +26,7 @@ <span style="display:none" id="dark_mode_pref"><%= env.get("preferences").as(Preferences).dark_mode %></span> <div class="pure-g"> <div class="pure-u-1 pure-u-md-2-24"></div> - <div class="pure-u-1 pure-u-md-20-24", id="contents"> + <div class="pure-u-1 pure-u-md-20-24" id="contents"> <div class="pure-g navbar h-box"> <% if navbar_search %> <div class="pure-u-1 pure-u-md-4-24"> |