diff options
Diffstat (limited to 'src')
23 files changed, 165 insertions, 534 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index e0bd0101..c8cac80e 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -217,7 +217,6 @@ public_folder "assets" Kemal.config.powered_by_header = false add_handler FilteredCompressHandler.new -add_handler APIHandler.new add_handler AuthHandler.new add_handler DenyFrame.new add_context_storage_type(Array(String)) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index c3d6124f..be739673 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -93,7 +93,7 @@ struct ChannelVideo def to_tuple {% begin %} { - {{*@type.instance_vars.map(&.name)}} + {{@type.instance_vars.map(&.name).splat}} } {% end %} end diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index beb86e08..351790d7 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -62,12 +62,6 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end -# Used in bypass_captcha_job.cr -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" -end - module Invidious::Channel::Tabs extend self diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 429d9246..09c2168b 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -48,7 +48,7 @@ struct ConfigPreferences def to_tuple {% begin %} { - {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}} + {{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}} } {% end %} end @@ -133,10 +133,6 @@ class Config # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new - # Key for Anti-Captcha - property captcha_key : String? = nil - # API URL for Anti-Captcha - property captcha_api_url : String = "https://api.anti-captcha.com" # Playlist length limit property playlist_length_limit : Int32 = 500 diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 6e5a975d..21b789bc 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -3,7 +3,7 @@ # ------------------- macro error_template(*args) - error_template_helper(env, {{*args}}) + error_template_helper(env, {{args.splat}}) end def github_details(summary : String, content : String) @@ -95,7 +95,7 @@ end # ------------------- macro error_atom(*args) - error_atom_helper(env, {{*args}}) + error_atom_helper(env, {{args.splat}}) end def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) @@ -121,7 +121,7 @@ end # ------------------- macro error_json(*args) - error_json_helper(env, {{*args}}) + error_json_helper(env, {{args.splat}}) end def error_json_helper( diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index d140a858..cece289b 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -134,74 +134,6 @@ class AuthHandler < Kemal::Handler end end -class APIHandler < Kemal::Handler - {% for method in %w(GET POST PUT HEAD DELETE PATCH OPTIONS) %} - only ["/api/v1/*"], {{method}} - {% end %} - exclude ["/api/v1/auth/notifications"], "GET" - exclude ["/api/v1/auth/notifications"], "POST" - - def call(env) - return call_next env unless only_match? env - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - # Since /api/v1/notifications is an event-stream, we don't want - # to wrap the response - return call_next env if exclude_match? env - - # Here we swap out the socket IO so we can modify the response as needed - output = env.response.output - env.response.output = IO::Memory.new - - begin - call_next env - - env.response.output.rewind - - if env.response.output.as(IO::Memory).size != 0 && - env.response.headers.includes_word?("Content-Type", "application/json") - response = JSON.parse(env.response.output) - - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - - if env.params.query["pretty"]?.try &.== "1" - response = response.to_pretty_json - else - response = response.to_json - end - else - response = env.response.output.gets_to_end - end - rescue ex - env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html") - env.response.status_code = 500 - - if env.response.headers.includes_word?("Content-Type", "application/json") - response = {"error" => ex.message || "Unspecified error"} - - if env.params.query["pretty"]?.try &.== "1" - response = response.to_pretty_json - else - response = response.to_json - end - end - ensure - env.response.output = output - env.response.print response - - env.response.flush - end - end -end - class DenyFrame < Kemal::Handler exclude ["/embed/*"] diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 6dc9860e..6add0237 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -78,15 +78,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = published response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -113,15 +104,6 @@ def create_notification_stream(env, topics, connection_channel) Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -155,15 +137,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr deleted file mode 100644 index 3f4080ba..00000000 --- a/src/invidious/helpers/json_filter.cr +++ /dev/null @@ -1,248 +0,0 @@ -module JSONFilter - alias BracketIndex = Hash(Int64, Int64) - - alias GroupedFieldsValue = String | Array(GroupedFieldsValue) - alias GroupedFieldsList = Array(GroupedFieldsValue) - - class FieldsParser - class ParseError < Exception - end - - # Returns the `Regex` pattern used to match nest groups - def self.nest_group_pattern : Regex - # uses a '.' character to match json keys as they are allowed - # to contain any unicode codepoint - /(?:|,)(?<groupname>[^,\n]*?)\(/ - end - - # Returns the `Regex` pattern used to check if there are any empty nest groups - def self.unnamed_nest_group_pattern : Regex - /^\(|\(\(|\/\(/ - end - - def self.parse_fields(fields_text : String, &) : Nil - if fields_text.empty? - raise FieldsParser::ParseError.new "Fields is empty" - end - - opening_bracket_count = fields_text.count('(') - closing_bracket_count = fields_text.count(')') - - if opening_bracket_count != closing_bracket_count - bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing" - raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})" - elsif match_result = unnamed_nest_group_pattern.match(fields_text) - raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}" - end - - # first, handle top-level single nested properties: items/id, playlistItems/snippet, etc - parse_single_nests(fields_text) { |nest_list| yield nest_list } - - # next, handle nest groups: items(id, etag, etc) - parse_nest_groups(fields_text) { |nest_list| yield nest_list } - end - - def self.parse_single_nests(fields_text : String, &) : Nil - single_nests = remove_nest_groups(fields_text) - - if !single_nests.empty? - property_nests = single_nests.split(',') - - property_nests.each do |nest| - nest_list = nest.split('/') - if nest_list.includes? "" - raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}" - end - yield nest_list - end - # else - # raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}" - end - end - - def self.parse_nest_groups(fields_text : String, &) : Nil - nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64) - bracket_pairs = get_bracket_pairs(fields_text, true) - - text_index = 0 - regex_index = 0 - - while regex_result = self.nest_group_pattern.match(fields_text, regex_index) - raw_match = regex_result[0] - group_name = regex_result["groupname"] - - text_index = regex_result.begin - regex_index = regex_result.end - - if text_index.nil? || regex_index.nil? - raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}" - end - - offset = raw_match.starts_with?(',') ? 1 : 0 - - opening_bracket_index = (text_index + group_name.size) + offset - closing_bracket_index = bracket_pairs[opening_bracket_index] - content_start = opening_bracket_index + 1 - - content = fields_text[content_start...closing_bracket_index] - - if content.empty? - raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}" - else - content = remove_nest_groups(content) - end - - while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index] - if nest_stack.size - nest_stack.pop - end - end - - group_name.split('/').each do |name| - nest_stack.push({ - group_name: name, - closing_bracket_index: closing_bracket_index, - }) - end - - if !content.empty? - properties = content.split(',') - - properties.each do |prop| - nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] } - - if !prop.empty? - if prop.includes?('/') - parse_single_nests(prop) { |list| nest_list += list } - else - nest_list.push prop - end - else - raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}" - end - - yield nest_list - end - end - end - end - - def self.remove_nest_groups(text : String) : String - content_bracket_pairs = get_bracket_pairs(text, false) - - content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket| - closing_bracket = content_bracket_pairs[opening_bracket] - last_comma = text.rindex(',', opening_bracket) || 0 - - text = text[0...last_comma] + text[closing_bracket + 1...text.size] - end - - return text.starts_with?(',') ? text[1...text.size] : text - end - - def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex - istart = [] of Int64 - bracket_index = BracketIndex.new - - text.each_char_with_index do |char, index| - if char == '(' - istart.push(index.to_i64) - end - - if char == ')' - begin - opening = istart.pop - if recursive || (!recursive && istart.size == 0) - bracket_index[opening] = index.to_i64 - end - rescue - raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}" - end - end - end - - if istart.size != 0 - idx = istart.pop - raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}" - end - - return bracket_index - end - end - - class FieldsGrouper - alias SkeletonValue = Hash(String, SkeletonValue) - - def self.create_json_skeleton(fields_text : String) : SkeletonValue - root_hash = {} of String => SkeletonValue - - FieldsParser.parse_fields(fields_text) do |nest_list| - current_item = root_hash - nest_list.each do |key| - if current_item[key]? - current_item = current_item[key] - else - current_item[key] = {} of String => SkeletonValue - current_item = current_item[key] - end - end - end - root_hash - end - - def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList - grouped_fields_list = GroupedFieldsList.new - json_skeleton.each do |key, value| - grouped_fields_list.push key - - nested_keys = create_grouped_fields_list(value) - grouped_fields_list.push nested_keys unless nested_keys.empty? - end - return grouped_fields_list - end - end - - class FilterError < Exception - end - - def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true) - skeleton = FieldsGrouper.create_json_skeleton(fields_text) - grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton) - filter(item, grouped_fields_list, in_place) - end - - def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any - item = item.clone unless in_place - - if !item.as_h? && !item.as_a? - raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}" - end - - top_level_keys = Array(String).new - grouped_fields_list.each do |value| - if value.is_a? String - top_level_keys.push value - elsif value.is_a? Array - if !top_level_keys.empty? - key_to_filter = top_level_keys.last - - if item.as_h? - filter(item[key_to_filter], value, in_place: true) - elsif item.as_a? - item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) } - end - else - raise FilterError.new "Tried to filter while top level keys list is empty" - end - end - end - - if item.as_h? - item.as_h.select! top_level_keys - elsif item.as_a? - item.as_a.map { |value| filter(value, top_level_keys, in_place: true) } - end - - item - end -end diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr deleted file mode 100644 index 71f8a938..00000000 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ /dev/null @@ -1,135 +0,0 @@ -class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob - def begin - loop do - begin - random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String}) - if !random_video - random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"} - end - {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path| - response = YT_POOL.client &.get(path) - if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - headers = response.cookies.add_request_headers(HTTP::Headers.new) - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => "https://www.youtube.com#{path}", - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/das_captcha", headers, form: inputs) - - response.cookies - .select { |cookie| cookie.name != "PREF" } - .each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - elsif response.headers["Location"]?.try &.includes?("/sorry/index") - location = response.headers["Location"].try { |u| URI.parse(u) } - headers = HTTP::Headers{":authority" => location.host.not_nil!} - response = YT_POOL.client &.get(location.request_target, headers) - - html = XML.parse_html(response.body) - form = html.xpath_node(%(//form[@action="index"])).not_nil! - site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] - s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] - - inputs = {} of String => String - form.xpath_nodes(%(.//input[@name])).map do |node| - inputs[node["name"]] = node["value"] - end - - captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url)) - captcha_client.family = CONFIG.force_resolve || Socket::Family::INET - response = JSON.parse(captcha_client.post("/createTask", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => location.to_s, - "websiteKey" => site_key, - "recaptchaDataSValue" => s_value, - }, - }.to_json).body) - - captcha_client.close - - raise response["error"].as_s if response["error"]? - task_id = response["taskId"].as_i - - loop do - sleep 10.seconds - - response = JSON.parse(captcha_client.post("/getTaskResult", - headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { - "clientKey" => CONFIG.captcha_key, - "taskId" => task_id, - }.to_json).body) - - if response["status"]?.try &.== "ready" - break - elsif response["errorId"]?.try &.as_i != 0 - raise response["errorDescription"].as_s - end - end - - inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || "" - response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs) - headers = HTTP::Headers{ - "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0], - } - cookies = HTTP::Cookies.from_client_headers(headers) - - cookies.each { |cookie| CONFIG.cookies << cookie } - - # Persist cookies between runs - File.write("config/config.yml", CONFIG.to_yaml) - end - end - rescue ex - LOGGER.error("BypassCaptchaJob: #{ex.message}") - ensure - sleep 1.minute - Fiber.yield - end - end - end -end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index b42ecd1a..12942906 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -191,6 +191,8 @@ module Invidious::Routes::API::V1::Misc json.object do json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]? json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]? + json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]? + json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]? json.field "params", params.try &.as_s json.field "pageType", pageType end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 9fb283c2..2922b060 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -32,11 +32,14 @@ module Invidious::Routes::API::V1::Search begin client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt" + client.before_request { |r| add_yt_headers(r) } + + url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body + client.close - body = JSON.parse(response[5..-1]).as_a + body = JSON.parse(response[19..-2]).as_a suggestions = body[1].as_a[0..-2] JSON.build do |json| diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 1017ac9d..9281f4dd 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -363,4 +363,47 @@ module Invidious::Routes::API::V1::Videos end end end + + def self.clips(env) + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "application/json" + + clip_id = env.params.url["id"] + region = env.params.query["region"]? + proxy = {"1", "true"}.any? &.== env.params.query["local"]? + + response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}") + return error_json(400, "Invalid clip ID") if response["error"]? + + video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s + return error_json(400, "Invalid clip ID") if video_id.nil? + + start_time = nil + end_time = nil + clip_title = nil + + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, clip_title = parse_clip_parameters(params) + end + + begin + video = get_video(video_id, region: region) + rescue ex : NotFoundException + return error_json(404, ex) + rescue ex + return error_json(500, ex) + end + + return JSON.build do |json| + json.object do + json.field "startTime", start_time + json.field "endTime", end_time + json.field "clipTitle", clip_title + json.field "video" do + Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy) + end + end + end + end end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 40bca008..e20a7139 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -407,14 +407,23 @@ module Invidious::Routes::Feeds end spawn do - rss = XML.parse_html(body) - rss.xpath_nodes("//feed/entry").each do |entry| - id = entry.xpath_node("videoid").not_nil!.content - author = entry.xpath_node("author/name").not_nil!.content - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - - video = get_video(id, force_refresh: true) + # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow? + namespaces = { + "yt" => "http://www.youtube.com/xml/schemas/2015", + "default" => "http://www.w3.org/2005/Atom", + } + rss = XML.parse(body) + rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| + id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content + author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content + published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content) + + begin + video = get_video(id, force_refresh: true) + rescue + next # skip this video since it raised an exception (e.g. it is a scheduled live event) + end if CONFIG.enable_user_notifications # Deliver notifications to `/api/v1/auth/notifications` diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 1d5aa914..ec18f3b8 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region) + client = make_client(URI.parse(new_host), region, force_resolve = true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) rescue ex error = ex.message end @@ -196,7 +196,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve = true) end end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 3d935f0a..aabe8dfc 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -275,6 +275,12 @@ module Invidious::Routes::Watch return error_template(400, "Invalid clip ID") if response["error"]? if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, _ = parse_clip_parameters(params) + env.params.query["start"] = start_time.to_s if start_time != nil + env.params.query["end"] = end_time.to_s if end_time != nil + end + return env.redirect "/watch?v=#{video_id}&#{env.params.query}" else return error_template(404, "The requested clip doesn't exist") diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index d6bd991c..ba05da19 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -235,6 +235,7 @@ module Invidious::Routing get "/api/v1/captions/:id", {{namespace}}::Videos, :captions get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations get "/api/v1/comments/:id", {{namespace}}::Videos, :comments + get "/api/v1/clips/:id", {{namespace}}::Videos, :clips # Feeds get "/api/v1/trending", {{namespace}}::Feeds, :trending diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index c2b5c758..bf968734 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -300,9 +300,9 @@ module Invidious::Search object["9:varint"] = ((page - 1) * 20).to_i64 end - # If the object is empty, return an empty string, - # otherwise encode to protobuf then to base64 - return "" if object.empty? + # Prevent censoring of self harm topics + # See https://github.com/iv-org/invidious/issues/4398 + object["30:varint"] = 1.to_i64 return object .try { |i| Protodec::Any.cast_json(i) } diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 9fbd1374..a8f02056 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -227,8 +227,22 @@ struct Video info.dig?("streamingData", "hlsManifestUrl").try &.as_s end - def dash_manifest_url - info.dig?("streamingData", "dashManifestUrl").try &.as_s + def dash_manifest_url : String? + raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s + return nil if raw_dash_url.nil? + + # Use manifest v5 parameter to reduce file size + # See https://github.com/iv-org/invidious/issues/4186 + dash_url = URI.parse(raw_dash_url) + dash_query = dash_url.query || "" + + if dash_query.empty? + dash_url.path = "#{dash_url.path}/mpd_version/5" + else + dash_url.query = "#{dash_query}&mpd_version=5" + end + + return dash_url.to_s end def genre_url : String? diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr new file mode 100644 index 00000000..29c57182 --- /dev/null +++ b/src/invidious/videos/clip.cr @@ -0,0 +1,22 @@ +require "json" + +# returns start_time, end_time and clip_title +def parse_clip_parameters(params) : {Float64?, Float64?, String?} + decoded_protobuf = params.try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + start_time = decoded_protobuf + .try(&.["50:0:embedded"]["2:1:varint"].as_i64) + .try { |i| i/1000 } + + end_time = decoded_protobuf + .try(&.["50:0:embedded"]["3:2:varint"].as_i64) + .try { |i| i/1000 } + + clip_title = decoded_protobuf + .try(&.["50:0:embedded"]["4:3:string"].as_s) + + return start_time, end_time, clip_title +end diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 031b46da..6d227cfc 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -82,11 +82,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> </div> <% when Category %> <% else %> @@ -160,11 +168,19 @@ </div> <div class="video-card-row flexible"> - <div class="flex-left"><a href="/channel/<%= item.ucid %>"> - <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> - <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> - </p> - </a></div> + <div class="flex-left"> + <% if !item.ucid.to_s.empty? %> + <a href="/channel/<%= item.ucid %>"> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + </a> + <% else %> + <p class="channel-name" dir="auto"><%= HTML.escape(item.author) %> + <%- if author_verified %> <i class="icon ion ion-md-checkmark-circle"></i><% end -%> + </p> + <% end %> + </div> <%= rendered "components/video-context-buttons" %> </div> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 07474896..1b020321 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -346,7 +346,7 @@ we're going to need to do it here in order to allow for translations. <h5 class="pure-g"> <div class="pure-u-14-24"> - <% if rv["ucid"]? %> + <% if !rv["ucid"].empty? %> <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></a></b> <% else %> <b style="width:100%"><%= rv["author"]? %><% if rv["author_verified"]? == "true" %> <i class="icon ion ion-md-checkmark-circle"></i><% end %></b> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 03ff0ee4..81cfb272 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,7 +1,6 @@ def add_yt_headers(request) - if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" - end + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" @@ -27,7 +26,7 @@ struct YoutubeConnectionPool def client(region = nil, &block) if region - conn = make_client(url, region) + conn = make_client(url, region, force_resolve = true) response = yield conn else conn = pool.checkout @@ -60,9 +59,14 @@ struct YoutubeConnectionPool end end -def make_client(url : URI, region = nil) +def make_client(url : URI, region = nil, force_resolve : Bool = false) client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) - client.family = CONFIG.force_resolve + + # Some services do not support IPv6. + if force_resolve + client.family = CONFIG.force_resolve + end + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" client.read_timeout = 10.seconds client.connect_timeout = 10.seconds @@ -81,8 +85,8 @@ def make_client(url : URI, region = nil) return client end -def make_client(url : URI, region = nil, &block) - client = make_client(url, region) +def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) + client = make_client(url, region, force_resolve) begin yield client ensure diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 56325cf7..0e72957e 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -822,9 +822,9 @@ module HelperExtractors end # Retrieves the ID required for querying the InnerTube browse endpoint. - # Raises when it's unable to do so + # Returns an empty string when it's unable to do so def self.get_browse_id(container) - return container.dig("navigationEndpoint", "browseEndpoint", "browseId").as_s + return container.dig?("navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || "" end end |