diff options
Diffstat (limited to 'src')
27 files changed, 503 insertions, 182 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index e0bd0101..3804197e 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -153,6 +153,15 @@ Invidious::Database.check_integrity(CONFIG) {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %} {% end %} +# Misc + +DECRYPT_FUNCTION = + if sig_helper_address = CONFIG.signature_server.presence + IV::DecryptFunction.new(sig_helper_address) + else + nil + end + # Start jobs if CONFIG.channel_threads > 0 @@ -163,11 +172,6 @@ if CONFIG.feed_threads > 0 Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB) end -DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling) -if CONFIG.decrypt_polling - Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new -end - if CONFIG.statistics_enabled Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE) end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index be739673..29546e38 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -232,7 +232,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) id: video_id, title: title, published: published, - updated: Time.utc, + updated: updated, ucid: ucid, author: author, length_seconds: length_seconds, diff --git a/src/invidious/comments/content.cr b/src/invidious/comments/content.cr index beefd9ad..1f55bfe6 100644 --- a/src/invidious/comments/content.cr +++ b/src/invidious/comments/content.cr @@ -5,35 +5,35 @@ def text_to_parsed_content(text : String) : JSON::Any # In first case line is just a simple node before # check patterns inside line # { 'text': line } - currentNodes = [] of JSON::Any - initialNode = {"text" => line} - currentNodes << (JSON.parse(initialNode.to_json)) + current_nodes = [] of JSON::Any + initial_node = {"text" => line} + current_nodes << (JSON.parse(initial_node.to_json)) # For each match with url pattern, get last node and preserve # last node before create new node with url information # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } } - line.scan(/https?:\/\/[^ ]*/).each do |urlMatch| + line.scan(/https?:\/\/[^ ]*/).each do |url_match| # Retrieve last node and update node without match - lastNode = currentNodes[currentNodes.size - 1].as_h - splittedLastNode = lastNode["text"].as_s.split(urlMatch[0]) - lastNode["text"] = JSON.parse(splittedLastNode[0].to_json) - currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) + last_node = current_nodes[-1].as_h + splitted_last_node = last_node["text"].as_s.split(url_match[0]) + last_node["text"] = JSON.parse(splitted_last_node[0].to_json) + current_nodes[-1] = JSON.parse(last_node.to_json) # Create new node with match and navigation infos - currentNode = {"text" => urlMatch[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => urlMatch[0]}}} - currentNodes << (JSON.parse(currentNode.to_json)) + current_node = {"text" => url_match[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => url_match[0]}}} + current_nodes << (JSON.parse(current_node.to_json)) # If text remain after match create new simple node with text after match - afterNode = {"text" => splittedLastNode.size > 1 ? splittedLastNode[1] : ""} - currentNodes << (JSON.parse(afterNode.to_json)) + after_node = {"text" => splitted_last_node.size > 1 ? splitted_last_node[1] : ""} + current_nodes << (JSON.parse(after_node.to_json)) end # After processing of matches inside line # Add \n at end of last node for preserve carriage return - lastNode = currentNodes[currentNodes.size - 1].as_h - lastNode["text"] = JSON.parse("#{currentNodes[currentNodes.size - 1]["text"]}\n".to_json) - currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json) + last_node = current_nodes[-1].as_h + last_node["text"] = JSON.parse("#{last_node["text"]}\n".to_json) + current_nodes[-1] = JSON.parse(last_node.to_json) # Finally add final nodes to nodes returned - currentNodes.each do |node| + current_nodes.each do |node| nodes << (node) end end @@ -53,8 +53,8 @@ def content_to_comment_html(content, video_id : String? = "") text = HTML.escape(run["text"].as_s) - if navigationEndpoint = run.dig?("navigationEndpoint") - text = parse_link_endpoint(navigationEndpoint, text, video_id) + if navigation_endpoint = run.dig?("navigationEndpoint") + text = parse_link_endpoint(navigation_endpoint, text, video_id) end text = "<b>#{text}</b>" if run["bold"]? diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 5340d4f5..c4ddcdb3 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -74,8 +74,6 @@ class Config # Database configuration using 12-Factor "Database URL" syntax @[YAML::Field(converter: Preferences::URIConverter)] property database_url : URI = URI.parse("") - # Use polling to keep decryption function up to date - property decrypt_polling : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel property full_refresh : Bool = false @@ -120,6 +118,10 @@ class Config # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) @[YAML::Field(converter: Preferences::FamilyConverter)] property force_resolve : Socket::Family = Socket::Family::UNSPEC + + # External signature solver server socket (either a path to a UNIX domain socket or "<IP>:<Port>") + property signature_server : String? = nil + # Port to listen for connections (overridden by command line argument) property port : Int32 = 3000 # Host to bind (overridden by command line argument) diff --git a/src/invidious/frontend/comments_youtube.cr b/src/invidious/frontend/comments_youtube.cr index aecac87f..a0e1d783 100644 --- a/src/invidious/frontend/comments_youtube.cr +++ b/src/invidious/frontend/comments_youtube.cr @@ -149,12 +149,12 @@ module Invidious::Frontend::Comments if comments["videoId"]? html << <<-END_HTML - <a href="https://www.youtube.com/watch?v=#{comments["videoId"]}&lc=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> + <a rel="noreferrer noopener" href="https://www.youtube.com/watch?v=#{comments["videoId"]}&lc=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> | END_HTML elsif comments["authorId"]? html << <<-END_HTML - <a href="https://www.youtube.com/channel/#{comments["authorId"]}/community?lb=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> + <a rel="noreferrer noopener" href="https://www.youtube.com/channel/#{comments["authorId"]}/community?lb=#{child["commentId"]}" title="#{translate(locale, "YouTube comment permalink")}">[YT]</a> | END_HTML end diff --git a/src/invidious/frontend/misc.cr b/src/invidious/frontend/misc.cr index 43ba9f5c..7a6cf79d 100644 --- a/src/invidious/frontend/misc.cr +++ b/src/invidious/frontend/misc.cr @@ -6,9 +6,9 @@ module Invidious::Frontend::Misc if prefs.automatic_instance_redirect current_page = env.get?("current_page").as(String) - redirect_url = "/redirect?referer=#{current_page}" + return "/redirect?referer=#{current_page}" else - redirect_url = "https://redirect.invidious.io#{env.request.resource}" + return "https://redirect.invidious.io#{env.request.resource}" end end end diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index bf56d826..fec3f62c 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -3,9 +3,9 @@ # IPv6 addresses. # class TCPSocket - def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, family = Socket::Family::UNSPEC) + def initialize(host, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC) Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| - super(addrinfo.family, addrinfo.type, addrinfo.protocol) + super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking) connect(addrinfo, timeout: connect_timeout) do |error| close error @@ -26,7 +26,7 @@ class HTTP::Client end hostname = @host.starts_with?('[') && @host.ends_with?(']') ? @host[1..-2] : @host - io = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout, @family + io = TCPSocket.new hostname, @port, @dns_timeout, @connect_timeout, family: @family io.read_timeout = @read_timeout if @read_timeout io.write_timeout = @write_timeout if @write_timeout io.sync = false @@ -35,7 +35,7 @@ class HTTP::Client if tls = @tls tcp_socket = io begin - io = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host) + io = OpenSSL::SSL::Socket::Client.new(tcp_socket, context: tls, sync_close: true, hostname: @host.rchop('.')) rescue exc # don't leak the TCP socket when the SSL connection failed tcp_socket.close diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 21b789bc..b2df682d 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -190,7 +190,7 @@ def error_redirect_helper(env : HTTP::Server::Context) <a href="/redirect?referer=#{env.get("current_page")}">#{switch_instance}</a> </li> <li> - <a href="https://youtube.com#{env.request.resource}">#{go_to_youtube}</a> + <a rel="noreferrer noopener" href="https://youtube.com#{env.request.resource}">#{go_to_youtube}</a> </li> </ul> END_HTML diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 174f620d..f3e3b951 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -97,7 +97,7 @@ class AuthHandler < Kemal::Handler if token = env.request.headers["Authorization"]? token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) session = URI.decode_www_form(token["session"].as_s) - scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, nil) + scopes, _, _ = validate_request(token, session, env.request, HMAC_KEY, nil) if email = Invidious::Database::SessionIDs.select_email(session) user = Invidious::Database::Users.select!(email: email) diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index 9f4077e1..684e6d14 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -95,7 +95,6 @@ module I18next::Plurals "hr" => PluralForms::Special_Hungarian_Serbian, "it" => PluralForms::Special_Spanish_Italian, "pt" => PluralForms::Special_French_Portuguese, - "pt" => PluralForms::Special_French_Portuguese, "sr" => PluralForms::Special_Hungarian_Serbian, } @@ -189,7 +188,7 @@ module I18next::Plurals # Emulate the `rule.numbers.size == 2 && rule.numbers[0] == 1` check # from original i18next code - private def is_simple_plural(form : PluralForms) : Bool + private def simple_plural?(form : PluralForms) : Bool case form when .single_gt_one? then return true when .single_not_one? then return true @@ -211,7 +210,7 @@ module I18next::Plurals idx = SuffixIndex.get_index(plural_form, count) # Simple plurals are handled differently in all versions (but v4) - if @simplify_plural_suffix && is_simple_plural(plural_form) + if @simplify_plural_suffix && simple_plural?(plural_form) return (idx == 1) ? "_plural" : "" end @@ -262,9 +261,9 @@ module I18next::Plurals when .special_hebrew? then return special_hebrew(count) when .special_odia? then return special_odia(count) # Mixed v3/v4 forms - when .special_spanish_italian? then return special_cldr_Spanish_Italian(count) - when .special_french_portuguese? then return special_cldr_French_Portuguese(count) - when .special_hungarian_serbian? then return special_cldr_Hungarian_Serbian(count) + when .special_spanish_italian? then return special_cldr_spanish_italian(count) + when .special_french_portuguese? then return special_cldr_french_portuguese(count) + when .special_hungarian_serbian? then return special_cldr_hungarian_serbian(count) else # default, if nothing matched above return 0_u8 @@ -535,7 +534,7 @@ module I18next::Plurals # # This rule is mostly compliant to CLDR v42 # - def self.special_cldr_Spanish_Italian(count : Int) : UInt8 + def self.special_cldr_spanish_italian(count : Int) : UInt8 return 0_u8 if (count == 1) # one return 1_u8 if (count != 0 && count % 1_000_000 == 0) # many return 2_u8 # other @@ -545,7 +544,7 @@ module I18next::Plurals # # This rule is mostly compliant to CLDR v42 # - def self.special_cldr_French_Portuguese(count : Int) : UInt8 + def self.special_cldr_french_portuguese(count : Int) : UInt8 return 0_u8 if (count == 0 || count == 1) # one return 1_u8 if (count % 1_000_000 == 0) # many return 2_u8 # other @@ -555,7 +554,7 @@ module I18next::Plurals # # This rule is mostly compliant to CLDR v42 # - def self.special_cldr_Hungarian_Serbian(count : Int) : UInt8 + def self.special_cldr_hungarian_serbian(count : Int) : UInt8 n_mod_10 = count % 10 n_mod_100 = count % 100 diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index e2e50905..b443073e 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -34,24 +34,11 @@ class Invidious::LogHandler < Kemal::BaseLogHandler context end - def puts(message : String) - @io << message << '\n' - @io.flush - end - def write(message : String) @io << message @io.flush end - def set_log_level(level : String) - @level = LogLevel.parse(level) - end - - def set_log_level(level : LogLevel) - @level = level - end - {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr new file mode 100644 index 00000000..9e72c1c7 --- /dev/null +++ b/src/invidious/helpers/sig_helper.cr @@ -0,0 +1,332 @@ +require "uri" +require "socket" +require "socket/tcp_socket" +require "socket/unix_socket" + +{% if flag?(:advanced_debug) %} + require "io/hexdump" +{% end %} + +private alias NetworkEndian = IO::ByteFormat::NetworkEndian + +module Invidious::SigHelper + enum UpdateStatus + Updated + UpdateNotRequired + Error + end + + # ------------------- + # Payload types + # ------------------- + + abstract struct Payload + end + + struct StringPayload < Payload + getter string : String + + def initialize(str : String) + raise Exception.new("SigHelper: String can't be empty") if str.empty? + @string = str + end + + def self.from_bytes(slice : Bytes) + size = IO::ByteFormat::NetworkEndian.decode(UInt16, slice) + if size == 0 # Error code + raise Exception.new("SigHelper: Server encountered an error") + end + + if (slice.bytesize - 2) != size + raise Exception.new("SigHelper: String size mismatch") + end + + if str = String.new(slice[2..]) + return self.new(str) + else + raise Exception.new("SigHelper: Can't read string from socket") + end + end + + def to_io(io) + # `.to_u16` raises if there is an overflow during the conversion + io.write_bytes(@string.bytesize.to_u16, NetworkEndian) + io.write(@string.to_slice) + end + end + + private enum Opcode + FORCE_UPDATE = 0 + DECRYPT_N_SIGNATURE = 1 + DECRYPT_SIGNATURE = 2 + GET_SIGNATURE_TIMESTAMP = 3 + GET_PLAYER_STATUS = 4 + PLAYER_UPDATE_TIMESTAMP = 5 + end + + private record Request, + opcode : Opcode, + payload : Payload? + + # ---------------------- + # High-level functions + # ---------------------- + + class Client + @mux : Multiplexor + + def initialize(uri_or_path) + @mux = Multiplexor.new(uri_or_path) + end + + # Forces the server to re-fetch the YouTube player, and extract the necessary + # components from it (nsig function code, sig function code, signature timestamp). + def force_update : UpdateStatus + request = Request.new(Opcode::FORCE_UPDATE, nil) + + value = send_request(request) do |bytes| + IO::ByteFormat::NetworkEndian.decode(UInt16, bytes) + end + + case value + when 0x0000 then return UpdateStatus::Error + when 0xFFFF then return UpdateStatus::UpdateNotRequired + when 0xF44F then return UpdateStatus::Updated + else + code = value.nil? ? "nil" : value.to_s(base: 16) + raise Exception.new("SigHelper: Invalid status code received #{code}") + end + end + + # Decrypt a provided n signature using the server's current nsig function + # code, and return the result (or an error). + def decrypt_n_param(n : String) : String? + request = Request.new(Opcode::DECRYPT_N_SIGNATURE, StringPayload.new(n)) + + n_dec = self.send_request(request) do |bytes| + StringPayload.from_bytes(bytes).string + end + + return n_dec + end + + # Decrypt a provided s signature using the server's current sig function + # code, and return the result (or an error). + def decrypt_sig(sig : String) : String? + request = Request.new(Opcode::DECRYPT_SIGNATURE, StringPayload.new(sig)) + + sig_dec = self.send_request(request) do |bytes| + StringPayload.from_bytes(bytes).string + end + + return sig_dec + end + + # Return the signature timestamp from the server's current player + def get_signature_timestamp : UInt64? + request = Request.new(Opcode::GET_SIGNATURE_TIMESTAMP, nil) + + return self.send_request(request) do |bytes| + IO::ByteFormat::NetworkEndian.decode(UInt64, bytes) + end + end + + # Return the current player's version + def get_player : UInt32? + request = Request.new(Opcode::GET_PLAYER_STATUS, nil) + + return self.send_request(request) do |bytes| + has_player = (bytes[0] == 0xFF) + player_version = IO::ByteFormat::NetworkEndian.decode(UInt32, bytes[1..4]) + has_player ? player_version : nil + end + end + + # Return when the player was last updated + def get_player_timestamp : UInt64? + request = Request.new(Opcode::PLAYER_UPDATE_TIMESTAMP, nil) + + return self.send_request(request) do |bytes| + IO::ByteFormat::NetworkEndian.decode(UInt64, bytes) + end + end + + private def send_request(request : Request, &) + channel = @mux.send(request) + slice = channel.receive + return yield slice + rescue ex + LOGGER.debug("SigHelper: Error when sending a request") + LOGGER.trace(ex.inspect_with_backtrace) + return nil + end + end + + # --------------------- + # Low level functions + # --------------------- + + class Multiplexor + alias TransactionID = UInt32 + record Transaction, channel = ::Channel(Bytes).new + + @prng = Random.new + @mutex = Mutex.new + @queue = {} of TransactionID => Transaction + + @conn : Connection + + def initialize(uri_or_path) + @conn = Connection.new(uri_or_path) + listen + end + + def listen : Nil + raise "Socket is closed" if @conn.closed? + + LOGGER.debug("SigHelper: Multiplexor listening") + + # TODO: reopen socket if unexpectedly closed + spawn do + loop do + receive_data + Fiber.yield + end + end + end + + def send(request : Request) + transaction = Transaction.new + transaction_id = @prng.rand(TransactionID) + + # Add transaction to queue + @mutex.synchronize do + # On a 32-bits random integer, this should never happen. Though, just in case, ... + if @queue[transaction_id]? + raise Exception.new("SigHelper: Duplicate transaction ID! You got a shiny pokemon!") + end + + @queue[transaction_id] = transaction + end + + write_packet(transaction_id, request) + + return transaction.channel + end + + def receive_data + transaction_id, slice = read_packet + + @mutex.synchronize do + if transaction = @queue.delete(transaction_id) + # Remove transaction from queue and send data to the channel + transaction.channel.send(slice) + LOGGER.trace("SigHelper: Transaction unqueued and data sent to channel") + else + raise Exception.new("SigHelper: Received transaction was not in queue") + end + end + end + + # Read a single packet from the socket + private def read_packet : {TransactionID, Bytes} + # Header + transaction_id = @conn.read_bytes(UInt32, NetworkEndian) + length = @conn.read_bytes(UInt32, NetworkEndian) + + LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} / length #{length}") + + if length > 67_000 + raise Exception.new("SigHelper: Packet longer than expected (#{length})") + end + + # Payload + slice = Bytes.new(length) + @conn.read(slice) if length > 0 + + LOGGER.trace("SigHelper: payload = #{slice}") + LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} - Done") + + return transaction_id, slice + end + + # Write a single packet to the socket + private def write_packet(transaction_id : TransactionID, request : Request) + LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} / opcode #{request.opcode}") + + io = IO::Memory.new(1024) + io.write_bytes(request.opcode.to_u8, NetworkEndian) + io.write_bytes(transaction_id, NetworkEndian) + + if payload = request.payload + payload.to_io(io) + end + + @conn.send(io) + @conn.flush + + LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} - Done") + end + end + + class Connection + @socket : UNIXSocket | TCPSocket + + {% if flag?(:advanced_debug) %} + @io : IO::Hexdump + {% end %} + + def initialize(host_or_path : String) + case host_or_path + when .starts_with?('/') + # Make sure that the file exists + if File.exists?(host_or_path) + @socket = UNIXSocket.new(host_or_path) + else + raise Exception.new("SigHelper: '#{host_or_path}' no such file") + end + when .starts_with?("tcp://") + uri = URI.parse(host_or_path) + @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!) + else + uri = URI.parse("tcp://#{host_or_path}") + @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!) + end + LOGGER.info("SigHelper: Using helper at '#{host_or_path}'") + + {% if flag?(:advanced_debug) %} + @io = IO::Hexdump.new(@socket, output: STDERR, read: true, write: true) + {% end %} + + @socket.sync = false + @socket.blocking = false + end + + def closed? : Bool + return @socket.closed? + end + + def close : Nil + @socket.close if !@socket.closed? + end + + def flush(*args, **options) + @socket.flush(*args, **options) + end + + def send(*args, **options) + @socket.send(*args, **options) + end + + # Wrap IO functions, with added debug tooling if needed + {% for function in %w(read read_bytes write write_bytes) %} + def {{function.id}}(*args, **options) + {% if flag?(:advanced_debug) %} + @io.{{function.id}}(*args, **options) + {% else %} + @socket.{{function.id}}(*args, **options) + {% end %} + end + {% end %} + end +end diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr index ee09415b..84a8a86d 100644 --- a/src/invidious/helpers/signatures.cr +++ b/src/invidious/helpers/signatures.cr @@ -1,73 +1,55 @@ -alias SigProc = Proc(Array(String), Int32, Array(String)) +require "http/params" +require "./sig_helper" -struct DecryptFunction - @decrypt_function = [] of {SigProc, Int32} - @decrypt_time = Time.monotonic +class Invidious::DecryptFunction + @last_update : Time = Time.utc - 42.days - def initialize(@use_polling = true) + def initialize(uri_or_path) + @client = SigHelper::Client.new(uri_or_path) + self.check_update end - def update_decrypt_function - @decrypt_function = fetch_decrypt_function - end - - private def fetch_decrypt_function(id = "CvFH_6DNRCY") - document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body - url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"] - player = YT_POOL.client &.get(url).body - - function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"] - function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?<body>[^}]+)}/m).not_nil!["body"] - function_body = function_body.split(";")[1..-2] - - var_name = function_body[0][0, 2] - var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?<body>(.*?))};/).not_nil!["body"] - - operations = {} of String => SigProc - var_body.split("},").each do |operation| - op_name = operation.match(/^[^:]+/).not_nil![0] - op_body = operation.match(/\{[^}]+/).not_nil![0] - - case op_body - when "{a.reverse()" - operations[op_name] = ->(a : Array(String), _b : Int32) { a.reverse } - when "{a.splice(0,b)" - operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a } - else - operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a } - end - end + def check_update + now = Time.utc - decrypt_function = [] of {SigProc, Int32} - function_body.each do |function| - function = function.lchop(var_name).delete("[].") + # If we have updated in the last 5 minutes, do nothing + return if (now - @last_update) > 5.minutes - op_name = function.match(/[^\(]+/).not_nil![0] - value = function.match(/\(\w,(?<value>[\d]+)\)/).not_nil!["value"].to_i + # Get the amount of time elapsed since when the player was updated, in the + # event where multiple invidious processes are run in parallel. + update_time_elapsed = (@client.get_player_timestamp || 301).seconds - decrypt_function << {operations[op_name], value} + if update_time_elapsed > 5.minutes + LOGGER.debug("Signature: Player might be outdated, updating") + @client.force_update + @last_update = Time.utc end - - return decrypt_function end - def decrypt_signature(fmt : Hash(String, JSON::Any)) - return "" if !fmt["s"]? || !fmt["sp"]? - - sp = fmt["sp"].as_s - sig = fmt["s"].as_s.split("") - if !@use_polling - now = Time.monotonic - if now - @decrypt_time > 60.seconds || @decrypt_function.size == 0 - @decrypt_function = fetch_decrypt_function - @decrypt_time = Time.monotonic - end - end + def decrypt_nsig(n : String) : String? + self.check_update + return @client.decrypt_n_param(n) + rescue ex + LOGGER.debug(ex.message || "Signature: Unknown error") + LOGGER.trace(ex.inspect_with_backtrace) + return nil + end - @decrypt_function.each do |proc, value| - sig = proc.call(sig, value) - end + def decrypt_signature(str : String) : String? + self.check_update + return @client.decrypt_sig(str) + rescue ex + LOGGER.debug(ex.message || "Signature: Unknown error") + LOGGER.trace(ex.inspect_with_backtrace) + return nil + end - return "&#{sp}=#{sig.join("")}" + def get_sts : UInt64? + self.check_update + return @client.get_signature_timestamp + rescue ex + LOGGER.debug(ex.message || "Signature: Unknown error") + LOGGER.trace(ex.inspect_with_backtrace) + return nil end end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index e438e3b9..8e9e9a6a 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -52,9 +52,9 @@ def recode_length_seconds(time) end def decode_interval(string : String) : Time::Span - rawMinutes = string.try &.to_i32? + raw_minutes = string.try &.to_i32? - if !rawMinutes + if !raw_minutes hours = /(?<hours>\d+)h/.match(string).try &.["hours"].try &.to_i32 hours ||= 0 @@ -63,7 +63,7 @@ def decode_interval(string : String) : Time::Span time = Time::Span.new(hours: hours, minutes: minutes) else - time = Time::Span.new(minutes: rawMinutes) + time = Time::Span.new(minutes: raw_minutes) end return time diff --git a/src/invidious/jobs/update_decrypt_function_job.cr b/src/invidious/jobs/update_decrypt_function_job.cr deleted file mode 100644 index 6fa0ae1b..00000000 --- a/src/invidious/jobs/update_decrypt_function_job.cr +++ /dev/null @@ -1,14 +0,0 @@ -class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob - def begin - loop do - begin - DECRYPT_FUNCTION.update_decrypt_function - rescue ex - LOGGER.error("UpdateDecryptFunctionJob : #{ex.message}") - ensure - sleep 1.minute - Fiber.yield - end - end - end -end diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index 9d930841..dd65e7a6 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -53,7 +53,7 @@ module Invidious::Routes::Account return error_template(401, "Password is a required field") end - new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } + new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v } if new_passwords.size <= 1 || new_passwords.uniq.size != 1 return error_template(400, "New passwords must match") @@ -240,7 +240,7 @@ module Invidious::Routes::Account return error_template(400, ex) end - scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } + scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v } callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 0c79692d..093669fe 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -179,8 +179,8 @@ module Invidious::Routes::API::V1::Misc begin resolved_url = YoutubeAPI.resolve_url(url.as(String)) endpoint = resolved_url["endpoint"] - pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" - if pageType == "WEB_PAGE_TYPE_UNKNOWN" + page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" + if page_type == "WEB_PAGE_TYPE_UNKNOWN" return error_json(400, "Unknown url") end @@ -196,7 +196,7 @@ module Invidious::Routes::API::V1::Misc json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]? json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]? json.field "params", params.try &.as_s - json.field "pageType", pageType + json.field "pageType", page_type end end end diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 5e269923..42282f44 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -219,7 +219,7 @@ module Invidious::Routes::API::V1::Videos storyboard[:storyboard_count].times do |i| url = storyboard[:url] - authority = /(i\d?).ytimg.com/.match(url).not_nil![1]? + authority = /(i\d?).ytimg.com/.match!(url)[1]? url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") url = "#{HOST_URL}/sb/#{authority}/#{url}" @@ -254,7 +254,7 @@ module Invidious::Routes::API::V1::Videos if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id)) annotations = cached_annotation.annotations else - index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') + index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0') # IA doesn't handle leading hyphens, # so we use https://archive.org/details/youtubeannotations_64 diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 108f2ccc..a70434ca 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -124,7 +124,7 @@ struct Invidious::User playlist = create_playlist(title, privacy, user) Invidious::Database::Playlists.update_description(playlist.id, description) - videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| + item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| if idx > CONFIG.playlist_length_limit raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") end @@ -182,7 +182,7 @@ struct Invidious::User if is_opml?(type, extension) subscriptions = XML.parse(body) user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0] end elsif extension == "json" || type == "application/json" subscriptions = JSON.parse(body) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 44ed53ee..0d26b395 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -98,20 +98,48 @@ struct Video # Methods for parsing streaming data + def convert_url(fmt) + if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) } + sp = cfr["sp"] + url = URI.parse(cfr["url"]) + params = url.query_params + + LOGGER.debug("Videos: Decoding '#{cfr}'") + + unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"]) + params[sp] = unsig if unsig + else + url = URI.parse(fmt["url"].as_s) + params = url.query_params + end + + n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"]) + params["n"] = n if n + params["pot"] = CONFIG.po_token if CONFIG.po_token + + params["host"] = url.host.not_nil! + if region = self.info["region"]?.try &.as_s + params["region"] = region + end + + url.query_params = params + LOGGER.trace("Videos: new url is '#{url}'") + + return url.to_s + rescue ex + LOGGER.debug("Videos: Error when parsing video URL") + LOGGER.trace(ex.inspect_with_backtrace) + return "" + end + def fmt_stream return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream - fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - fmt_stream.each do |fmt| - if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } - s.each do |k, v| - fmt[k] = JSON::Any.new(v) - end - fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") - end + fmt_stream = info.dig?("streamingData", "formats") + .try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}&pot=#{CONFIG.po_token}") - fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? + fmt_stream.each do |fmt| + fmt["url"] = JSON::Any.new(self.convert_url(fmt)) end fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } @@ -121,21 +149,17 @@ struct Video def adaptive_fmts return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts - fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - fmt_stream.each do |fmt| - if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } - s.each do |k, v| - fmt[k] = JSON::Any.new(v) - end - fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") - end - fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}&pot=#{CONFIG.po_token}") - fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? + fmt_stream = info.dig("streamingData", "adaptiveFormats") + .try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) + + fmt_stream.each do |fmt| + fmt["url"] = JSON::Any.new(self.convert_url(fmt)) end fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } @adaptive_fmts = fmt_stream + return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) end @@ -394,10 +418,6 @@ end def fetch_video(id, region) info = extract_video_info(video_id: id) - allowed_regions = info - .dig?("microformat", "playerMicroformatRenderer", "availableCountries") - .try &.as_a.map &.as_s || [] of String - if reason = info["reason"]? if reason == "Video unavailable" raise NotFoundException.new(reason.as_s || "") diff --git a/src/invidious/views/components/video-context-buttons.ecr b/src/invidious/views/components/video-context-buttons.ecr index 385ed6b3..22458a03 100644 --- a/src/invidious/views/components/video-context-buttons.ecr +++ b/src/invidious/views/components/video-context-buttons.ecr @@ -1,6 +1,6 @@ <div class="flex-right flexible"> <div class="icon-buttons"> - <a title="<%=translate(locale, "videoinfo_watch_on_youTube")%>" href="https://www.youtube.com/watch<%=endpoint_params%>"> + <a title="<%=translate(locale, "videoinfo_watch_on_youTube")%>" rel="noreferrer noopener" href="https://www.youtube.com/watch<%=endpoint_params%>"> <i class="icon ion-logo-youtube"></i> </a> <a title="<%=translate(locale, "Audio mode")%>" href="/watch<%=endpoint_params%>&listen=1"> diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index 24ba437d..c27ddba6 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -83,7 +83,7 @@ <% if !playlist.is_a? InvidiousPlaylist %> <div class="pure-u-2-3"> - <a href="https://www.youtube.com/playlist?list=<%= playlist.id %>"> + <a rel="noreferrer noopener" href="https://www.youtube.com/playlist?list=<%= playlist.id %>"> <%= translate(locale, "View playlist on YouTube") %> </a> <span> | </span> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 9e7467dd..36679bce 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -123,8 +123,8 @@ we're going to need to do it here in order to allow for translations. link_yt_embed = IV::HttpServer::Utils.add_params_to_url(link_yt_embed, link_yt_param) end -%> - <a id="link-yt-watch" data-base-url="<%= link_yt_watch %>" href="<%= link_yt_watch %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a> - (<a id="link-yt-embed" data-base-url="<%= link_yt_embed %>" href="<%= link_yt_embed %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>) + <a id="link-yt-watch" rel="noreferrer noopener" data-base-url="<%= link_yt_watch %>" href="<%= link_yt_watch %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a> + (<a id="link-yt-embed" rel="noreferrer noopener" data-base-url="<%= link_yt_embed %>" href="<%= link_yt_embed %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>) </span> <p id="watch-on-another-invidious-instance"> diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index d3dbcc0e..0ac785e6 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -24,7 +24,7 @@ struct YoutubeConnectionPool @pool = build_pool() end - def client(&block) + def client(&) conn = pool.checkout begin response = yield conn @@ -69,7 +69,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) return client end -def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) +def make_client(url : URI, region = nil, force_resolve : Bool = false, &) client = make_client(url, region, force_resolve) begin yield client diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 0e72957e..38dc2c04 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -109,7 +109,6 @@ private module Parsers end live_now = false - paid = false premium = false premiere_timestamp = item_contents.dig?("upcomingEventData", "startTime").try { |t| Time.unix(t.as_s.to_i64) } @@ -856,7 +855,7 @@ end # # This function yields the container so that items can be parsed separately. # -def extract_items(initial_data : InitialData, &block) +def extract_items(initial_data : InitialData, &) if unpackaged_data = initial_data["contents"]?.try &.as_h elsif unpackaged_data = initial_data["response"]?.try &.as_h elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 1).try &.as_h diff --git a/src/invidious/yt_backend/extractors_utils.cr b/src/invidious/yt_backend/extractors_utils.cr index 11d95958..c83a2de5 100644 --- a/src/invidious/yt_backend/extractors_utils.cr +++ b/src/invidious/yt_backend/extractors_utils.cr @@ -83,5 +83,5 @@ end def extract_selected_tab(tabs) # Extract the selected tab from the array of tabs Youtube returns - return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"] + return tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"] end diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 0efbe949..6d585bf2 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -272,7 +272,7 @@ module YoutubeAPI # Return, as a Hash, the "context" data required to request the # youtube API endpoints. # - private def make_context(client_config : ClientConfig | Nil) : Hash + private def make_context(client_config : ClientConfig | Nil, video_id = "dQw4w9WgXcQ") : Hash # Use the default client config if nil is passed client_config ||= DEFAULT_CLIENT_CONFIG @@ -292,7 +292,7 @@ module YoutubeAPI if client_config.screen == "EMBED" client_context["thirdParty"] = { - "embedUrl" => "https://www.youtube.com/embed/dQw4w9WgXcQ", + "embedUrl" => "https://www.youtube.com/embed/#{video_id}", } of String => String | Int64 end @@ -457,19 +457,29 @@ module YoutubeAPI params : String, client_config : ClientConfig | Nil = nil ) + # Playback context, separate because it can be different between clients + playback_ctx = { + "html5Preference" => "HTML5_PREF_WANTS", + "referer" => "https://www.youtube.com/watch?v=#{video_id}", + } of String => String | Int64 + + if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s } + if sts = DECRYPT_FUNCTION.try &.get_sts + playback_ctx["signatureTimestamp"] = sts.to_i64 + end + end + # JSON Request data, required by the API data = { "contentCheckOk" => true, "videoId" => video_id, - "context" => self.make_context(client_config), + "context" => self.make_context(client_config, video_id), "racyCheckOk" => true, "user" => { "lockedSafetyMode" => false, }, "playbackContext" => { - "contentPlaybackContext" => { - "html5Preference": "HTML5_PREF_WANTS", - }, + "contentPlaybackContext" => playback_ctx, }, "serviceIntegrityDimensions" => { "poToken" => CONFIG.po_token, |