diff options
| -rw-r--r-- | assets/js/player.js | 1 | ||||
| -rw-r--r-- | config/config.example.yml | 16 | ||||
| -rw-r--r-- | locales/de.json | 1 | ||||
| -rw-r--r-- | locales/en-US.json | 1 | ||||
| -rw-r--r-- | src/invidious.cr | 2 | ||||
| -rw-r--r-- | src/invidious/config.cr | 1 | ||||
| -rw-r--r-- | src/invidious/helpers/errors.cr | 2 | ||||
| -rw-r--r-- | src/invidious/helpers/utils.cr | 62 | ||||
| -rw-r--r-- | src/invidious/jobs/instance_refresh_job.cr | 97 | ||||
| -rw-r--r-- | src/invidious/routes/misc.cr | 11 | ||||
| -rw-r--r-- | src/invidious/routes/preferences.cr | 5 | ||||
| -rw-r--r-- | src/invidious/user/preferences.cr | 1 | ||||
| -rw-r--r-- | src/invidious/videos.cr | 80 | ||||
| -rw-r--r-- | src/invidious/videos/parser.cr | 45 | ||||
| -rw-r--r-- | src/invidious/videos/video_preferences.cr | 6 | ||||
| -rw-r--r-- | src/invidious/views/components/player.ecr | 1 | ||||
| -rw-r--r-- | src/invidious/views/user/preferences.ecr | 5 |
17 files changed, 205 insertions, 132 deletions
diff --git a/assets/js/player.js b/assets/js/player.js index d32062c6..353a5296 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -3,7 +3,6 @@ var player_data = JSON.parse(document.getElementById('player_data').textContent) var video_data = JSON.parse(document.getElementById('video_data').textContent); var options = { - preload: 'auto', liveui: true, playbackRates: [0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], controlBar: { diff --git a/config/config.example.yml b/config/config.example.yml index 219aa03f..e9eebfde 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -708,6 +708,22 @@ default_user_preferences: # ----------------------------- ## + ## This option controls the value of the HTML5 <video> element's + ## "preload" attribute. + ## + ## If set to 'false', no video data will be loaded until the user + ## explicitly starts the video by clicking the "Play" button. + ## If set to 'true', the web browser will buffer some video data + ## while the page is loading. + ## + ## See: https://www.w3schools.com/tags/att_video_preload.asp + ## + ## Accepted values: true, false + ## Default: true + ## + #preload: true + + ## ## Automatically play videos on page load. ## ## Accepted values: true, false diff --git a/locales/de.json b/locales/de.json index 3d82d610..151f2abe 100644 --- a/locales/de.json +++ b/locales/de.json @@ -47,6 +47,7 @@ "Preferences": "Einstellungen", "preferences_category_player": "Wiedergabeeinstellungen", "preferences_video_loop_label": "Immer wiederholen: ", + "preferences_preload_label": "Videodaten vorladen: ", "preferences_autoplay_label": "Automatisch abspielen: ", "preferences_continue_label": "Immer automatisch nächstes Video abspielen: ", "preferences_continue_autoplay_label": "Nächstes Video automatisch abspielen: ", diff --git a/locales/en-US.json b/locales/en-US.json index 441bb57c..7827d9c6 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -71,6 +71,7 @@ "Preferences": "Preferences", "preferences_category_player": "Player preferences", "preferences_video_loop_label": "Always loop: ", + "preferences_preload_label": "Preload video data: ", "preferences_autoplay_label": "Autoplay: ", "preferences_continue_label": "Play next by default: ", "preferences_continue_autoplay_label": "Autoplay next video: ", diff --git a/src/invidious.cr b/src/invidious.cr index 3804197e..63f2a9cc 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -189,6 +189,8 @@ Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new +Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new + Invidious::Jobs.start_all def popular_videos diff --git a/src/invidious/config.cr b/src/invidious/config.cr index c4ddcdb3..a097b7f1 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -13,6 +13,7 @@ struct ConfigPreferences property annotations : Bool = false property annotations_subscribed : Bool = false + property preload : Bool = true property autoplay : Bool = false property captions : Array(String) = ["", "", ""] property comments : Array(String) = ["youtube", ""] diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index b2df682d..b7643194 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -43,6 +43,8 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce # URLs for the error message below url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md" url_search_issues = "https://github.com/iv-org/invidious/issues" + url_search_issues += "?q=is:issue+is:open+" + url_search_issues += URI.encode_www_form("[Bug] #{issue_title}") url_switch = "https://redirect.invidious.io" + env.request.resource diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 8e9e9a6a..4d9bb28d 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -323,68 +323,6 @@ def parse_range(range) return 0_i64, nil end -def fetch_random_instance - begin - instance_api_client = make_client(URI.parse("https://api.invidious.io")) - - # Timeouts - instance_api_client.connect_timeout = 10.seconds - instance_api_client.dns_timeout = 10.seconds - - instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a - instance_api_client.close - rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException - instance_list = [] of JSON::Any - end - - filtered_instance_list = [] of String - - instance_list.each do |data| - # TODO Check if current URL is onion instance and use .onion types if so. - if data[1]["type"] == "https" - # Instances can have statistics disabled, which is an requirement of version validation. - # as_nil? doesn't exist. Thus we'll have to handle the error raised if as_nil fails. - begin - data[1]["stats"].as_nil - next - rescue TypeCastError - end - - # stats endpoint could also lack the software dict. - next if data[1]["stats"]["software"]?.nil? - - # Makes sure the instance isn't too outdated. - if remote_version = data[1]["stats"]?.try &.["software"]?.try &.["version"] - remote_commit_date = remote_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) - next if !remote_commit_date - - remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) - local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) - - next if (remote_commit_date - local_commit_date).abs.days > 30 - - begin - data[1]["monitor"].as_nil - health = data[1]["monitor"].as_h["dailyRatios"][0].as_h["ratio"] - filtered_instance_list << data[0].as_s if health.to_s.to_f > 90 - rescue TypeCastError - # We can't check the health if the monitoring is broken. Thus we'll just add it to the list - # and move on. Ideally we'll ignore any instance that has broken health monitoring but due to the fact that - # it's an error that often occurs with all the instances at the same time, we have to just skip the check. - filtered_instance_list << data[0].as_s - end - end - end - end - - # If for some reason no instances managed to get fetched successfully then we'll just redirect to redirect.invidious.io - if filtered_instance_list.size == 0 - return "redirect.invidious.io" - end - - return filtered_instance_list.sample(1)[0] -end - def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = "…") : String str = uri.to_s.sub(/^https?:\/\//, "") if str.size > max_length diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr new file mode 100644 index 00000000..cb4280b9 --- /dev/null +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -0,0 +1,97 @@ +class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob + # We update the internals of a constant as so it can be accessed from anywhere + # within the codebase + # + # "INSTANCES" => Array(Tuple(String, String)) # region, instance + + INSTANCES = {"INSTANCES" => [] of Tuple(String, String)} + + def initialize + end + + def begin + loop do + refresh_instances + LOGGER.info("InstanceListRefreshJob: Done, sleeping for 30 minutes") + sleep 30.minute + Fiber.yield + end + end + + # Refreshes the list of instances used for redirects. + # + # Does the following three checks for each instance + # - Is it a clear-net instance? + # - Is it an instance with a good uptime? + # - Is it an updated instance? + private def refresh_instances + raw_instance_list = self.fetch_instances + filtered_instance_list = [] of Tuple(String, String) + + raw_instance_list.each do |instance_data| + # TODO allow Tor hidden service instances when the current instance + # is also a hidden service. Same for i2p and any other non-clearnet instances. + begin + domain = instance_data[0] + info = instance_data[1] + stats = info["stats"] + + next unless info["type"] == "https" + next if bad_uptime?(info["monitor"]) + next if outdated?(stats["software"]["version"]) + + filtered_instance_list << {info["region"].as_s, domain.as_s} + rescue ex + if domain + LOGGER.info("InstanceListRefreshJob: failed to parse information from '#{domain}' because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ") + else + LOGGER.info("InstanceListRefreshJob: failed to parse information from an instance because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ") + end + end + end + + if !filtered_instance_list.empty? + INSTANCES["INSTANCES"] = filtered_instance_list + end + end + + # Fetches information regarding instances from api.invidious.io or an otherwise configured URL + private def fetch_instances : Array(JSON::Any) + begin + # We directly call the stdlib HTTP::Client here as it allows us to negate the effects + # of the force_resolve config option. This is needed as api.invidious.io does not support ipv6 + # and as such the following request raises if we were to use force_resolve with the ipv6 value. + instance_api_client = HTTP::Client.new(URI.parse("https://api.invidious.io")) + + # Timeouts + instance_api_client.connect_timeout = 10.seconds + instance_api_client.dns_timeout = 10.seconds + + raw_instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a + instance_api_client.close + rescue ex : Socket::ConnectError | IO::TimeoutError | JSON::ParseException + raw_instance_list = [] of JSON::Any + end + + return raw_instance_list + end + + # Checks if the given target instance is outdated + private def outdated?(target_instance_version) : Bool + remote_commit_date = target_instance_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) + return false if !remote_commit_date + + remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) + local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) + + return (remote_commit_date - local_commit_date).abs.days > 30 + end + + # Checks if the uptime of the target instance is greater than 90% over a 30 day period + private def bad_uptime?(target_instance_health_monitor) : Bool + return true if !target_instance_health_monitor["down"].as_bool == false + return true if target_instance_health_monitor["uptime"].as_f < 90 + + return false + end +end diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr index d6bd9571..8b620d63 100644 --- a/src/invidious/routes/misc.cr +++ b/src/invidious/routes/misc.cr @@ -40,7 +40,16 @@ module Invidious::Routes::Misc def self.cross_instance_redirect(env) referer = get_referer(env) - instance_url = fetch_random_instance + + instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"] + if instance_list.empty? + instance_url = "redirect.invidious.io" + else + # Sample returns an array + # Instances are packaged as {region, domain} in the instance list + instance_url = instance_list.sample(1)[0][1] + end + env.redirect "https://#{instance_url}#{referer}" end end diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 05bc2714..39ca77c0 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -27,6 +27,10 @@ module Invidious::Routes::PreferencesRoute annotations_subscribed ||= "off" annotations_subscribed = annotations_subscribed == "on" + preload = env.params.body["preload"]?.try &.as(String) + preload ||= "off" + preload = preload == "on" + autoplay = env.params.body["autoplay"]?.try &.as(String) autoplay ||= "off" autoplay = autoplay == "on" @@ -144,6 +148,7 @@ module Invidious::Routes::PreferencesRoute preferences = Preferences.from_json({ annotations: annotations, annotations_subscribed: annotations_subscribed, + preload: preload, autoplay: autoplay, captions: captions, comments: comments, diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index b3059403..0a8525f3 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -4,6 +4,7 @@ struct Preferences property annotations : Bool = CONFIG.default_user_preferences.annotations property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed + property preload : Bool = CONFIG.default_user_preferences.preload property autoplay : Bool = CONFIG.default_user_preferences.autoplay property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 921132f0..ae09e736 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -27,12 +27,6 @@ struct Video @captions = [] of Invidious::Videos::Captions::Metadata @[DB::Field(ignore: true)] - property adaptive_fmts : Array(Hash(String, JSON::Any))? - - @[DB::Field(ignore: true)] - property fmt_stream : Array(Hash(String, JSON::Any))? - - @[DB::Field(ignore: true)] property description : String? module JSONConverter @@ -98,72 +92,24 @@ struct Video # Methods for parsing streaming data - def convert_url(fmt) - if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) } - sp = cfr["sp"] - url = URI.parse(cfr["url"]) - params = url.query_params - - LOGGER.debug("Videos: Decoding '#{cfr}'") - - unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"]) - params[sp] = unsig if unsig + def fmt_stream : Array(Hash(String, JSON::Any)) + if formats = info.dig?("streamingData", "formats") + return formats + .as_a.map(&.as_h) + .sort_by! { |f| f["width"]?.try &.as_i || 0 } else - url = URI.parse(fmt["url"].as_s) - params = url.query_params - end - - n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"]) - params["n"] = n if n - - if token = CONFIG.po_token - params["pot"] = token - end - - params["host"] = url.host.not_nil! - if region = self.info["region"]?.try &.as_s - params["region"] = region - end - - url.query_params = params - LOGGER.trace("Videos: new url is '#{url}'") - - return url.to_s - rescue ex - LOGGER.debug("Videos: Error when parsing video URL") - LOGGER.trace(ex.inspect_with_backtrace) - return "" - end - - def fmt_stream - return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream - - fmt_stream = info.dig?("streamingData", "formats") - .try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - - fmt_stream.each do |fmt| - fmt["url"] = JSON::Any.new(self.convert_url(fmt)) + return [] of Hash(String, JSON::Any) end - - fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } - @fmt_stream = fmt_stream - return @fmt_stream.as(Array(Hash(String, JSON::Any))) end - def adaptive_fmts - return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts - - fmt_stream = info.dig("streamingData", "adaptiveFormats") - .try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) - - fmt_stream.each do |fmt| - fmt["url"] = JSON::Any.new(self.convert_url(fmt)) + def adaptive_fmts : Array(Hash(String, JSON::Any)) + if formats = info.dig?("streamingData", "adaptiveFormats") + return formats + .as_a.map(&.as_h) + .sort_by! { |f| f["width"]?.try &.as_i || 0 } + else + return [] of Hash(String, JSON::Any) end - - fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } - @adaptive_fmts = fmt_stream - - return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) end def video_streams diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 811a0a03..fb8935d9 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -142,10 +142,21 @@ def extract_video_info(video_id : String) params.delete("reason") end - {"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f| + {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| params[f] = player_response[f] if player_response[f]? end + # Convert URLs, if those are present + if streaming_data = player_response["streamingData"]? + %w[formats adaptiveFormats].each do |key| + streaming_data.as_h[key]?.try &.as_a.each do |format| + format.as_h["url"] = JSON::Any.new(convert_url(format)) + end + end + + params["streamingData"] = streaming_data + end + # Data structure version, for cache control params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) @@ -454,3 +465,35 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any return params end + +private def convert_url(fmt) + if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) } + sp = cfr["sp"] + url = URI.parse(cfr["url"]) + params = url.query_params + + LOGGER.debug("convert_url: Decoding '#{cfr}'") + + unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"]) + params[sp] = unsig if unsig + else + url = URI.parse(fmt["url"].as_s) + params = url.query_params + end + + n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"]) + params["n"] = n if n + + if token = CONFIG.po_token + params["pot"] = token + end + + url.query_params = params + LOGGER.trace("convert_url: new url is '#{url}'") + + return url.to_s +rescue ex + LOGGER.debug("convert_url: Error when parsing video URL") + LOGGER.trace(ex.inspect_with_backtrace) + return "" +end diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr index 34cf7ff0..48177bd8 100644 --- a/src/invidious/videos/video_preferences.cr +++ b/src/invidious/videos/video_preferences.cr @@ -2,6 +2,7 @@ struct VideoPreferences include JSON::Serializable property annotations : Bool + property preload : Bool property autoplay : Bool property comments : Array(String) property continue : Bool @@ -28,6 +29,7 @@ end def process_video_params(query, preferences) annotations = query["iv_load_policy"]?.try &.to_i? + preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe } autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } comments = query["comments"]?.try &.split(",").map(&.downcase) continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } @@ -50,6 +52,7 @@ def process_video_params(query, preferences) if preferences # region ||= preferences.region annotations ||= preferences.annotations.to_unsafe + preload ||= preferences.preload.to_unsafe autoplay ||= preferences.autoplay.to_unsafe comments ||= preferences.comments continue ||= preferences.continue.to_unsafe @@ -70,6 +73,7 @@ def process_video_params(query, preferences) end annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe + preload ||= CONFIG.default_user_preferences.preload.to_unsafe autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe comments ||= CONFIG.default_user_preferences.comments continue ||= CONFIG.default_user_preferences.continue.to_unsafe @@ -89,6 +93,7 @@ def process_video_params(query, preferences) save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe annotations = annotations == 1 + preload = preload == 1 autoplay = autoplay == 1 continue = continue == 1 continue_autoplay = continue_autoplay == 1 @@ -128,6 +133,7 @@ def process_video_params(query, preferences) params = VideoPreferences.new({ annotations: annotations, + preload: preload, autoplay: autoplay, comments: comments, continue: continue, diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index c3c02df0..5c28358b 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -1,5 +1,6 @@ <video style="outline:none;width:100%;background-color:#000" playsinline poster="<%= thumbnail %>" id="player" class="on-video_player video-js player-style-<%= params.player_style %>" + preload="<% if params.preload %>auto<% else %>none<% end %>" <% if params.autoplay %>autoplay<% end %> <% if params.video_loop %>loop<% end %> <% if params.controls %>controls<% end %>> diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index b89c73ca..cf8b5593 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -13,6 +13,11 @@ </div> <div class="pure-control-group"> + <label for="preload"><%= translate(locale, "preferences_preload_label") %></label> + <input name="preload" id="preload" type="checkbox" <% if preferences.preload %>checked<% end %>> + </div> + + <div class="pure-control-group"> <label for="autoplay"><%= translate(locale, "preferences_autoplay_label") %></label> <input name="autoplay" id="autoplay" type="checkbox" <% if preferences.autoplay %>checked<% end %>> </div> |