diff options
Diffstat (limited to 'src')
39 files changed, 2242 insertions, 2988 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 0a10027b..2a4c373c 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,7 +23,7 @@ require "pg" require "sqlite3" require "xml" require "yaml" -require "zip" +require "compress/zip" require "protodec/utils" require "./invidious/helpers/*" require "./invidious/*" @@ -48,9 +48,9 @@ ARCHIVE_URL = URI.parse("https://archive.org") LOGIN_URL = URI.parse("https://accounts.google.com") PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") REDDIT_URL = URI.parse("https://www.reddit.com") -TEXTCAPTCHA_URL = URI.parse("http://textcaptcha.com") +TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com") YT_URL = URI.parse("https://www.youtube.com") -YT_IMG_URL = URI.parse("https://i.ytimg.com") +HOST_URL = make_host_url(CONFIG, Kemal.config) CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} @@ -84,23 +84,25 @@ LOCALES = { "es" => load_locale("es"), "eu" => load_locale("eu"), "fr" => load_locale("fr"), + "hu" => load_locale("hu-HU"), "is" => load_locale("is"), "it" => load_locale("it"), "ja" => load_locale("ja"), "nb-NO" => load_locale("nb-NO"), "nl" => load_locale("nl"), - "pt-BR" => load_locale("pt-BR"), "pl" => load_locale("pl"), + "pt-BR" => load_locale("pt-BR"), + "pt-PT" => load_locale("pt-PT"), "ro" => load_locale("ro"), "ru" => load_locale("ru"), + "sv" => load_locale("sv-SE"), "tr" => load_locale("tr"), "uk" => load_locale("uk"), "zh-CN" => load_locale("zh-CN"), "zh-TW" => load_locale("zh-TW"), } -YT_POOL = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05) -YT_IMG_POOL = QUICPool.new(YT_IMG_URL, capacity: CONFIG.pool_size, timeout: 0.05) +YT_POOL = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.1) config = CONFIG logger = Invidious::LogHandler.new @@ -194,15 +196,6 @@ if config.statistics_enabled end end -top_videos = [] of Video -if config.top_enabled - spawn do - pull_top_videos(config, PG_DB) do |videos| - top_videos = videos - end - end -end - popular_videos = [] of ChannelVideo spawn do pull_popular_videos(PG_DB) do |videos| @@ -210,10 +203,11 @@ spawn do end end -decrypt_function = [] of {SigProc, Int32} +DECRYPT_FUNCTION = [] of {SigProc, Int32} spawn do update_decrypt_function do |function| - decrypt_function = function + DECRYPT_FUNCTION.clear + function.each { |i| DECRYPT_FUNCTION << i } end end @@ -250,10 +244,20 @@ spawn do end before_all do |env| - host_url = make_host_url(config, Kemal.config) + begin + preferences = Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}") + rescue + preferences = Preferences.from_json("{}") + end + env.response.headers["X-XSS-Protection"] = "1; mode=block" env.response.headers["X-Content-Type-Options"] = "nosniff" - env.response.headers["Content-Security-Policy"] = "default-src blob: data: 'self' #{host_url} 'unsafe-inline' 'unsafe-eval'; media-src blob: 'self' #{host_url} https://*.googlevideo.com:443" + extra_media_csp = "" + if CONFIG.disabled?("local") || !preferences.local + extra_media_csp += " https://*.googlevideo.com:443" + end + # TODO: Remove style-src's 'unsafe-inline', requires to remove all inline styles (<style> [..] </style>, style=" [..] ") + env.response.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self'; manifest-src 'self'; media-src 'self' blob:#{extra_media_csp}" env.response.headers["Referrer-Policy"] = "same-origin" if (Kemal.config.ssl || config.https_only) && config.hsts @@ -271,12 +275,6 @@ before_all do |env| "/latest_version", }.any? { |r| env.request.resource.starts_with? r } - begin - preferences = Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}") - rescue - preferences = Preferences.from_json("{}") - end - if env.request.cookies.has_key? "SID" sid = env.request.cookies["SID"].value @@ -362,12 +360,6 @@ get "/" do |env| templated "empty" when "Popular" templated "popular" - when "Top" - if config.top_enabled - templated "top" - else - templated "empty" - end when "Trending" env.redirect "/feed/trending" when "Subscriptions" @@ -382,6 +374,8 @@ get "/" do |env| else templated "popular" end + else + templated "empty" end end @@ -516,16 +510,16 @@ get "/watch" do |env| comment_html ||= "" end - fmt_stream = video.fmt_stream(decrypt_function) - adaptive_fmts = video.adaptive_fmts(decrypt_function) + fmt_stream = video.fmt_stream + adaptive_fmts = video.adaptive_fmts if params.local - fmt_stream.each { |fmt| fmt["url"] = URI.parse(fmt["url"]).full_path } - adaptive_fmts.each { |fmt| fmt["url"] = URI.parse(fmt["url"]).full_path } + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) } + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) } end - video_streams = video.video_streams(adaptive_fmts) - audio_streams = video.audio_streams(adaptive_fmts) + video_streams = video.video_streams + audio_streams = video.audio_streams # Older videos may not have audio sources available. # We redirect here so they're not unplayable @@ -555,33 +549,23 @@ get "/watch" do |env| aspect_ratio = "16:9" - video.description_html = fill_links(video.description_html, "https", "www.youtube.com") - video.description_html = replace_links(video.description_html) - - host_url = make_host_url(config, Kemal.config) - - if video.player_response["streamingData"]?.try &.["hlsManifestUrl"]? - hlsvp = video.player_response["streamingData"]["hlsManifestUrl"].as_s - hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", host_url) - end - thumbnail = "/vi/#{video.id}/maxres.jpg" if params.raw if params.listen - url = audio_streams[0]["url"] + url = audio_streams[0]["url"].as_s audio_streams.each do |fmt| - if fmt["bitrate"] == params.quality.rchop("k") - url = fmt["url"] + if fmt["bitrate"].as_i == params.quality.rchop("k").to_i + url = fmt["url"].as_s end end else - url = fmt_stream[0]["url"] + url = fmt_stream[0]["url"].as_s fmt_stream.each do |fmt| - if fmt["label"].split(" - ")[0] == params.quality - url = fmt["url"] + if fmt["quality"].as_s == params.quality + url = fmt["url"].as_s end end end @@ -589,24 +573,6 @@ get "/watch" do |env| next env.redirect url end - rvs = [] of Hash(String, String) - video.info["rvs"]?.try &.split(",").each do |rv| - rvs << HTTP::Params.parse(rv).to_h - end - - rating = video.info["avg_rating"].to_f64 - if video.views > 0 - engagement = ((video.dislikes.to_f + video.likes.to_f)/video.views * 100) - else - engagement = 0 - end - - playability_status = video.player_response["playabilityStatus"]? - if playability_status && playability_status["status"] == "LIVE_STREAM_OFFLINE" && !video.premiere_timestamp - reason = playability_status["reason"]?.try &.as_s - end - reason ||= "" - templated "watch" end @@ -719,6 +685,7 @@ get "/embed/:id" do |env| end next env.redirect url + else nil # Continue end params = process_video_params(env.params.query, preferences) @@ -757,16 +724,16 @@ get "/embed/:id" do |env| notifications.delete(id) end - fmt_stream = video.fmt_stream(decrypt_function) - adaptive_fmts = video.adaptive_fmts(decrypt_function) + fmt_stream = video.fmt_stream + adaptive_fmts = video.adaptive_fmts if params.local - fmt_stream.each { |fmt| fmt["url"] = URI.parse(fmt["url"]).full_path } - adaptive_fmts.each { |fmt| fmt["url"] = URI.parse(fmt["url"]).full_path } + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) } + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) } end - video_streams = video.video_streams(adaptive_fmts) - audio_streams = video.audio_streams(adaptive_fmts) + video_streams = video.video_streams + audio_streams = video.audio_streams if audio_streams.empty? && !video.live_now if params.quality == "dash" @@ -793,25 +760,13 @@ get "/embed/:id" do |env| aspect_ratio = nil - video.description_html = fill_links(video.description_html, "https", "www.youtube.com") - video.description_html = replace_links(video.description_html) - - host_url = make_host_url(config, Kemal.config) - - if video.player_response["streamingData"]?.try &.["hlsManifestUrl"]? - hlsvp = video.player_response["streamingData"]["hlsManifestUrl"].as_s - hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", host_url) - end - thumbnail = "/vi/#{video.id}/maxres.jpg" if params.raw - url = fmt_stream[0]["url"] + url = fmt_stream[0]["url"].as_s fmt_stream.each do |fmt| - if fmt["label"].split(" - ")[0] == params.quality - url = fmt["url"] - end + url = fmt["url"].as_s if fmt["quality"].as_s == params.quality end next env.redirect url @@ -838,8 +793,14 @@ get "/view_all_playlists" do |env| user = user.as(User) - items = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 ORDER BY created", user.email, as: InvidiousPlaylist) - items.map! do |item| + items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_created.map! do |item| + item.author = "" + item + end + + items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_saved.map! do |item| item.author = "" item end @@ -910,6 +871,25 @@ post "/create_playlist" do |env| env.redirect "/playlist?list=#{playlist.id}" end +get "/subscribe_playlist" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + referer = get_referer(env) + + if !user + next env.redirect "/" + end + + user = user.as(User) + + playlist_id = env.params.query["list"] + playlist = get_playlist(PG_DB, playlist_id, locale) + subscribe_playlist(PG_DB, user, playlist) + + env.redirect "/playlist?list=#{playlist.id}" +end + get "/delete_playlist" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -925,10 +905,6 @@ get "/delete_playlist" do |env| sid = sid.as(String) plid = env.params.query["list"]? - if !plid || !plid.starts_with?("IV") - next env.redirect referer - end - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if !playlist || playlist.author != user.email next env.redirect referer @@ -1227,17 +1203,17 @@ post "/playlist_ajax" do |env| end end - playlist_video = PlaylistVideo.new( - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, length_seconds: video.length_seconds, - published: video.published, - plid: playlist_id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX) - ) + published: video.published, + plid: playlist_id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) video_array = playlist_video.to_a args = arg_array(video_array) @@ -1250,6 +1226,10 @@ post "/playlist_ajax" do |env| PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index), updated = $2 WHERE id = $3", index, Time.utc, playlist_id) when "action_move_video_before" # TODO: Playlist stub + else + error_message = {"error" => "Unsupported action #{action}"}.to_json + env.response.status_code = 400 + next error_message end if redirect @@ -1333,16 +1313,14 @@ get "/opensearch.xml" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? env.response.content_type = "application/opensearchdescription+xml" - host = make_host_url(config, Kemal.config) - XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("OpenSearchDescription", xmlns: "http://a9.com/-/spec/opensearch/1.1/") do xml.element("ShortName") { xml.text "Invidious" } xml.element("LongName") { xml.text "Invidious Search" } xml.element("Description") { xml.text "Search for videos, channels, and playlists on Invidious" } xml.element("InputEncoding") { xml.text "UTF-8" } - xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "#{host}/favicon.ico" } - xml.element("Url", type: "text/html", method: "get", template: "#{host}/search?q={searchTerms}") + xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "#{HOST_URL}/favicon.ico" } + xml.element("Url", type: "text/html", method: "get", template: "#{HOST_URL}/search?q={searchTerms}") end end end @@ -1451,7 +1429,6 @@ post "/login" do |env| traceback = IO::Memory.new # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82 - # TODO: Convert to QUIC begin client = QUIC::Client.new(LOGIN_URL) headers = HTTP::Headers.new @@ -1544,7 +1521,7 @@ post "/login" do |env| case prompt_type when "TWO_STEP_VERIFICATION" prompt_type = 2 - when "LOGIN_CHALLENGE" + else # "LOGIN_CHALLENGE" prompt_type = 4 end @@ -1837,7 +1814,7 @@ post "/login" do |env| env.response.status_code = 400 next templated "error" end - when "text" + else # "text" answer = Digest::MD5.hexdigest(answer.downcase.strip) found_valid_captcha = false @@ -1862,8 +1839,8 @@ post "/login" do |env| sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) user, sid = create_user(sid, email, password) user_array = user.to_a + user_array[4] = user_array[4].to_json # User preferences - user_array[4] = user_array[4].to_json args = arg_array(user_array) PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) @@ -2090,10 +2067,6 @@ post "/preferences" do |env| end config.default_user_preferences.feed_menu = admin_feed_menu - top_enabled = env.params.body["top_enabled"]?.try &.as(String) - top_enabled ||= "off" - config.top_enabled = top_enabled == "on" - captcha_enabled = env.params.body["captcha_enabled"]?.try &.as(String) captcha_enabled ||= "off" config.captcha_enabled = captcha_enabled == "on" @@ -2248,6 +2221,10 @@ post "/watch_ajax" do |env| end when "action_mark_unwatched" PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE email = $2", id, user.email) + else + error_message = {"error" => "Unsupported action #{action}"}.to_json + env.response.status_code = 400 + next error_message end if redirect @@ -2311,8 +2288,7 @@ get "/modify_notifications" do |env| end headers = cookies.add_request_headers(headers) - match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/) - if match + if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/) session_token = match["session_token"] else next env.redirect referer @@ -2402,6 +2378,10 @@ post "/subscription_ajax" do |env| end when "action_remove_subscriptions" PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email) + else + error_message = {"error" => "Unsupported action #{action}"}.to_json + env.response.status_code = 400 + next error_message end if redirect @@ -2447,20 +2427,39 @@ get "/subscription_manager" do |env| end subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) - subscriptions.sort_by! { |channel| channel.author.downcase } if action_takeout - host_url = make_host_url(config, Kemal.config) - if format == "json" env.response.content_type = "application/json" env.response.headers["content-disposition"] = "attachment" - next { - "subscriptions" => user.subscriptions, - "watch_history" => user.watched, - "preferences" => user.preferences, - }.to_json + playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + + next JSON.build do |json| + json.object do + json.field "subscriptions", user.subscriptions + json.field "watch_history", user.watched + json.field "preferences", user.preferences + json.field "playlists" do + json.array do + playlists.each do |playlist| + json.object do + json.field "title", playlist.title + json.field "description", html_to_content(playlist.description_html) + json.field "privacy", playlist.privacy.to_s + json.field "videos" do + json.array do + PG_DB.query_all("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 500", playlist.id, playlist.index, as: String).each do |video_id| + json.string video_id + end + end + end + end + end + end + end + end + end else env.response.content_type = "application/xml" env.response.headers["content-disposition"] = "attachment" @@ -2478,7 +2477,7 @@ get "/subscription_manager" do |env| if format == "newpipe" xmlUrl = "https://www.youtube.com/feeds/videos.xml?channel_id=#{channel.id}" else - xmlUrl = "#{host_url}/feed/channel/#{channel.id}" + xmlUrl = "#{HOST_URL}/feed/channel/#{channel.id}" end xml.element("outline", text: channel.author, title: channel.author, @@ -2520,42 +2519,13 @@ post "/data_control" do |env| if user user = user.as(User) - spawn do - # Since import can take a while, if we're not done after 20 seconds - # push out content to prevent timeout. - - # Interesting to note is that Chrome will try to render before the content has finished loading, - # which is why we include a loading icon. Firefox and its derivatives will not see this page, - # instead redirecting immediately once the connection has closed. - - # https://stackoverflow.com/q/2091239 is helpful but not directly applicable here. - - sleep 20.seconds - env.response.puts %(<meta http-equiv="refresh" content="0; url=#{referer}">) - env.response.puts %(<link rel="stylesheet" href="/css/ionicons.min.css?v=#{ASSET_COMMIT}">) - env.response.puts %(<link rel="stylesheet" href="/css/default.css?v=#{ASSET_COMMIT}">) - if env.get("preferences").as(Preferences).dark_mode == "dark" - env.response.puts %(<link rel="stylesheet" href="/css/darktheme.css?v=#{ASSET_COMMIT}">) - else - env.response.puts %(<link rel="stylesheet" href="/css/lighttheme.css?v=#{ASSET_COMMIT}">) - end - env.response.puts %(<h3><div class="loading"><i class="icon ion-ios-refresh"></i></div></h3>) - env.response.flush - - loop do - env.response.puts %(<!-- keepalive #{Time.utc.to_unix} -->) - env.response.flush - - sleep (20 + rand(11)).seconds - end - end + # TODO: Find a way to prevent browser timeout HTTP::FormData.parse(env.request) do |part| body = part.body.gets_to_end - if body.empty? - next - end + next if body.empty? + # TODO: Unify into single import based on content-type case part.name when "import_invidious" body = JSON.parse(body) @@ -2576,9 +2546,55 @@ post "/data_control" do |env| end if body["preferences"]? - user.preferences = Preferences.from_json(body["preferences"].to_json, user.preferences) + user.preferences = Preferences.from_json(body["preferences"].to_json) PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email) end + + if playlists = body["playlists"]?.try &.as_a? + playlists.each do |item| + title = item["title"]?.try &.as_s?.try &.delete("<>") + description = item["description"]?.try &.as_s?.try &.delete("\r") + privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } + + next if !title + next if !description + next if !privacy + + playlist = create_playlist(PG_DB, title, privacy, user) + PG_DB.exec("UPDATE playlists SET description = $1 WHERE id = $2", description, playlist.id) + + videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| + raise "Playlist cannot have more than 500 videos" if idx > 500 + + video_id = video_id.try &.as_s? + next if !video_id + + begin + video = get_video(video_id, PG_DB) + rescue ex + next + end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: playlist.id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + video_array = playlist_video.to_a + args = arg_array(video_array) + + PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) + PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index), updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist.id) + end + end + end when "import_youtube" subscriptions = XML.parse(body) user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| @@ -2618,7 +2634,7 @@ post "/data_control" do |env| PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_newpipe" - Zip::Reader.open(IO::Memory.new(body)) do |file| + Compress::Zip::Reader.open(IO::Memory.new(body)) do |file| file.each_entry do |entry| if entry.filename == "newpipe.db" tempfile = File.tempfile(".db") @@ -2642,6 +2658,7 @@ post "/data_control" do |env| end end end + else nil # Ignore end end end @@ -2983,6 +3000,10 @@ post "/token_ajax" do |env| case action when .starts_with? "action_revoke_token" PG_DB.exec("DELETE FROM session_ids * WHERE id = $1 AND email = $2", session, user.email) + else + error_message = {"error" => "Unsupported action #{action}"}.to_json + env.response.status_code = 400 + next error_message end if redirect @@ -2997,12 +3018,7 @@ end get "/feed/top" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - if config.top_enabled - templated "top" - else - env.redirect "/" - end + env.redirect "/" end get "/feed/popular" do |env| @@ -3125,12 +3141,10 @@ get "/feed/channel/:ucid" do |env| next error_message end - rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}").body - rss = XML.parse_html(rss) + response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}") + rss = XML.parse_html(response.body) - videos = [] of SearchVideo - - rss.xpath_nodes("//feed/entry").each do |entry| + videos = rss.xpath_nodes("//feed/entry").map do |entry| video_id = entry.xpath_node("videoid").not_nil!.content title = entry.xpath_node("title").not_nil!.content @@ -3142,41 +3156,39 @@ get "/feed/channel/:ucid" do |env| description_html = entry.xpath_node("group/description").not_nil!.to_s views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64 - videos << SearchVideo.new( - title: title, - id: video_id, - author: author, - ucid: ucid, - published: published, - views: views, - description_html: description_html, - length_seconds: 0, - live_now: false, - paid: false, - premium: false, - premiere_timestamp: nil - ) + SearchVideo.new({ + title: title, + id: video_id, + author: author, + ucid: ucid, + published: published, + views: views, + description_html: description_html, + length_seconds: 0, + live_now: false, + paid: false, + premium: false, + premiere_timestamp: nil, + }) end - host_url = make_host_url(config, Kemal.config) - XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", "xml:lang": "en-US") do - xml.element("link", rel: "self", href: "#{host_url}#{env.request.resource}") + xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") xml.element("id") { xml.text "yt:channel:#{channel.ucid}" } xml.element("yt:channelId") { xml.text channel.ucid } xml.element("title") { xml.text channel.author } - xml.element("link", rel: "alternate", href: "#{host_url}/channel/#{channel.ucid}") + xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}") xml.element("author") do xml.element("name") { xml.text channel.author } - xml.element("uri") { xml.text "#{host_url}/channel/#{channel.ucid}" } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" } end videos.each do |video| - video.to_xml(host_url, channel.auto_generated, params, xml) + video.to_xml(channel.auto_generated, params, xml) end end end @@ -3210,19 +3222,18 @@ get "/feed/private" do |env| params = HTTP::Params.parse(env.params.query["params"]? || "") videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) - host_url = make_host_url(config, Kemal.config) XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", "xml:lang": "en-US") do - xml.element("link", "type": "text/html", rel: "alternate", href: "#{host_url}/feed/subscriptions") + xml.element("link", "type": "text/html", rel: "alternate", href: "#{HOST_URL}/feed/subscriptions") xml.element("link", "type": "application/atom+xml", rel: "self", - href: "#{host_url}#{env.request.resource}") + href: "#{HOST_URL}#{env.request.resource}") xml.element("title") { xml.text translate(locale, "Invidious Private Feed for `x`", user.email) } (notifications + videos).each do |video| - video.to_xml(locale, host_url, params, xml) + video.to_xml(locale, params, xml) end end end @@ -3236,8 +3247,6 @@ get "/feed/playlist/:plid" do |env| plid = env.params.url["plid"] params = HTTP::Params.parse(env.params.query["params"]? || "") - - host_url = make_host_url(config, Kemal.config) path = env.request.path if plid.starts_with? "IV" @@ -3248,18 +3257,18 @@ get "/feed/playlist/:plid" do |env| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", "xml:lang": "en-US") do - xml.element("link", rel: "self", href: "#{host_url}#{env.request.resource}") + xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") xml.element("id") { xml.text "iv:playlist:#{plid}" } xml.element("iv:playlistId") { xml.text plid } xml.element("title") { xml.text playlist.title } - xml.element("link", rel: "alternate", href: "#{host_url}/playlist?list=#{plid}") + xml.element("link", rel: "alternate", href: "#{HOST_URL}/playlist?list=#{plid}") xml.element("author") do xml.element("name") { xml.text playlist.author } end videos.each do |video| - video.to_xml(host_url, false, xml) + video.to_xml(false, xml) end end end @@ -3278,7 +3287,8 @@ get "/feed/playlist/:plid" do |env| when "url", "href" full_path = URI.parse(node[attribute.name]).full_path query_string_opt = full_path.starts_with?("/watch?v=") ? "&#{params}" : "" - node[attribute.name] = "#{host_url}#{full_path}#{query_string_opt}" + node[attribute.name] = "#{HOST_URL}#{full_path}#{query_string_opt}" + else nil # Skip end end end @@ -3286,7 +3296,7 @@ get "/feed/playlist/:plid" do |env| document = document.to_xml(options: XML::SaveOptions::NO_DECL) document.scan(/<uri>(?<url>[^<]+)<\/uri>/).each do |match| - content = "#{host_url}#{URI.parse(match["url"]).full_path}" + content = "#{HOST_URL}#{URI.parse(match["url"]).full_path}" document = document.gsub(match[0], "<uri>#{content}</uri>") end @@ -3389,18 +3399,18 @@ post "/feed/webhook/:token" do |env| }.to_json PG_DB.exec("NOTIFY notifications, E'#{payload}'") - video = ChannelVideo.new( - id: id, - title: video.title, - published: published, - updated: updated, - ucid: video.ucid, - author: author, - length_seconds: video.length_seconds, - live_now: video.live_now, + video = ChannelVideo.new({ + id: id, + title: video.title, + published: published, + updated: updated, + ucid: video.ucid, + author: author, + length_seconds: video.length_seconds, + live_now: video.live_now, premiere_timestamp: video.premiere_timestamp, - views: video.views, - ) + views: video.views, + }) PG_DB.query_all("UPDATE users SET feed_needs_update = true, notifications = array_append(notifications, $1) \ WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", @@ -3465,14 +3475,12 @@ get "/c/:user" do |env| user = env.params.url["user"] response = YT_POOL.client &.get("/c/#{user}") - document = XML.parse_html(response.body) + html = XML.parse_html(response.body) - anchor = document.xpath_node(%q(//a[contains(@class,"branded-page-header-title-link")])) - if !anchor - next env.redirect "/" - end + ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] + next env.redirect "/" if !ucid - env.redirect anchor["href"] + env.redirect "/channel/#{ucid}" end # Legacy endpoint for /user/:username @@ -3562,14 +3570,14 @@ get "/channel/:ucid" do |env| item.author end end - items = items.select { |item| item.is_a?(SearchPlaylist) }.map { |item| item.as(SearchPlaylist) } + items = items.select(&.is_a?(SearchPlaylist)).map(&.as(SearchPlaylist)) items.each { |item| item.author = "" } else sort_options = {"newest", "oldest", "popular"} sort_by ||= "newest" - items, count = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) - items.select! { |item| !item.paid } + count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + items.reject! &.paid env.set "search", "channel:#{channel.ucid} " end @@ -3664,7 +3672,7 @@ get "/channel/:ucid/community" do |env| end begin - items = JSON.parse(fetch_channel_community(ucid, continuation, locale, config, Kemal.config, "json", thin_mode)) + items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) rescue ex env.response.status_code = 500 error_message = ex.message @@ -3717,7 +3725,6 @@ get "/api/v1/storyboards/:id" do |env| end storyboards = video.storyboards - width = env.params.query["width"]? height = env.params.query["height"]? @@ -3725,7 +3732,7 @@ get "/api/v1/storyboards/:id" do |env| response = JSON.build do |json| json.object do json.field "storyboards" do - generate_storyboards(json, id, storyboards, config, Kemal.config) + generate_storyboards(json, id, storyboards) end end end @@ -3755,8 +3762,7 @@ get "/api/v1/storyboards/:id" do |env| end_time = storyboard[:interval].milliseconds storyboard[:storyboard_count].times do |i| - host_url = make_host_url(config, Kemal.config) - url = storyboard[:url].gsub("$M", i).gsub("https://i9.ytimg.com", host_url) + url = storyboard[:url].gsub("$M", i).gsub("https://i9.ytimg.com", HOST_URL) storyboard[:storyboard_height].times do |j| storyboard[:storyboard_width].times do |k| @@ -4038,7 +4044,7 @@ get "/api/v1/annotations/:id" do |env| cache_annotation(PG_DB, id, annotations) end - when "youtube" + else # "youtube" response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}") if response.status_code != 200 @@ -4079,7 +4085,7 @@ get "/api/v1/videos/:id" do |env| next error_message end - video.to_json(locale, config, Kemal.config, decrypt_function) + video.to_json(locale) end get "/api/v1/trending" do |env| @@ -4101,7 +4107,7 @@ get "/api/v1/trending" do |env| videos = JSON.build do |json| json.array do trending.each do |video| - video.to_json(locale, config, Kemal.config, json) + video.to_json(locale, json) end end end @@ -4117,7 +4123,7 @@ get "/api/v1/popular" do |env| JSON.build do |json| json.array do popular_videos.each do |video| - video.to_json(locale, config, Kemal.config, json) + video.to_json(locale, json) end end end @@ -4127,41 +4133,7 @@ get "/api/v1/top" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? env.response.content_type = "application/json" - - if !config.top_enabled - error_message = {"error" => "Administrator has disabled this endpoint."}.to_json - env.response.status_code = 400 - next error_message - end - - JSON.build do |json| - json.array do - top_videos.each do |video| - # Top videos have much more information than provided below (adaptiveFormats, etc) - # but can be very out of date, so we only provide a subset here - - json.object do - json.field "title", video.title - json.field "videoId", video.id - json.field "videoThumbnails" do - generate_thumbnails(json, video.id, config, Kemal.config) - end - - json.field "lengthSeconds", video.length_seconds - json.field "viewCount", video.views - - json.field "author", video.author - json.field "authorId", video.ucid - json.field "authorUrl", "/channel/#{video.ucid}" - json.field "published", video.published.to_unix - json.field "publishedText", translate(locale, "`x` ago", recode_date(video.published, locale)) - - json.field "description", html_to_content(video.description_html) - json.field "descriptionHtml", video.description_html - end - end - end - end + "[]" end get "/api/v1/channels/:ucid" do |env| @@ -4192,7 +4164,7 @@ get "/api/v1/channels/:ucid" do |env| count = 0 else begin - videos, count = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) rescue ex error_message = {"error" => ex.message}.to_json env.response.status_code = 500 @@ -4238,7 +4210,7 @@ get "/api/v1/channels/:ucid" do |env| qualities.each do |quality| json.object do - json.field "url", channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "url", channel.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") json.field "width", quality json.field "height", quality end @@ -4261,7 +4233,7 @@ get "/api/v1/channels/:ucid" do |env| json.field "latestVideos" do json.array do videos.each do |video| - video.to_json(locale, config, Kemal.config, json) + video.to_json(locale, json) end end end @@ -4322,7 +4294,7 @@ end end begin - videos, count = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) + count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) rescue ex error_message = {"error" => ex.message}.to_json env.response.status_code = 500 @@ -4332,7 +4304,7 @@ end JSON.build do |json| json.array do videos.each do |video| - video.to_json(locale, config, Kemal.config, json) + video.to_json(locale, json) end end end @@ -4358,7 +4330,7 @@ end JSON.build do |json| json.array do videos.each do |video| - video.to_json(locale, config, Kemal.config, json) + video.to_json(locale, json) end end end @@ -4373,9 +4345,9 @@ end ucid = env.params.url["ucid"] continuation = env.params.query["continuation"]? - sort_by = env.params.query["sort"]?.try &.downcase - sort_by ||= env.params.query["sort_by"]?.try &.downcase - sort_by ||= "last" + sort_by = env.params.query["sort"]?.try &.downcase || + env.params.query["sort_by"]?.try &.downcase || + "last" begin channel = get_about_info(ucid, locale) @@ -4397,9 +4369,7 @@ end json.field "playlists" do json.array do items.each do |item| - if item.is_a?(SearchPlaylist) - item.to_json(locale, config, Kemal.config, json) - end + item.to_json(locale, json) if item.is_a?(SearchPlaylist) end end end @@ -4428,7 +4398,7 @@ end # sort_by = env.params.query["sort_by"]?.try &.downcase begin - fetch_channel_community(ucid, continuation, locale, config, Kemal.config, format, thin_mode) + fetch_channel_community(ucid, continuation, locale, format, thin_mode) rescue ex env.response.status_code = 400 error_message = {"error" => ex.message}.to_json @@ -4454,7 +4424,7 @@ get "/api/v1/channels/search/:ucid" do |env| JSON.build do |json| json.array do search_results.each do |item| - item.to_json(locale, config, Kemal.config, json) + item.to_json(locale, json) end end end @@ -4499,7 +4469,7 @@ get "/api/v1/search" do |env| JSON.build do |json| json.array do search_results.each do |item| - item.to_json(locale, config, Kemal.config, json) + item.to_json(locale, json) end end end @@ -4515,10 +4485,8 @@ get "/api/v1/search/suggestions" do |env| query ||= "" begin - client = QUIC::Client.new("suggestqueries.google.com") - client.family = CONFIG.force_resolve || Socket::Family::INET - client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC - response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback").body + headers = HTTP::Headers{":authority" => "suggestqueries.google.com"} + response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body body = response[35..-2] body = JSON.parse(body).as_a @@ -4578,7 +4546,7 @@ end next error_message end - response = playlist.to_json(offset, locale, config, Kemal.config, continuation: continuation) + response = playlist.to_json(offset, locale, continuation: continuation) if format == "html" response = JSON.parse(response) @@ -4642,7 +4610,7 @@ get "/api/v1/mixes/:rdid" do |env| json.field "videoThumbnails" do json.array do - generate_thumbnails(json, video.id, config, Kemal.config) + generate_thumbnails(json, video.id) end end @@ -4677,7 +4645,7 @@ get "/api/v1/auth/notifications" do |env| topics = env.params.query["topics"]?.try &.split(",").uniq.first(1000) topics ||= [] of String - create_notification_stream(env, config, Kemal.config, decrypt_function, topics, connection_channel) + create_notification_stream(env, topics, connection_channel) end post "/api/v1/auth/notifications" do |env| @@ -4686,7 +4654,7 @@ post "/api/v1/auth/notifications" do |env| topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) topics ||= [] of String - create_notification_stream(env, config, Kemal.config, decrypt_function, topics, connection_channel) + create_notification_stream(env, topics, connection_channel) end get "/api/v1/auth/preferences" do |env| @@ -4700,7 +4668,7 @@ post "/api/v1/auth/preferences" do |env| user = env.get("user").as(User) begin - preferences = Preferences.from_json(env.request.body || "{}", user.preferences) + preferences = Preferences.from_json(env.request.body || "{}") rescue preferences = user.preferences end @@ -4730,7 +4698,7 @@ get "/api/v1/auth/feed" do |env| json.field "notifications" do json.array do notifications.each do |video| - video.to_json(locale, config, Kemal.config, json) + video.to_json(locale, json) end end end @@ -4738,7 +4706,7 @@ get "/api/v1/auth/feed" do |env| json.field "videos" do json.array do videos.each do |video| - video.to_json(locale, config, Kemal.config, json) + video.to_json(locale, json) end end end @@ -4810,7 +4778,7 @@ get "/api/v1/auth/playlists" do |env| JSON.build do |json| json.array do playlists.each do |playlist| - playlist.to_json(0, locale, config, Kemal.config, json) + playlist.to_json(0, locale, json) end end end @@ -4841,10 +4809,8 @@ post "/api/v1/auth/playlists" do |env| next error_message end - host_url = make_host_url(config, Kemal.config) - playlist = create_playlist(PG_DB, title, privacy, user) - env.response.headers["Location"] = "#{host_url}/api/v1/auth/playlists/#{playlist.id}" + env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}" env.response.status_code = 201 { "title" => title, @@ -4956,17 +4922,17 @@ post "/api/v1/auth/playlists/:plid/videos" do |env| next error_message end - playlist_video = PlaylistVideo.new( - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, length_seconds: video.length_seconds, - published: video.published, - plid: plid, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX) - ) + published: video.published, + plid: plid, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) video_array = playlist_video.to_a args = arg_array(video_array) @@ -4974,11 +4940,9 @@ post "/api/v1/auth/playlists/:plid/videos" do |env| PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = video_count + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, plid) - host_url = make_host_url(config, Kemal.config) - - env.response.headers["Location"] = "#{host_url}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}" + env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{plid}/videos/#{playlist_video.index.to_u64.to_s(16).upcase}" env.response.status_code = 201 - playlist_video.to_json(locale, config, Kemal.config, index: playlist.index.size) + playlist_video.to_json(locale, index: playlist.index.size) end delete "/api/v1/auth/playlists/:plid/videos/:index" do |env| @@ -5156,7 +5120,7 @@ get "/api/manifest/dash/id/:id" do |env| next end - if dashmpd = video.player_response["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s + if dashmpd = video.dash_manifest_url manifest = YT_POOL.client &.get(URI.parse(dashmpd).full_path).body manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl| @@ -5173,16 +5137,16 @@ get "/api/manifest/dash/id/:id" do |env| next manifest end - adaptive_fmts = video.adaptive_fmts(decrypt_function) + adaptive_fmts = video.adaptive_fmts if local adaptive_fmts.each do |fmt| - fmt["url"] = URI.parse(fmt["url"]).full_path + fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).full_path) end end - audio_streams = video.audio_streams(adaptive_fmts) - video_streams = video.video_streams(adaptive_fmts).sort_by { |stream| {stream["size"].split("x")[0].to_i, stream["fps"].to_i} }.reverse + audio_streams = video.audio_streams + video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011", @@ -5192,24 +5156,22 @@ get "/api/manifest/dash/id/:id" do |env| i = 0 {"audio/mp4", "audio/webm"}.each do |mime_type| - mime_streams = audio_streams.select { |stream| stream["type"].starts_with? mime_type } - if mime_streams.empty? - next - end + mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type } + next if mime_streams.empty? xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do mime_streams.each do |fmt| - codecs = fmt["type"].split("codecs=")[1].strip('"') - bandwidth = fmt["bitrate"].to_i * 1000 - itag = fmt["itag"] - url = fmt["url"] + codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') + bandwidth = fmt["bitrate"].as_i + itag = fmt["itag"].as_i + url = fmt["url"].as_s xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011", value: "2") xml.element("BaseURL") { xml.text url } - xml.element("SegmentBase", indexRange: fmt["index"]) do - xml.element("Initialization", range: fmt["init"]) + xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do + xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}") end end end @@ -5218,21 +5180,24 @@ get "/api/manifest/dash/id/:id" do |env| i += 1 end + potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144} + {"video/mp4", "video/webm"}.each do |mime_type| - mime_streams = video_streams.select { |stream| stream["type"].starts_with? mime_type } + mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type } next if mime_streams.empty? heights = [] of Int32 xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do mime_streams.each do |fmt| - codecs = fmt["type"].split("codecs=")[1].strip('"') - bandwidth = fmt["bitrate"] - itag = fmt["itag"] - url = fmt["url"] - width, height = fmt["size"].split("x").map { |i| i.to_i } + codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"') + bandwidth = fmt["bitrate"].as_i + itag = fmt["itag"].as_i + url = fmt["url"].as_s + width = fmt["width"].as_i + height = fmt["height"].as_i # Resolutions reported by YouTube player (may not accurately reflect source) - height = [4320, 2160, 1440, 1080, 720, 480, 360, 240, 144].sort_by { |i| (height - i).abs }[0] + height = potential_heights.min_by { |i| (height - i).abs } next if unique_res && heights.includes? height heights << height @@ -5240,8 +5205,8 @@ get "/api/manifest/dash/id/:id" do |env| startWithSAP: "1", maxPlayoutRate: "1", bandwidth: bandwidth, frameRate: fmt["fps"]) do xml.element("BaseURL") { xml.text url } - xml.element("SegmentBase", indexRange: fmt["index"]) do - xml.element("Initialization", range: fmt["init"]) + xml.element("SegmentBase", indexRange: "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}") do + xml.element("Initialization", range: "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}") end end end @@ -5255,10 +5220,10 @@ get "/api/manifest/dash/id/:id" do |env| end get "/api/manifest/hls_variant/*" do |env| - manifest = YT_POOL.client &.get(env.request.path) + response = YT_POOL.client &.get(env.request.path) - if manifest.status_code != 200 - env.response.status_code = manifest.status_code + if response.status_code != 200 + env.response.status_code = response.status_code next end @@ -5267,12 +5232,10 @@ get "/api/manifest/hls_variant/*" do |env| env.response.content_type = "application/x-mpegURL" env.response.headers.add("Access-Control-Allow-Origin", "*") - host_url = make_host_url(config, Kemal.config) - - manifest = manifest.body + manifest = response.body if local - manifest = manifest.gsub("https://www.youtube.com", host_url) + manifest = manifest.gsub("https://www.youtube.com", HOST_URL) manifest = manifest.gsub("index.m3u8", "index.m3u8?local=true") end @@ -5280,10 +5243,10 @@ get "/api/manifest/hls_variant/*" do |env| end get "/api/manifest/hls_playlist/*" do |env| - manifest = YT_POOL.client &.get(env.request.path) + response = YT_POOL.client &.get(env.request.path) - if manifest.status_code != 200 - env.response.status_code = manifest.status_code + if response.status_code != 200 + env.response.status_code = response.status_code next end @@ -5292,9 +5255,7 @@ get "/api/manifest/hls_playlist/*" do |env| env.response.content_type = "application/x-mpegURL" env.response.headers.add("Access-Control-Allow-Origin", "*") - host_url = make_host_url(config, Kemal.config) - - manifest = manifest.body + manifest = response.body if local manifest = manifest.gsub(/^https:\/\/r\d---.{11}\.c\.youtube\.com[^\n]*/m) do |match| @@ -5329,7 +5290,7 @@ get "/api/manifest/hls_playlist/*" do |env| raw_params["local"] = "true" - "#{host_url}/videoplayback?#{raw_params}" + "#{HOST_URL}/videoplayback?#{raw_params}" end end @@ -5355,7 +5316,7 @@ get "/latest_version" do |env| end id ||= env.params.query["id"]? - itag ||= env.params.query["itag"]? + itag ||= env.params.query["itag"]?.try &.to_i region = env.params.query["region"]? @@ -5370,26 +5331,16 @@ get "/latest_version" do |env| video = get_video(id, PG_DB, region: region) - fmt_stream = video.fmt_stream(decrypt_function) - adaptive_fmts = video.adaptive_fmts(decrypt_function) + fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } + url = fmt.try &.["url"]?.try &.as_s - urls = (fmt_stream + adaptive_fmts).select { |fmt| fmt["itag"] == itag } - if urls.empty? + if !url env.response.status_code = 404 next - elsif urls.size > 1 - env.response.status_code = 409 - next end - url = urls[0]["url"] - if local - url = URI.parse(url).full_path.not_nil! - end - - if title - url += "&title=#{title}" - end + url = URI.parse(url).full_path.not_nil! if local + url = "#{url}&title=#{title}" if title env.redirect url end @@ -5482,8 +5433,8 @@ get "/videoplayback" do |env| end client = make_client(URI.parse(host), region) - response = HTTP::Client::Response.new(500) + error = "" 5.times do begin response = client.head(url, headers) @@ -5508,12 +5459,14 @@ get "/videoplayback" do |env| host = "https://r#{fvip}---#{mn}.googlevideo.com" client = make_client(URI.parse(host), region) rescue ex + error = ex.message end end if response.status_code >= 400 env.response.status_code = response.status_code - next + env.response.content_type = "text/plain" + next error end if url.includes? "&file=seg.ts" @@ -5644,11 +5597,9 @@ get "/videoplayback" do |env| end get "/ggpht/*" do |env| - host = "https://yt3.ggpht.com" - client = make_client(URI.parse(host)) url = env.request.path.lchop("/ggpht") - headers = HTTP::Headers.new + headers = HTTP::Headers{":authority" => "yt3.ggpht.com"} REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? headers[header] = env.request.headers[header] @@ -5656,7 +5607,7 @@ get "/ggpht/*" do |env| end begin - client.get(url, headers) do |response| + YT_POOL.client &.get(url, headers) do |response| env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -5689,16 +5640,16 @@ get "/sb/:id/:storyboard/:index" do |env| storyboard = env.params.url["storyboard"] index = env.params.url["index"] + url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}" + + headers = HTTP::Headers.new + if storyboard.starts_with? "storyboard_live" - host = "https://i.ytimg.com" + headers[":authority"] = "i.ytimg.com" else - host = "https://i9.ytimg.com" + headers[":authority"] = "i9.ytimg.com" end - client = make_client(URI.parse(host)) - - url = "/sb/#{id}/#{storyboard}/#{index}?#{env.params.query}" - headers = HTTP::Headers.new REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? headers[header] = env.request.headers[header] @@ -5706,7 +5657,7 @@ get "/sb/:id/:storyboard/:index" do |env| end begin - client.get(url, headers) do |response| + YT_POOL.client &.get(url, headers) do |response| env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -5714,6 +5665,7 @@ get "/sb/:id/:storyboard/:index" do |env| end end + env.response.headers["Connection"] = "close" env.response.headers["Access-Control-Allow-Origin"] = "*" if response.status_code >= 300 @@ -5731,11 +5683,9 @@ get "/s_p/:id/:name" do |env| id = env.params.url["id"] name = env.params.url["name"] - host = "https://i9.ytimg.com" - client = make_client(URI.parse(host)) url = env.request.resource - headers = HTTP::Headers.new + headers = HTTP::Headers{":authority" => "i9.ytimg.com"} REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? headers[header] = env.request.headers[header] @@ -5743,7 +5693,7 @@ get "/s_p/:id/:name" do |env| end begin - client.get(url, headers) do |response| + YT_POOL.client &.get(url, headers) do |response| env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -5798,9 +5748,11 @@ get "/vi/:id/:name" do |env| id = env.params.url["id"] name = env.params.url["name"] + headers = HTTP::Headers{":authority" => "i.ytimg.com"} + if name == "maxres.jpg" - build_thumbnails(id, config, Kemal.config).each do |thumb| - if YT_IMG_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg").status_code == 200 + build_thumbnails(id).each do |thumb| + if YT_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg", headers).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -5808,7 +5760,6 @@ get "/vi/:id/:name" do |env| end url = "/vi/#{id}/#{name}" - headers = HTTP::Headers.new REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? headers[header] = env.request.headers[header] @@ -5816,7 +5767,7 @@ get "/vi/:id/:name" do |env| end begin - YT_IMG_POOL.client &.get(url, headers) do |response| + YT_POOL.client &.get(url, headers) do |response| env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -5838,8 +5789,8 @@ get "/vi/:id/:name" do |env| end get "/Captcha" do |env| - client = make_client(LOGIN_URL) - response = client.get(env.request.resource) + headers = HTTP::Headers{":authority" => "accounts.google.com"} + response = YT_POOL.client &.get(env.request.resource, headers) env.response.headers["Content-Type"] = response.headers["Content-Type"] response.body end @@ -5904,7 +5855,7 @@ end error 500 do |env| error_message = <<-END_HTML Looks like you've found a bug in Invidious. Feel free to open a new issue - <a href="https://github.com/omarroth/invidious/issues">here</a> + <a href="https://github.com/iv-org/invidious/issues">here</a> or send an email to <a href="mailto:#{CONFIG.admin_email}">#{CONFIG.admin_email}</a>. END_HTML diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 35ef5df2..007aa06c 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -1,22 +1,35 @@ struct InvidiousChannel - db_mapping({ - id: String, - author: String, - updated: Time, - deleted: Bool, - subscribed: Time?, - }) + include DB::Serializable + + property id : String + property author : String + property updated : Time + property deleted : Bool + property subscribed : Time? end struct ChannelVideo - def to_json(locale, config, kemal_config, json : JSON::Builder) + include DB::Serializable + + property id : String + property title : String + property published : Time + property updated : Time + property ucid : String + property author : String + property length_seconds : Int32 = 0 + property live_now : Bool = false + property premiere_timestamp : Time? = nil + property views : Int64? = nil + + def to_json(locale, json : JSON::Builder) json.object do json.field "type", "shortVideo" json.field "title", self.title json.field "videoId", self.id json.field "videoThumbnails" do - generate_thumbnails(json, self.id, config, Kemal.config) + generate_thumbnails(json, self.id) end json.field "lengthSeconds", self.length_seconds @@ -31,17 +44,17 @@ struct ChannelVideo end end - def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil) + def to_json(locale, json : JSON::Builder | Nil = nil) if json - to_json(locale, config, kemal_config, json) + to_json(locale, json) else JSON.build do |json| - to_json(locale, config, kemal_config, json) + to_json(locale, json) end end end - def to_xml(locale, host_url, query_params, xml : XML::Builder) + def to_xml(locale, query_params, xml : XML::Builder) query_params["v"] = self.id xml.element("entry") do @@ -49,17 +62,17 @@ struct ChannelVideo xml.element("yt:videoId") { xml.text self.id } xml.element("yt:channelId") { xml.text self.ucid } xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{host_url}/watch?#{query_params}") + xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") xml.element("author") do xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } end xml.element("content", type: "xhtml") do xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{host_url}/watch?#{query_params}") do - xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg") + xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do + xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") end end end @@ -69,64 +82,51 @@ struct ChannelVideo xml.element("media:group") do xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg", + xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", width: "320", height: "180") end end end - def to_xml(locale, config, kemal_config, xml : XML::Builder | Nil = nil) + def to_xml(locale, xml : XML::Builder | Nil = nil) if xml - to_xml(locale, config, kemal_config, xml) + to_xml(locale, xml) else XML.build do |xml| - to_xml(locale, config, kemal_config, xml) + to_xml(locale, xml) end end end - - db_mapping({ - id: String, - title: String, - published: Time, - updated: Time, - ucid: String, - author: String, - length_seconds: {type: Int32, default: 0}, - live_now: {type: Bool, default: false}, - premiere_timestamp: {type: Time?, default: nil}, - views: {type: Int64?, default: nil}, - }) end struct AboutRelatedChannel - db_mapping({ - ucid: String, - author: String, - author_url: String, - author_thumbnail: String, - }) + include DB::Serializable + + property ucid : String + property author : String + property author_url : String + property author_thumbnail : String end # TODO: Refactor into either SearchChannel or InvidiousChannel struct AboutChannel - db_mapping({ - ucid: String, - author: String, - auto_generated: Bool, - author_url: String, - author_thumbnail: String, - banner: String?, - description_html: String, - paid: Bool, - total_views: Int64, - sub_count: Int32, - joined: Time, - is_family_friendly: Bool, - allowed_regions: Array(String), - related_channels: Array(AboutRelatedChannel), - tabs: Array(String), - }) + include DB::Serializable + + property ucid : String + property author : String + property auto_generated : Bool + property author_url : String + property author_thumbnail : String + property banner : String? + property description_html : String + property paid : Bool + property total_views : Int64 + property sub_count : Int32 + property joined : Time + property is_family_friendly : Bool + property allowed_regions : Array(String) + property related_channels : Array(AboutRelatedChannel) + property tabs : Array(String) end class ChannelRedirect < Exception @@ -213,33 +213,20 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) page = 1 - url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated) - response = YT_POOL.client &.get(url) + response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) + videos = [] of SearchVideo begin - json = JSON.parse(response.body) + initial_data = JSON.parse(response.body).as_a.find &.["response"]? + raise "Could not extract JSON" if !initial_data + videos = extract_videos(initial_data.as_h, author, ucid) rescue ex if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") || response.body.includes?("https://www.google.com/sorry/index") raise "Could not extract channel info. Instance is likely blocked." end - - raise "Could not extract JSON" end - if json["content_html"]? && !json["content_html"].as_s.empty? - document = XML.parse_html(json["content_html"].as_s) - nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])) - - if auto_generated - videos = extract_videos(nodeset) - else - videos = extract_videos(nodeset, ucid, author) - end - end - - videos ||= [] of ChannelVideo - rss.xpath_nodes("//feed/entry").each do |entry| video_id = entry.xpath_node("videoid").not_nil!.content title = entry.xpath_node("title").not_nil!.content @@ -260,18 +247,18 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) premiere_timestamp = channel_video.try &.premiere_timestamp - video = ChannelVideo.new( - id: video_id, - title: title, - published: published, - updated: Time.utc, - ucid: ucid, - author: author, - length_seconds: length_seconds, - live_now: live_now, + video = ChannelVideo.new({ + id: video_id, + title: title, + published: published, + updated: Time.utc, + ucid: ucid, + author: author, + length_seconds: length_seconds, + live_now: live_now, premiere_timestamp: premiere_timestamp, - views: views, - ) + views: views, + }) emails = db.query_all("UPDATE users SET notifications = array_append(notifications, $1) \ WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email", @@ -303,38 +290,24 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) ids = [] of String loop do - url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated) - response = YT_POOL.client &.get(url) - json = JSON.parse(response.body) - - if json["content_html"]? && !json["content_html"].as_s.empty? - document = XML.parse_html(json["content_html"].as_s) - nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])) - else - break - end - - nodeset = nodeset.not_nil! - - if auto_generated - videos = extract_videos(nodeset) - else - videos = extract_videos(nodeset, ucid, author) - end - - count = nodeset.size - videos = videos.map { |video| ChannelVideo.new( - id: video.id, - title: video.title, - published: video.published, - updated: Time.utc, - ucid: video.ucid, - author: video.author, - length_seconds: video.length_seconds, - live_now: video.live_now, + response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) + initial_data = JSON.parse(response.body).as_a.find &.["response"]? + raise "Could not extract JSON" if !initial_data + videos = extract_videos(initial_data.as_h, author, ucid) + + count = videos.size + videos = videos.map { |video| ChannelVideo.new({ + id: video.id, + title: video.title, + published: video.published, + updated: Time.utc, + ucid: video.ucid, + author: video.author, + length_seconds: video.length_seconds, + live_now: video.live_now, premiere_timestamp: video.premiere_timestamp, - views: video.views - ) } + views: video.views, + }) } videos.each do |video| ids << video.id @@ -377,7 +350,13 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid) end - channel = InvidiousChannel.new(ucid, author, Time.utc, false, nil) + channel = InvidiousChannel.new({ + id: ucid, + author: author, + updated: Time.utc, + deleted: false, + subscribed: nil, + }) return channel end @@ -387,23 +366,11 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by) url = produce_channel_playlists_url(ucid, continuation, sort_by, auto_generated) response = YT_POOL.client &.get(url) - json = JSON.parse(response.body) - - if json["load_more_widget_html"].as_s.empty? - continuation = nil - else - continuation = XML.parse_html(json["load_more_widget_html"].as_s) - continuation = continuation.xpath_node(%q(//button[@data-uix-load-more-href])) - if continuation - continuation = extract_channel_playlists_cursor(continuation["data-uix-load-more-href"], auto_generated) - end - end - - html = XML.parse_html(json["content_html"].as_s) - nodeset = html.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])) + continuation = response.body.match(/"continuation":"(?<continuation>[^"]+)"/).try &.["continuation"]? + initial_data = JSON.parse(response.body).as_a.find(&.["response"]?).try &.as_h else - url = "/channel/#{ucid}/playlists?disable_polymer=1&flow=list&view=1" + url = "/channel/#{ucid}/playlists?flow=list&view=1" case sort_by when "last", "last_added" @@ -412,55 +379,63 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by) url += "&sort=da" when "newest", "newest_created" url += "&sort=dd" + else nil # Ignore end response = YT_POOL.client &.get(url) - html = XML.parse_html(response.body) - - continuation = html.xpath_node(%q(//button[@data-uix-load-more-href])) - if continuation - continuation = extract_channel_playlists_cursor(continuation["data-uix-load-more-href"], auto_generated) - end - - nodeset = html.xpath_nodes(%q(//ul[@id="browse-items-primary"]/li[contains(@class, "feed-item-container")])) + continuation = response.body.match(/"continuation":"(?<continuation>[^"]+)"/).try &.["continuation"]? + initial_data = extract_initial_data(response.body) end - if auto_generated - items = extract_shelf_items(nodeset, ucid, author) - else - items = extract_items(nodeset, ucid, author) - end + return [] of SearchItem, nil if !initial_data + items = extract_items(initial_data) + continuation = extract_channel_playlists_cursor(continuation, auto_generated) if continuation return items, continuation end -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest") +def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) object = { "80226972:embedded" => { "2:string" => ucid, "3:base64" => { - "2:string" => "videos", - "6:varint": 2_i64, - "7:varint": 1_i64, - "12:varint": 1_i64, - "13:string": "", - "23:varint": 0_i64, + "2:string" => "videos", + "6:varint" => 2_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "13:string" => "", + "23:varint" => 0_i64, }, }, } - if auto_generated - seed = Time.unix(1525757349) - until seed >= Time.utc - seed += 1.month - end - timestamp = seed - (page - 1).months + if !v2 + if auto_generated + seed = Time.unix(1525757349) + until seed >= Time.utc + seed += 1.month + end + timestamp = seed - (page - 1).months - object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64 - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}" + object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x36_i64 + object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{timestamp.to_unix}" + else + object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64 + object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}" + end else object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0_i64 - object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = "#{page}" + + object["80226972:embedded"]["3:base64"].as(Hash)["61:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({ + "1:embedded" => { + "1:varint" => 6307666885028338688_i64, + "2:embedded" => { + "1:string" => Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json({ + "1:varint" => 30_i64 * (page - 1), + }))), + }, + }, + }))) end case sort_by @@ -469,6 +444,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x01_i64 when "oldest" object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 0x02_i64 + else nil # Ignore end object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"]))) @@ -487,12 +463,12 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated "80226972:embedded" => { "2:string" => ucid, "3:base64" => { - "2:string" => "playlists", - "6:varint": 2_i64, - "7:varint": 1_i64, - "12:varint": 1_i64, - "13:string": "", - "23:varint": 0_i64, + "2:string" => "playlists", + "6:varint" => 2_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "13:string" => "", + "23:varint" => 0_i64, }, }, } @@ -513,6 +489,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64 when "last", "last_added" object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64 + else nil # Ignore end end @@ -527,9 +504,8 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" end -def extract_channel_playlists_cursor(url, auto_generated) - cursor = URI.parse(url).query_params - .try { |i| URI.decode_www_form(i["continuation"]) } +def extract_channel_playlists_cursor(cursor, auto_generated) + cursor = URI.decode_www_form(cursor) .try { |i| Base64.decode(i) } .try { |i| IO::Memory.new(i) } .try { |i| Protodec::Any.parse(i) } @@ -554,7 +530,7 @@ def extract_channel_playlists_cursor(url, auto_generated) end # TODO: Add "sort_by" -def fetch_channel_community(ucid, continuation, locale, config, kemal_config, format, thin_mode) +def fetch_channel_community(ucid, continuation, locale, format, thin_mode) response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en") if response.status_code != 200 response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en") @@ -581,16 +557,8 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo headers = HTTP::Headers.new headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] - headers["content-type"] = "application/x-www-form-urlencoded" - headers["x-client-data"] = "CIi2yQEIpbbJAQipncoBCNedygEIqKPKAQ==" - headers["x-spf-previous"] = "" - headers["x-spf-referer"] = "" - - headers["x-youtube-client-name"] = "1" - headers["x-youtube-client-version"] = "2.20180719" - - session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[A-Za-z0-9\_\-\=]+)"/).try &.["session_token"]? || "" + session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || "" post_req = { session_token: session_token, } @@ -630,13 +598,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo next if !post - if !post["contentText"]? - content_html = "" - else - content_html = post["contentText"]["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s || - post["contentText"]["runs"]?.try &.as_a.try { |r| content_to_comment_html(r).try &.to_s } || "" - end - + content_html = post["contentText"]?.try { |t| parse_content(t) } || "" author = post["authorText"]?.try &.["simpleText"]? || "" json.object do @@ -705,7 +667,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo json.field "title", attachment["title"]["simpleText"].as_s json.field "videoId", video_id json.field "videoThumbnails" do - generate_thumbnails(json, video_id, config, kemal_config) + generate_thumbnails(json, video_id) end json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s) @@ -906,12 +868,12 @@ def get_about_info(ucid, locale) related_author_thumbnail = node.xpath_node(%q(.//img)).try &.["data-thumb"] related_author_thumbnail ||= "" - AboutRelatedChannel.new( - ucid: related_id, - author: related_title, - author_url: related_author_url, + AboutRelatedChannel.new({ + ucid: related_id, + author: related_title, + author_url: related_author_url, author_thumbnail: related_author_thumbnail, - ) + }) end joined = about.xpath_node(%q(//span[contains(., "Joined")])) @@ -933,68 +895,61 @@ def get_about_info(ucid, locale) tabs = about.xpath_nodes(%q(//ul[@id="channel-navigation-menu"]/li/a/span)).map { |node| node.content.downcase } - AboutChannel.new( - ucid: ucid, - author: author, - auto_generated: auto_generated, - author_url: author_url, - author_thumbnail: author_thumbnail, - banner: banner, - description_html: description_html, - paid: paid, - total_views: total_views, - sub_count: sub_count, - joined: joined, + AboutChannel.new({ + ucid: ucid, + author: author, + auto_generated: auto_generated, + author_url: author_url, + author_thumbnail: author_thumbnail, + banner: banner, + description_html: description_html, + paid: paid, + total_views: total_views, + sub_count: sub_count, + joined: joined, is_family_friendly: is_family_friendly, - allowed_regions: allowed_regions, - related_channels: related_channels, - tabs: tabs - ) + allowed_regions: allowed_regions, + related_channels: related_channels, + tabs: tabs, + }) +end + +def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") + url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: false) + response = YT_POOL.client &.get(url) + initial_data = JSON.parse(response.body).as_a.find &.["response"]? + return response if !initial_data + needs_v2 = initial_data + .try &.["response"]?.try &.["alerts"]? + .try &.as_a.any? { |alert| + alert.try &.["alertRenderer"]?.try &.["type"]?.try { |t| t == "ERROR" } + } + if needs_v2 + url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) + response = YT_POOL.client &.get(url) + end + response end def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") - count = 0 videos = [] of SearchVideo 2.times do |i| - url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) - response = YT_POOL.client &.get(url) - json = JSON.parse(response.body) - - if json["content_html"]? && !json["content_html"].as_s.empty? - document = XML.parse_html(json["content_html"].as_s) - nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])) - - if !json["load_more_widget_html"]?.try &.as_s.empty? - count += 30 - end - - if auto_generated - videos += extract_videos(nodeset) - else - videos += extract_videos(nodeset, ucid, author) - end - else - break - end + response = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) + initial_data = JSON.parse(response.body).as_a.find &.["response"]? + break if !initial_data + videos.concat extract_videos(initial_data.as_h, author, ucid) end - return videos, count + return videos.size, videos end def get_latest_videos(ucid) - videos = [] of SearchVideo - - url = produce_channel_videos_url(ucid, 0) - response = YT_POOL.client &.get(url) - json = JSON.parse(response.body) - - if json["content_html"]? && !json["content_html"].as_s.empty? - document = XML.parse_html(json["content_html"].as_s) - nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])) - - videos = extract_videos(nodeset, ucid) - end + response = get_channel_videos_response(ucid, 1) + initial_data = JSON.parse(response.body).as_a.find &.["response"]? + return [] of SearchVideo if !initial_data + author = initial_data["response"]?.try &.["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s + items = extract_videos(initial_data.as_h, author, ucid) - return videos + return items end diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 2938247a..407cef78 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -1,11 +1,23 @@ class RedditThing - JSON.mapping({ - kind: String, - data: RedditComment | RedditLink | RedditMore | RedditListing, - }) + include JSON::Serializable + + property kind : String + property data : RedditComment | RedditLink | RedditMore | RedditListing end class RedditComment + include JSON::Serializable + + property author : String + property body_html : String + property replies : RedditThing | String + property score : Int32 + property depth : Int32 + property permalink : String + + @[JSON::Field(converter: RedditComment::TimeConverter)] + property created_utc : Time + module TimeConverter def self.from_json(value : JSON::PullParser) : Time Time.unix(value.read_float.to_i) @@ -15,51 +27,38 @@ class RedditComment json.number(value.to_unix) end end - - JSON.mapping({ - author: String, - body_html: String, - replies: RedditThing | String, - score: Int32, - depth: Int32, - permalink: String, - created_utc: { - type: Time, - converter: RedditComment::TimeConverter, - }, - }) end struct RedditLink - JSON.mapping({ - author: String, - score: Int32, - subreddit: String, - num_comments: Int32, - id: String, - permalink: String, - title: String, - }) + include JSON::Serializable + + property author : String + property score : Int32 + property subreddit : String + property num_comments : Int32 + property id : String + property permalink : String + property title : String end struct RedditMore - JSON.mapping({ - children: Array(String), - count: Int32, - depth: Int32, - }) + include JSON::Serializable + + property children : Array(String) + property count : Int32 + property depth : Int32 end class RedditListing - JSON.mapping({ - children: Array(RedditThing), - modhash: String, - }) + include JSON::Serializable + + property children : Array(RedditThing) + property modhash : String end def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, sort_by = "top") video = get_video(id, db, region: region) - session_token = video.info["session_token"]? + session_token = video.session_token case cursor when nil, "" @@ -85,17 +84,9 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so session_token: session_token, } - headers = HTTP::Headers.new - - headers["content-type"] = "application/x-www-form-urlencoded" - headers["cookie"] = video.info["cookie"] - - headers["x-client-data"] = "CIi2yQEIpbbJAQipncoBCNedygEIqKPKAQ==" - headers["x-spf-previous"] = "https://www.youtube.com/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999" - headers["x-spf-referer"] = "https://www.youtube.com/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999" - - headers["x-youtube-client-name"] = "1" - headers["x-youtube-client-version"] = "2.20180719" + headers = HTTP::Headers{ + "cookie" => video.cookie, + } response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US", headers, form: post_req)) response = JSON.parse(response.body) @@ -150,8 +141,7 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so node_comment = node["commentRenderer"] end - content_html = node_comment["contentText"]["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s || - node_comment["contentText"]["runs"]?.try &.as_a.try { |r| content_to_comment_html(r).try &.to_s } || "" + content_html = node_comment["contentText"]?.try { |t| parse_content(t) } || "" author = node_comment["authorText"]?.try &.["simpleText"]? || "" json.field "author", author @@ -294,7 +284,7 @@ def template_youtube_comments(comments, locale, thin_mode) <div class="pure-u-23-24"> <p> <a href="javascript:void(0)" data-continuation="#{child["replies"]["continuation"]}" - onclick="get_youtube_replies(this)">#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}</a> + data-onclick="get_youtube_replies">#{translate(locale, "View `x` replies", number_with_separator(child["replies"]["replyCount"]))}</a> </p> </div> </div> @@ -356,6 +346,7 @@ def template_youtube_comments(comments, locale, thin_mode) </div> </div> END_HTML + else nil # Ignore end end @@ -413,7 +404,7 @@ def template_youtube_comments(comments, locale, thin_mode) <div class="pure-u-1"> <p> <a href="javascript:void(0)" data-continuation="#{comments["continuation"]}" - onclick="get_youtube_replies(this, true)">#{translate(locale, "Load more")}</a> + data-onclick="get_youtube_replies" data-load-more>#{translate(locale, "Load more")}</a> </p> </div> </div> @@ -451,7 +442,7 @@ def template_reddit_comments(root, locale) html << <<-END_HTML <p> - <a href="javascript:void(0)" onclick="toggle_parent(this)">[ - ]</a> + <a href="javascript:void(0)" data-onclick="toggle_parent">[ - ]</a> <b><a href="https://www.reddit.com/user/#{child.author}">#{child.author}</a></b> #{translate(locale, "`x` points", number_with_separator(child.score))} <span title="#{child.created_utc.to_s(translate(locale, "%a %B %-d %T %Y UTC"))}">#{translate(locale, "`x` ago", recode_date(child.created_utc, locale))}</span> @@ -522,6 +513,11 @@ def fill_links(html, scheme, host) return html.to_xml(options: XML::SaveOptions::NO_DECL) end +def parse_content(content : JSON::Any) : String + content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s || + content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r).try &.to_s } || "" +end + def content_to_comment_html(content) comment_html = content.map do |run| text = HTML.escape(run["text"].as_s) @@ -556,7 +552,7 @@ def content_to_comment_html(content) video_id = watch_endpoint["videoId"].as_s if length_seconds - text = %(<a href="javascript:void(0)" onclick="player.currentTime(#{length_seconds})">#{text}</a>) + text = %(<a href="javascript:void(0)" data-onclick="jump_to_time" data-jump-time="#{length_seconds}">#{text}</a>) else text = %(<a href="/watch?v=#{video_id}">#{text}</a>) end @@ -609,6 +605,8 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top") object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 when "new", "newest" object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64 + else # top + object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 end continuation = object.try { |i| Protodec::Any.cast_json(object) } diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 87b10bc9..d0b6c5a3 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -74,10 +74,10 @@ class FilteredCompressHandler < Kemal::Handler if request_headers.includes_word?("Accept-Encoding", "gzip") env.response.headers["Content-Encoding"] = "gzip" - env.response.output = Gzip::Writer.new(env.response.output, sync_close: true) + env.response.output = Compress::Gzip::Writer.new(env.response.output, sync_close: true) elsif request_headers.includes_word?("Accept-Encoding", "deflate") env.response.headers["Content-Encoding"] = "deflate" - env.response.output = Flate::Writer.new(env.response.output, sync_close: true) + env.response.output = Compress::Deflate::Writer.new(env.response.output, sync_close: true) end call_next env diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 2341d3be..6571f818 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -1,217 +1,100 @@ require "./macros" struct Nonce - db_mapping({ - nonce: String, - expire: Time, - }) + include DB::Serializable + + property nonce : String + property expire : Time end struct SessionId - db_mapping({ - id: String, - email: String, - issued: String, - }) + include DB::Serializable + + property id : String + property email : String + property issued : String end struct Annotation - db_mapping({ - id: String, - annotations: String, - }) + include DB::Serializable + + property id : String + property annotations : String end struct ConfigPreferences - module StringToArray - def self.to_json(value : Array(String), json : JSON::Builder) - json.array do - value.each do |element| - json.string element - end - end - end - - def self.from_json(value : JSON::PullParser) : Array(String) - begin - result = [] of String - value.read_array do - result << HTML.escape(value.read_string[0, 100]) - end - rescue ex - result = [HTML.escape(value.read_string[0, 100]), ""] - end - - result - end - - def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder) - yaml.sequence do - value.each do |element| - yaml.scalar element - end - end - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String) - begin - unless node.is_a?(YAML::Nodes::Sequence) - node.raise "Expected sequence, not #{node.class}" - end - - result = [] of String - node.nodes.each do |item| - unless item.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{item.class}" - end - - result << HTML.escape(item.value[0, 100]) - end - rescue ex - if node.is_a?(YAML::Nodes::Scalar) - result = [HTML.escape(node.value[0, 100]), ""] - else - result = ["", ""] - end - end - - result - end - end - - module BoolToString - def self.to_json(value : String, json : JSON::Builder) - json.string value - end - - def self.from_json(value : JSON::PullParser) : String - begin - result = value.read_string - - if result.empty? - CONFIG.default_user_preferences.dark_mode - else - result - end - rescue ex - if value.read_bool - "dark" - else - "light" - end - end - end - - def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) - yaml.scalar value - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String - unless node.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{node.class}" - end - - case node.value - when "true" - "dark" - when "false" - "light" - when "" - CONFIG.default_user_preferences.dark_mode - else - node.value - end - end + include YAML::Serializable + + property annotations : Bool = false + property annotations_subscribed : Bool = false + property autoplay : Bool = false + property captions : Array(String) = ["", "", ""] + property comments : Array(String) = ["youtube", ""] + property continue : Bool = false + property continue_autoplay : Bool = true + property dark_mode : String = "" + property latest_only : Bool = false + property listen : Bool = false + property local : Bool = false + property locale : String = "en-US" + property max_results : Int32 = 40 + property notifications_only : Bool = false + property player_style : String = "invidious" + property quality : String = "hd720" + property default_home : String = "Popular" + property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"] + property related_videos : Bool = true + property sort : String = "published" + property speed : Float32 = 1.0_f32 + property thin_mode : Bool = false + property unseen_only : Bool = false + property video_loop : Bool = false + property volume : Int32 = 100 + + def to_tuple + {% begin %} + { + {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}} + } + {% end %} end - - yaml_mapping({ - annotations: {type: Bool, default: false}, - annotations_subscribed: {type: Bool, default: false}, - autoplay: {type: Bool, default: false}, - captions: {type: Array(String), default: ["", "", ""], converter: StringToArray}, - comments: {type: Array(String), default: ["youtube", ""], converter: StringToArray}, - continue: {type: Bool, default: false}, - continue_autoplay: {type: Bool, default: true}, - dark_mode: {type: String, default: "", converter: BoolToString}, - latest_only: {type: Bool, default: false}, - listen: {type: Bool, default: false}, - local: {type: Bool, default: false}, - locale: {type: String, default: "en-US"}, - max_results: {type: Int32, default: 40}, - notifications_only: {type: Bool, default: false}, - player_style: {type: String, default: "invidious"}, - quality: {type: String, default: "hd720"}, - default_home: {type: String, default: "Popular"}, - feed_menu: {type: Array(String), default: ["Popular", "Trending", "Subscriptions", "Playlists"]}, - related_videos: {type: Bool, default: true}, - sort: {type: String, default: "published"}, - speed: {type: Float32, default: 1.0_f32}, - thin_mode: {type: Bool, default: false}, - unseen_only: {type: Bool, default: false}, - video_loop: {type: Bool, default: false}, - volume: {type: Int32, default: 100}, - }) end struct Config - module ConfigPreferencesConverter - def self.to_yaml(value : Preferences, yaml : YAML::Nodes::Builder) - value.to_yaml(yaml) - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Preferences - Preferences.new(*ConfigPreferences.new(ctx, node).to_tuple) - end - end - - module FamilyConverter - def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder) - case value - when Socket::Family::UNSPEC - yaml.scalar nil - when Socket::Family::INET - yaml.scalar "ipv4" - when Socket::Family::INET6 - yaml.scalar "ipv6" - end - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family - if node.is_a?(YAML::Nodes::Scalar) - case node.value.downcase - when "ipv4" - Socket::Family::INET - when "ipv6" - Socket::Family::INET6 - else - Socket::Family::UNSPEC - end - else - node.raise "Expected scalar, not #{node.class}" - end - end - end - - module StringToCookies - def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder) - (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml) - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies - unless node.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{node.class}" - end - - cookies = HTTP::Cookies.new - node.value.split(";").each do |cookie| - next if cookie.strip.empty? - name, value = cookie.split("=", 2) - cookies << HTTP::Cookie.new(name.strip, value.strip) - end - - cookies - end - end + include YAML::Serializable + + property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions) + property feed_threads : Int32 # Number of threads to use for updating feeds + property db : DBConfig # Database configuration + property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel + property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// + property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions + property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required + property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property captcha_enabled : Bool = true + property login_enabled : Bool = true + property registration_enabled : Bool = true + property statistics_enabled : Bool = false + property admins : Array(String) = [] of String + property external_port : Int32? = nil + property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("") + property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs + property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc. + property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards + property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc. + property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely + property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' + + @[YAML::Field(converter: Preferences::FamilyConverter)] + property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) + property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument) + property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument) + property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) + property admin_email : String = "omarroth@protonmail.com" # Email for bug reports + + @[YAML::Field(converter: Preferences::StringToCookies)] + property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format + property captcha_key : String? = nil # Key for Anti-Captcha def disabled?(option) case disabled = CONFIG.disable_proxy @@ -223,77 +106,20 @@ struct Config else return false end + else + return false end end - - YAML.mapping({ - channel_threads: Int32, # Number of threads to use for crawling videos from channels (for updating subscriptions) - feed_threads: Int32, # Number of threads to use for updating feeds - db: DBConfig, # Database configuration - full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel - https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https:// - hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions - domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required - use_pubsub_feeds: {type: Bool | Int32, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) - top_enabled: {type: Bool, default: true}, - captcha_enabled: {type: Bool, default: true}, - login_enabled: {type: Bool, default: true}, - registration_enabled: {type: Bool, default: true}, - statistics_enabled: {type: Bool, default: false}, - admins: {type: Array(String), default: [] of String}, - external_port: {type: Int32?, default: nil}, - default_user_preferences: {type: Preferences, - default: Preferences.new(*ConfigPreferences.from_yaml("").to_tuple), - converter: ConfigPreferencesConverter, - }, - dmca_content: {type: Array(String), default: [] of String}, # For compliance with DMCA, disables download widget using list of video IDs - check_tables: {type: Bool, default: false}, # Check table integrity, automatically try to add any missing columns, create tables, etc. - cache_annotations: {type: Bool, default: false}, # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards - banner: {type: String?, default: nil}, # Optional banner to be displayed along top of page for announcements, etc. - hsts: {type: Bool?, default: true}, # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely - disable_proxy: {type: Bool? | Array(String)?, default: false}, # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' - force_resolve: {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) - port: {type: Int32, default: 3000}, # Port to listen for connections (overrided by command line argument) - host_binding: {type: String, default: "0.0.0.0"}, # Host to bind (overrided by command line argument) - pool_size: {type: Int32, default: 100}, # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) - admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports - cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format - captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha - }) end struct DBConfig - yaml_mapping({ - user: String, - password: String, - host: String, - port: Int32, - dbname: String, - }) -end - -def rank_videos(db, n) - top = [] of {Float64, String} + include YAML::Serializable - db.query("SELECT id, wilson_score, published FROM videos WHERE views > 5000 ORDER BY published DESC LIMIT 1000") do |rs| - rs.each do - id = rs.read(String) - wilson_score = rs.read(Float64) - published = rs.read(Time) - - # Exponential decay, older videos tend to rank lower - temperature = wilson_score * Math.exp(-0.000005*((Time.utc - published).total_minutes)) - top << {temperature, id} - end - end - - top.sort! - - # Make hottest come first - top.reverse! - top = top.map { |a, b| b } - - return top[0..n - 1] + property user : String + property password : String + property host : String + property port : Int32 + property dbname : String end def login_req(f_req) @@ -334,293 +160,179 @@ def html_to_content(description_html : String) return description end -def extract_videos(nodeset, ucid = nil, author_name = nil) - videos = extract_items(nodeset, ucid, author_name) - videos.select { |item| item.is_a?(SearchVideo) }.map { |video| video.as(SearchVideo) } +def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) + extract_items(initial_data, author_fallback, author_id_fallback).select(&.is_a?(SearchVideo)).map(&.as(SearchVideo)) end -def extract_items(nodeset, ucid = nil, author_name = nil) - # TODO: Make this a 'common', so it makes more sense to be used here - items = [] of SearchItem - - nodeset.each do |node| - anchor = node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a)) - if !anchor - next - end - title = anchor.content.strip - id = anchor["href"] - - if anchor["href"].starts_with? "https://www.googleadservices.com" - next - end - - author_id = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-byline")]/a)).try &.["href"].split("/")[-1] || ucid || "" - author = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-byline")]/a)).try &.content.strip || author_name || "" - description_html = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-description")])).try &.to_s || "" - - tile = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-tile")])) - if !tile - next - end - - case tile["class"] - when .includes? "yt-lockup-playlist" - plid = HTTP::Params.parse(URI.parse(id).query.not_nil!)["list"] - - anchor = node.xpath_node(%q(.//div[contains(@class, "yt-lockup-meta")]/a)) - - if !anchor - anchor = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/a)) - end - - video_count = node.xpath_node(%q(.//span[@class="formatted-video-count-label"]/b)) || - node.xpath_node(%q(.//span[@class="formatted-video-count-label"])) - if video_count - video_count = video_count.content - - if video_count == "50+" - author = "YouTube" - author_id = "UC-9-kyTW8ZkZNDHQJ6FgpwQ" - end - - video_count = video_count.gsub(/\D/, "").to_i? - end - video_count ||= 0 - - videos = [] of SearchPlaylistVideo - node.xpath_nodes(%q(.//*[contains(@class, "yt-lockup-playlist-items")]/li)).each do |video| - anchor = video.xpath_node(%q(.//a)) - if anchor - video_title = anchor.content.strip - id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"] - end - video_title ||= "" - id ||= "" - - anchor = video.xpath_node(%q(.//span/span)) - if anchor - length_seconds = decode_length_seconds(anchor.content) - end - length_seconds ||= 0 - - videos << SearchPlaylistVideo.new( - video_title, - id, - length_seconds - ) - end - - playlist_thumbnail = node.xpath_node(%q(.//span/img)).try &.["data-thumb"]? - playlist_thumbnail ||= node.xpath_node(%q(.//span/img)).try &.["src"] - - items << SearchPlaylist.new( - title: title, - id: plid, - author: author, - ucid: author_id, - video_count: video_count, - videos: videos, - thumbnail: playlist_thumbnail - ) - when .includes? "yt-lockup-channel" - author = title.strip - - ucid = node.xpath_node(%q(.//button[contains(@class, "yt-uix-subscription-button")])).try &.["data-channel-external-id"]? - ucid ||= id.split("/")[-1] - - author_thumbnail = node.xpath_node(%q(.//div/span/img)).try &.["data-thumb"]? - author_thumbnail ||= node.xpath_node(%q(.//div/span/img)).try &.["src"] - if author_thumbnail - author_thumbnail = URI.parse(author_thumbnail) - author_thumbnail.scheme = "https" - author_thumbnail = author_thumbnail.to_s - end - - author_thumbnail ||= "" - - subscriber_count = node.xpath_node(%q(.//span[contains(@class, "subscriber-count")])) - .try &.["title"].try { |text| short_text_to_number(text) } || 0 - - video_count = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li)).try &.content.split(" ")[0].gsub(/\D/, "").to_i? - - items << SearchChannel.new( - author: author, - ucid: ucid, - author_thumbnail: author_thumbnail, - subscriber_count: subscriber_count, - video_count: video_count || 0, - description_html: description_html, - auto_generated: video_count ? false : true, - ) - else - id = id.lchop("/watch?v=") - - metadata = node.xpath_node(%q(.//div[contains(@class,"yt-lockup-meta")]/ul)) - - published = metadata.try &.xpath_node(%q(.//li[contains(text(), " ago")])).try { |node| decode_date(node.content.sub(/^[a-zA-Z]+ /, "")) } - published ||= metadata.try &.xpath_node(%q(.//span[@data-timestamp])).try { |node| Time.unix(node["data-timestamp"].to_i64) } - published ||= Time.utc - - view_count = metadata.try &.xpath_node(%q(.//li[contains(text(), " views")])).try &.content.gsub(/\D/, "").to_i64? - view_count ||= 0_i64 - - length_seconds = node.xpath_node(%q(.//span[@class="video-time"])).try { |node| decode_length_seconds(node.content) } - length_seconds ||= -1 - - live_now = node.xpath_node(%q(.//span[contains(@class, "yt-badge-live")])) ? true : false - premium = node.xpath_node(%q(.//span[text()="Premium"])) ? true : false - - if !premium || node.xpath_node(%q(.//span[contains(text(), "Free episode")])) - paid = false - else +def extract_item(item : JSON::Any, author_fallback : String? = nil, author_id_fallback : String? = nil) + if i = (item["videoRenderer"]? || item["gridVideoRenderer"]?) + video_id = i["videoId"].as_s + title = i["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || "" + + author_info = i["ownerText"]?.try &.["runs"].as_a[0]? + author = author_info.try &.["text"].as_s || author_fallback || "" + author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || "" + + published = i["publishedTimeText"]?.try &.["simpleText"]?.try { |t| decode_date(t.as_s) } || Time.local + view_count = i["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64 + description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || "" + length_seconds = i["lengthText"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || + i["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?).try &.["thumbnailOverlayTimeStatusRenderer"]? + .try &.["text"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || 0 + + live_now = false + paid = false + premium = false + + premiere_timestamp = i["upcomingEventData"]?.try &.["startTime"]?.try { |t| Time.unix(t.as_s.to_i64) } + + i["badges"]?.try &.as_a.each do |badge| + b = badge["metadataBadgeRenderer"] + case b["label"].as_s + when "LIVE NOW" + live_now = true + when "New", "4K", "CC" + # TODO + when "Premium" paid = true - end - premiere_timestamp = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/span[@class="localized-date"])).try &.["data-timestamp"]?.try &.to_i64 - if premiere_timestamp - premiere_timestamp = Time.unix(premiere_timestamp) + # TODO: Potentially available as i["topStandaloneBadge"]["metadataBadgeRenderer"] + premium = true + else nil # Ignore end - - items << SearchVideo.new( - title: title, - id: id, - author: author, - ucid: author_id, - published: published, - views: view_count, - description_html: description_html, - length_seconds: length_seconds, - live_now: live_now, - paid: paid, - premium: premium, - premiere_timestamp: premiere_timestamp - ) end - end - return items + SearchVideo.new({ + title: title, + id: video_id, + author: author, + ucid: author_id, + published: published, + views: view_count, + description_html: description_html, + length_seconds: length_seconds, + live_now: live_now, + paid: paid, + premium: premium, + premiere_timestamp: premiere_timestamp, + }) + elsif i = item["channelRenderer"]? + author = i["title"]["simpleText"]?.try &.as_s || author_fallback || "" + author_id = i["channelId"]?.try &.as_s || author_id_fallback || "" + + author_thumbnail = i["thumbnail"]["thumbnails"]?.try &.as_a[0]?.try { |u| "https:#{u["url"]}" } || "" + subscriber_count = i["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s.try { |s| short_text_to_number(s.split(" ")[0]) } || 0 + + auto_generated = false + auto_generated = true if !i["videoCountText"]? + video_count = i["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0 + description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || "" + + SearchChannel.new({ + author: author, + ucid: author_id, + author_thumbnail: author_thumbnail, + subscriber_count: subscriber_count, + video_count: video_count, + description_html: description_html, + auto_generated: auto_generated, + }) + elsif i = item["gridPlaylistRenderer"]? + title = i["title"]["runs"].as_a[0]?.try &.["text"].as_s || "" + plid = i["playlistId"]?.try &.as_s || "" + + video_count = i["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0 + playlist_thumbnail = i["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || "" + + SearchPlaylist.new({ + title: title, + id: plid, + author: author_fallback || "", + ucid: author_id_fallback || "", + video_count: video_count, + videos: [] of SearchPlaylistVideo, + thumbnail: playlist_thumbnail, + }) + elsif i = item["playlistRenderer"]? + title = i["title"]["simpleText"]?.try &.as_s || "" + plid = i["playlistId"]?.try &.as_s || "" + + video_count = i["videoCount"]?.try &.as_s.to_i || 0 + playlist_thumbnail = i["thumbnails"].as_a[0]?.try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"].as_s || "" + + author_info = i["shortBylineText"]?.try &.["runs"].as_a[0]? + author = author_info.try &.["text"].as_s || author_fallback || "" + author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_id_fallback || "" + + videos = i["videos"]?.try &.as_a.map do |v| + v = v["childVideoRenderer"] + v_title = v["title"]["simpleText"]?.try &.as_s || "" + v_id = v["videoId"]?.try &.as_s || "" + v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0 + SearchPlaylistVideo.new({ + title: v_title, + id: v_id, + length_seconds: v_length_seconds, + }) + end || [] of SearchPlaylistVideo + + # TODO: i["publishedTimeText"]? + + SearchPlaylist.new({ + title: title, + id: plid, + author: author, + ucid: author_id, + video_count: video_count, + videos: videos, + thumbnail: playlist_thumbnail, + }) + elsif i = item["radioRenderer"]? # Mix + # TODO + elsif i = item["showRenderer"]? # Show + # TODO + elsif i = item["shelfRenderer"]? + elsif i = item["horizontalCardListRenderer"]? + elsif i = item["searchPyvRenderer"]? # Ad + end end -def extract_shelf_items(nodeset, ucid = nil, author_name = nil) - items = [] of SearchPlaylist - - nodeset.each do |shelf| - shelf_anchor = shelf.xpath_node(%q(.//h2[contains(@class, "branded-page-module-title")])) - next if !shelf_anchor - - title = shelf_anchor.xpath_node(%q(.//span[contains(@class, "branded-page-module-title-text")])).try &.content.strip - title ||= "" - - id = shelf_anchor.xpath_node(%q(.//a)).try &.["href"] - next if !id - - shelf_is_playlist = false - videos = [] of SearchPlaylistVideo - - shelf.xpath_nodes(%q(.//ul[contains(@class, "yt-uix-shelfslider-list") or contains(@class, "expanded-shelf-content-list")]/li)).each do |child_node| - type = child_node.xpath_node(%q(./div)) - if !type - next - end - - case type["class"] - when .includes? "yt-lockup-video" - shelf_is_playlist = true - - anchor = child_node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a)) - if anchor - video_title = anchor.content.strip - video_id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"] - end - video_title ||= "" - video_id ||= "" - - anchor = child_node.xpath_node(%q(.//span[@class="video-time"])) - if anchor - length_seconds = decode_length_seconds(anchor.content) - end - length_seconds ||= 0 - - videos << SearchPlaylistVideo.new( - video_title, - video_id, - length_seconds - ) - when .includes? "yt-lockup-playlist" - anchor = child_node.xpath_node(%q(.//h3[contains(@class, "yt-lockup-title")]/a)) - if anchor - playlist_title = anchor.content.strip - params = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!) - plid = params["list"] - end - playlist_title ||= "" - plid ||= "" - - playlist_thumbnail = child_node.xpath_node(%q(.//span/img)).try &.["data-thumb"]? - playlist_thumbnail ||= child_node.xpath_node(%q(.//span/img)).try &.["src"] - - video_count = child_node.xpath_node(%q(.//span[@class="formatted-video-count-label"]/b)) || - child_node.xpath_node(%q(.//span[@class="formatted-video-count-label"])) - if video_count - video_count = video_count.content.gsub(/\D/, "").to_i? - end - video_count ||= 50 - - videos = [] of SearchPlaylistVideo - child_node.xpath_nodes(%q(.//*[contains(@class, "yt-lockup-playlist-items")]/li)).each do |video| - anchor = video.xpath_node(%q(.//a)) - if anchor - video_title = anchor.content.strip - id = HTTP::Params.parse(URI.parse(anchor["href"]).query.not_nil!)["v"] - end - video_title ||= "" - id ||= "" - - anchor = video.xpath_node(%q(.//span/span)) - if anchor - length_seconds = decode_length_seconds(anchor.content) - end - length_seconds ||= 0 - - videos << SearchPlaylistVideo.new( - video_title, - id, - length_seconds - ) - end - - items << SearchPlaylist.new( - title: playlist_title, - id: plid, - author: author_name, - ucid: ucid, - video_count: video_count, - videos: videos, - thumbnail: playlist_thumbnail - ) - end - end - - if shelf_is_playlist - plid = HTTP::Params.parse(URI.parse(id).query.not_nil!)["list"] - - items << SearchPlaylist.new( - title: title, - id: plid, - author: author_name, - ucid: ucid, - video_count: videos.size, - videos: videos, - thumbnail: "https://i.ytimg.com/vi/#{videos[0].id}/mqdefault.jpg" - ) - end - end +def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) + items = [] of SearchItem - return items + channel_v2_response = initial_data + .try &.["response"]? + .try &.["continuationContents"]? + .try &.["gridContinuation"]? + .try &.["items"]? + + if channel_v2_response + channel_v2_response.try &.as_a.each { |item| + extract_item(item, author_fallback, author_id_fallback) + .try { |t| items << t } + } + else + initial_data.try { |t| t["contents"]? || t["response"]? } + .try { |t| t["twoColumnBrowseResultsRenderer"]?.try &.["tabs"].as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]?.try &.["tabRenderer"]["content"] || + t["twoColumnSearchResultsRenderer"]?.try &.["primaryContents"] || + t["continuationContents"]? } + .try { |t| t["sectionListRenderer"]? || t["sectionListContinuation"]? } + .try &.["contents"].as_a + .each { |c| c.try &.["itemSectionRenderer"]?.try &.["contents"].as_a + .try { |t| t[0]?.try &.["shelfRenderer"]?.try &.["content"]["expandedShelfContentsRenderer"]?.try &.["items"].as_a || + t[0]?.try &.["gridRenderer"]?.try &.["items"].as_a || t } + .each { |item| + extract_item(item, author_fallback, author_id_fallback) + .try { |t| items << t } + } } + end + + items end def check_enum(db, logger, enum_name, struct_type = nil) + return # TODO + if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) logger.puts("CREATE TYPE #{enum_name}") @@ -642,18 +354,14 @@ def check_table(db, logger, table_name, struct_type = nil) end end - if !struct_type - return - end + return if !struct_type - struct_array = struct_type.to_type_tuple + struct_array = struct_type.type_array column_array = get_column_array(db, table_name) column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/) - .try &.["types"].split(",").map { |line| line.strip } + .try &.["types"].split(",").map { |line| line.strip }.reject &.starts_with?("CONSTRAINT") - if !column_types - return - end + return if !column_types struct_array.each_with_index do |name, i| if name != column_array[i]? @@ -704,6 +412,15 @@ def check_table(db, logger, table_name, struct_type = nil) end end end + + return if column_array.size <= struct_array.size + + column_array.each do |column| + if !struct_array.includes? column + logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") + db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") + end + end end class PG::ResultSet @@ -732,9 +449,7 @@ def cache_annotation(db, id, annotations) body = XML.parse(annotations) nodeset = body.xpath_nodes(%q(/document/annotations/annotation)) - if nodeset == 0 - return - end + return if nodeset == 0 has_legacy_annotations = false nodeset.each do |node| @@ -744,13 +459,10 @@ def cache_annotation(db, id, annotations) end end - if has_legacy_annotations - # TODO: Update on conflict? - db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations) - end + db.exec("INSERT INTO annotations VALUES ($1, $2) ON CONFLICT DO NOTHING", id, annotations) if has_legacy_annotations end -def create_notification_stream(env, config, kemal_config, decrypt_function, topics, connection_channel) +def create_notification_stream(env, topics, connection_channel) connection = Channel(PQ::Notification).new(8) connection_channel.send({true, connection}) @@ -765,12 +477,12 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi loop do time_span = [0, 0, 0, 0] time_span[rand(4)] = rand(30) + 5 - published = Time.utc - Time::Span.new(time_span[0], time_span[1], time_span[2], time_span[3]) + published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3]) video_id = TEST_IDS[rand(TEST_IDS.size)] video = get_video(video_id, PG_DB) video.published = published - response = JSON.parse(video.to_json(locale, config, kemal_config, decrypt_function)) + response = JSON.parse(video.to_json(locale)) if fields_text = env.params.query["fields"]? begin @@ -804,7 +516,7 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi when .match(/UC[A-Za-z0-9_-]{22}/) PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15", topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video| - response = JSON.parse(video.to_json(locale, config, Kemal.config)) + response = JSON.parse(video.to_json(locale)) if fields_text = env.params.query["fields"]? begin @@ -846,7 +558,7 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi video = get_video(video_id, PG_DB) video.published = Time.unix(published) - response = JSON.parse(video.to_json(locale, config, Kemal.config, decrypt_function)) + response = JSON.parse(video.to_json(locale)) if fields_text = env.params.query["fields"]? begin @@ -884,26 +596,46 @@ def create_notification_stream(env, config, kemal_config, decrypt_function, topi end end -def extract_initial_data(body) - initial_data = body.match(/window\["ytInitialData"\] = (?<info>.*?);\n/).try &.["info"] || "{}" +def extract_initial_data(body) : Hash(String, JSON::Any) + initial_data = body.match(/window\["ytInitialData"\]\s*=\s*(?<info>.*?);+\n/).try &.["info"] || "{}" if initial_data.starts_with?("JSON.parse(\"") - return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s) + return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s).as_h else - return JSON.parse(initial_data) + return JSON.parse(initial_data).as_h end end def proxy_file(response, env) if response.headers.includes_word?("Content-Encoding", "gzip") - Gzip::Writer.open(env.response) do |deflate| - response.pipe(deflate) + Compress::Gzip::Writer.open(env.response) do |deflate| + IO.copy response.body_io, deflate end elsif response.headers.includes_word?("Content-Encoding", "deflate") - Flate::Writer.open(env.response) do |deflate| - response.pipe(deflate) + Compress::Deflate::Writer.open(env.response) do |deflate| + IO.copy response.body_io, deflate end else - response.pipe(env.response) + IO.copy response.body_io, env.response + end +end + +# See https://github.com/kemalcr/kemal/pull/576 +class HTTP::Server::Response::Output + def close + return if closed? + + unless response.wrote_headers? + response.content_length = @out_count + end + + ensure_headers_written + + super + + if @chunked + @io << "0\r\n\r\n" + @io.flush + end end end diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 4c9bb2d6..0faa2e32 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -24,6 +24,8 @@ def translate(locale : Hash(String, JSON::Any) | Nil, translation : String, text if !locale[translation].as_s.empty? translation = locale[translation].as_s end + else + raise "Invalid translation #{translation}" end end diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index 609e53c9..4594c1e0 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr @@ -67,7 +67,7 @@ def refresh_feeds(db, logger, config) begin # Drop outdated views column_array = get_column_array(db, view_name) - ChannelVideo.to_type_tuple.each_with_index do |name, i| + ChannelVideo.type_array.each_with_index do |name, i| if name != column_array[i]? logger.puts("DROP MATERIALIZED VIEW #{view_name}") db.exec("DROP MATERIALIZED VIEW #{view_name}") @@ -170,41 +170,6 @@ def subscribe_to_feeds(db, logger, key, config) end end -def pull_top_videos(config, db) - loop do - begin - top = rank_videos(db, 40) - rescue ex - sleep 1.minute - Fiber.yield - - next - end - - if top.size == 0 - sleep 1.minute - Fiber.yield - - next - end - - videos = [] of Video - - top.each do |id| - begin - videos << get_video(id, db) - rescue ex - next - end - end - - yield videos - - sleep 1.minute - Fiber.yield - end -end - def pull_popular_videos(db) loop do videos = db.query_all("SELECT DISTINCT ON (ucid) * FROM channel_videos WHERE ucid IN \ @@ -225,6 +190,7 @@ def update_decrypt_function decrypt_function = fetch_decrypt_function yield decrypt_function rescue ex + # TODO: Log error next ensure sleep 1.minute @@ -236,12 +202,13 @@ end def bypass_captcha(captcha_key, logger) loop do begin - {"/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw")}.each do |path| + {"/watch?v=CvFH_6DNRCY&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw")}.each do |path| response = YT_POOL.client &.get(path) if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") html = XML.parse_html(response.body) form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil! - site_key = form.xpath_node(%(.//div[@class="g-recaptcha"])).try &.["data-sitekey"] + site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] + s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] inputs = {} of String => String form.xpath_nodes(%(.//input[@name])).map do |node| @@ -253,16 +220,14 @@ def bypass_captcha(captcha_key, logger) response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: { "clientKey" => CONFIG.captcha_key, "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => "https://www.youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999", - "websiteKey" => site_key, + "type" => "NoCaptchaTaskProxyless", + "websiteURL" => "https://www.youtube.com#{path}", + "websiteKey" => site_key, + "recaptchaDataSValue" => s_value, }, }.to_json).body) - if response["error"]? - raise response["error"].as_s - end - + raise response["error"].as_s if response["error"]? task_id = response["taskId"].as_i loop do @@ -281,42 +246,44 @@ def bypass_captcha(captcha_key, logger) end inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s + headers["Cookies"] = response["solution"]["cookies"].as_h.map { |k, v| "#{k}=#{v}" }.join("; ") response = YT_POOL.client &.post("/das_captcha", headers, form: inputs) yield response.cookies.select { |cookie| cookie.name != "PREF" } elsif response.headers["Location"]?.try &.includes?("/sorry/index") location = response.headers["Location"].try { |u| URI.parse(u) } - client = QUIC::Client.new(location.host.not_nil!) - response = client.get(location.full_path) + headers = HTTP::Headers{":authority" => location.host.not_nil!} + response = YT_POOL.client &.get(location.full_path, headers) html = XML.parse_html(response.body) form = html.xpath_node(%(//form[@action="index"])).not_nil! - site_key = form.xpath_node(%(.//div[@class="g-recaptcha"])).try &.["data-sitekey"] + site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"] + s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"] inputs = {} of String => String form.xpath_nodes(%(.//input[@name])).map do |node| inputs[node["name"]] = node["value"] end - response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: { + captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com")) + captcha_client.family = CONFIG.force_resolve || Socket::Family::INET + response = JSON.parse(captcha_client.post("/createTask", body: { "clientKey" => CONFIG.captcha_key, "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => location.to_s, - "websiteKey" => site_key, + "type" => "NoCaptchaTaskProxyless", + "websiteURL" => location.to_s, + "websiteKey" => site_key, + "recaptchaDataSValue" => s_value, }, }.to_json).body) - if response["error"]? - raise response["error"].as_s - end - + raise response["error"].as_s if response["error"]? task_id = response["taskId"].as_i loop do sleep 10.seconds - response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: { + response = JSON.parse(captcha_client.post("/getTaskResult", body: { "clientKey" => CONFIG.captcha_key, "taskId" => task_id, }.to_json).body) @@ -329,9 +296,8 @@ def bypass_captcha(captcha_key, logger) end inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s - client.close - client = QUIC::Client.new("www.google.com") - response = client.post(location.full_path, form: inputs) + headers["Cookies"] = response["solution"]["cookies"].as_h.map { |k, v| "#{k}=#{v}" }.join("; ") + response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs) headers = HTTP::Headers{ "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0], } diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr index ddfb9f8e..8b74bc86 100644 --- a/src/invidious/helpers/macros.cr +++ b/src/invidious/helpers/macros.cr @@ -1,43 +1,51 @@ -macro db_mapping(mapping) - def initialize({{*mapping.keys.map { |id| "@#{id}".id }}}) +module DB::Serializable + macro included + {% verbatim do %} + macro finished + def self.type_array + \{{ @type.instance_vars + .reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] } + .map { |name| name.stringify } + }} + end + + def initialize(tuple) + \{% for var in @type.instance_vars %} + \{% ann = var.annotation(::DB::Field) %} + \{% if ann && ann[:ignore] %} + \{% else %} + @\{{var.name}} = tuple[:\{{var.name.id}}] + \{% end %} + \{% end %} + end + + def to_a + \{{ @type.instance_vars + .reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] } + .map { |name| name } + }} + end + end + {% end %} end - - def to_a - return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ] - end - - def self.to_type_tuple - return { {{*mapping.keys.map { |id| "#{id}" }}} } - end - - DB.mapping( {{mapping}} ) -end - -macro json_mapping(mapping) - def initialize({{*mapping.keys.map { |id| "@#{id}".id }}}) - end - - def to_a - return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ] - end - - patched_json_mapping( {{mapping}} ) - YAML.mapping( {{mapping}} ) end -macro yaml_mapping(mapping) - def initialize({{*mapping.keys.map { |id| "@#{id}".id }}}) - end - - def to_a - return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ] - end - - def to_tuple - return { {{*mapping.keys.map { |id| "@#{id}".id }}} } +module JSON::Serializable + macro included + {% verbatim do %} + macro finished + def initialize(tuple) + \{% for var in @type.instance_vars %} + \{% ann = var.annotation(::JSON::Field) %} + \{% if ann && ann[:ignore] %} + \{% else %} + @\{{var.name}} = tuple[:\{{var.name.id}}] + \{% end %} + \{% end %} + end + end + {% end %} end - - YAML.mapping({{mapping}}) end macro templated(filename, template = "template") diff --git a/src/invidious/helpers/patch_mapping.cr b/src/invidious/helpers/patch_mapping.cr deleted file mode 100644 index 19bd8ca1..00000000 --- a/src/invidious/helpers/patch_mapping.cr +++ /dev/null @@ -1,166 +0,0 @@ -# Overloads https://github.com/crystal-lang/crystal/blob/0.28.0/src/json/from_json.cr#L24 -def Object.from_json(string_or_io, default) : self - parser = JSON::PullParser.new(string_or_io) - new parser, default -end - -# Adds configurable 'default' -macro patched_json_mapping(_properties_, strict = false) - {% for key, value in _properties_ %} - {% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %} - {% end %} - - {% for key, value in _properties_ %} - {% _properties_[key][:key_id] = key.id.gsub(/\?$/, "") %} - {% end %} - - {% for key, value in _properties_ %} - @{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }} - - {% if value[:setter] == nil ? true : value[:setter] %} - def {{value[:key_id]}}=(_{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}) - @{{value[:key_id]}} = _{{value[:key_id]}} - end - {% end %} - - {% if value[:getter] == nil ? true : value[:getter] %} - def {{key.id}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }} - @{{value[:key_id]}} - end - {% end %} - - {% if value[:presence] %} - @{{value[:key_id]}}_present : Bool = false - - def {{value[:key_id]}}_present? - @{{value[:key_id]}}_present - end - {% end %} - {% end %} - - def initialize(%pull : ::JSON::PullParser, default = nil) - {% for key, value in _properties_ %} - %var{key.id} = nil - %found{key.id} = false - {% end %} - - %location = %pull.location - begin - %pull.read_begin_object - rescue exc : ::JSON::ParseException - raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc) - end - until %pull.kind.end_object? - %key_location = %pull.location - key = %pull.read_object_key - case key - {% for key, value in _properties_ %} - when {{value[:key] || value[:key_id].stringify}} - %found{key.id} = true - begin - %var{key.id} = - {% if value[:nilable] || value[:default] != nil %} %pull.read_null_or { {% end %} - - {% if value[:root] %} - %pull.on_key!({{value[:root]}}) do - {% end %} - - {% if value[:converter] %} - {{value[:converter]}}.from_json(%pull) - {% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %} - {{value[:type]}}.new(%pull) - {% else %} - ::Union({{value[:type]}}).new(%pull) - {% end %} - - {% if value[:root] %} - end - {% end %} - - {% if value[:nilable] || value[:default] != nil %} } {% end %} - rescue exc : ::JSON::ParseException - raise ::JSON::MappingError.new(exc.message, self.class.to_s, {{value[:key] || value[:key_id].stringify}}, *%key_location, exc) - end - {% end %} - else - {% if strict %} - raise ::JSON::MappingError.new("Unknown JSON attribute: #{key}", self.class.to_s, nil, *%key_location, nil) - {% else %} - %pull.skip - {% end %} - end - end - %pull.read_next - - {% for key, value in _properties_ %} - {% unless value[:nilable] || value[:default] != nil %} - if %var{key.id}.nil? && !%found{key.id} && !::Union({{value[:type]}}).nilable? - raise ::JSON::MappingError.new("Missing JSON attribute: {{(value[:key] || value[:key_id]).id}}", self.class.to_s, nil, *%location, nil) - end - {% end %} - - {% if value[:nilable] %} - {% if value[:default] != nil %} - @{{value[:key_id]}} = %found{key.id} ? %var{key.id} : (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}}) - {% else %} - @{{value[:key_id]}} = %var{key.id} - {% end %} - {% elsif value[:default] != nil %} - @{{value[:key_id]}} = %var{key.id}.nil? ? (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}}) : %var{key.id} - {% else %} - @{{value[:key_id]}} = (%var{key.id}).as({{value[:type]}}) - {% end %} - - {% if value[:presence] %} - @{{value[:key_id]}}_present = %found{key.id} - {% end %} - {% end %} - end - - def to_json(json : ::JSON::Builder) - json.object do - {% for key, value in _properties_ %} - _{{value[:key_id]}} = @{{value[:key_id]}} - - {% unless value[:emit_null] %} - unless _{{value[:key_id]}}.nil? - {% end %} - - json.field({{value[:key] || value[:key_id].stringify}}) do - {% if value[:root] %} - {% if value[:emit_null] %} - if _{{value[:key_id]}}.nil? - nil.to_json(json) - else - {% end %} - - json.object do - json.field({{value[:root]}}) do - {% end %} - - {% if value[:converter] %} - if _{{value[:key_id]}} - {{ value[:converter] }}.to_json(_{{value[:key_id]}}, json) - else - nil.to_json(json) - end - {% else %} - _{{value[:key_id]}}.to_json(json) - {% end %} - - {% if value[:root] %} - {% if value[:emit_null] %} - end - {% end %} - end - end - {% end %} - end - - {% unless value[:emit_null] %} - end - {% end %} - {% end %} - end - end -end diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr index ab864f03..f811500f 100644 --- a/src/invidious/helpers/signatures.cr +++ b/src/invidious/helpers/signatures.cr @@ -1,8 +1,8 @@ alias SigProc = Proc(Array(String), Int32, Array(String)) def fetch_decrypt_function(id = "CvFH_6DNRCY") - document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1").body - url = document.match(/src="(?<url>\/yts\/jsbin\/player_ias-[^\/]+\/en_US\/base.js)"/).not_nil!["url"] + document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body + url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"] player = YT_POOL.client &.get(url).body function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"] @@ -40,12 +40,12 @@ def fetch_decrypt_function(id = "CvFH_6DNRCY") return decrypt_function end -def decrypt_signature(fmt, op) +def decrypt_signature(fmt : Hash(String, JSON::Any)) return "" if !fmt["s"]? || !fmt["sp"]? - sp = fmt["sp"] - sig = fmt["s"].split("") - op.each do |proc, value| + sp = fmt["sp"].as_s + sig = fmt["s"].as_s.split("") + DECRYPT_FUNCTION.each do |proc, value| sig = proc.call(sig, value) end diff --git a/src/invidious/helpers/static_file_handler.cr b/src/invidious/helpers/static_file_handler.cr index 20d92b9c..be9d36ab 100644 --- a/src/invidious/helpers/static_file_handler.cr +++ b/src/invidious/helpers/static_file_handler.cr @@ -81,12 +81,12 @@ def send_file(env : HTTP::Server::Context, file_path : String, data : Slice(UInt condition = config.is_a?(Hash) && config["gzip"]? == true && filesize > minsize && Kemal::Utils.zip_types(file_path) if condition && request_headers.includes_word?("Accept-Encoding", "gzip") env.response.headers["Content-Encoding"] = "gzip" - Gzip::Writer.open(env.response) do |deflate| + Compress::Gzip::Writer.open(env.response) do |deflate| IO.copy(file, deflate) end elsif condition && request_headers.includes_word?("Accept-Encoding", "deflate") env.response.headers["Content-Encoding"] = "deflate" - Flate::Writer.open(env.response) do |deflate| + Compress::Deflate::Writer.open(env.response) do |deflate| IO.copy(file, deflate) end else diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index 30f7d4f4..39aae367 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -1,3 +1,5 @@ +require "crypto/subtle" + def generate_token(email, scopes, expire, key, db) session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}" PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", session, email, Time.utc) @@ -41,15 +43,10 @@ def sign_token(key, hash) string_to_sign = [] of String hash.each do |key, value| - if key == "signature" - next - end + next if key == "signature" - if value.is_a?(JSON::Any) - case value - when .as_a? - value = value.as_a.map { |item| item.as_s } - end + if value.is_a?(JSON::Any) && value.as_a? + value = value.as_a.map { |i| i.as_s } end case value @@ -76,32 +73,31 @@ def validate_request(token, session, request, key, db, locale = nil) raise translate(locale, "Hidden field \"token\" is a required field") end - if token["signature"] != sign_token(key, token) - raise translate(locale, "Invalid signature") + expire = token["expire"]?.try &.as_i + if expire.try &.< Time.utc.to_unix + raise translate(locale, "Token is expired, please try again") end if token["session"] != session raise translate(locale, "Erroneous token") end - if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time})) - if nonce[1] > Time.utc - db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0]) - else - raise translate(locale, "Erroneous token") - end - end - scopes = token["scopes"].as_a.map { |v| v.as_s } scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}" - if !scopes_include_scope(scopes, scope) raise translate(locale, "Invalid scope") end - expire = token["expire"]?.try &.as_i - if expire.try &.< Time.utc.to_unix - raise translate(locale, "Token is expired, please try again") + if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token)) + raise translate(locale, "Invalid signature") + end + + if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time})) + if nonce[1] > Time.utc + db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0]) + else + raise translate(locale, "Erroneous token") + end end return {scopes, expire, token["signature"].as_s} diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index e43ae71d..a51f15ce 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -2,13 +2,16 @@ require "lsquic" require "pool/connection" def add_yt_headers(request) - request.headers["x-youtube-client-name"] ||= "1" - request.headers["x-youtube-client-version"] ||= "1.20180719" request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" request.headers["accept-language"] ||= "en-us,en;q=0.5" - request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + return if request.resource.starts_with? "/sorry/index" + request.headers["x-youtube-client-name"] ||= "1" + request.headers["x-youtube-client-version"] ||= "2.20200609" + if !CONFIG.cookies.empty? + request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end end struct QUICPool @@ -77,7 +80,8 @@ def elapsed_text(elapsed) end def make_client(url : URI, region = nil) - client = HTTPClient.new(url) + # TODO: Migrate any applicable endpoints to QUIC + client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC client.read_timeout = 10.seconds client.connect_timeout = 10.seconds @@ -99,7 +103,7 @@ end def decode_length_seconds(string) length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i length_seconds = [0] * (3 - length_seconds.size) + length_seconds - length_seconds = Time::Span.new(length_seconds[0], length_seconds[1], length_seconds[2]) + length_seconds = Time::Span.new hours: length_seconds[0], minutes: length_seconds[1], seconds: length_seconds[2] length_seconds = length_seconds.total_seconds.to_i return length_seconds @@ -161,6 +165,7 @@ def decode_date(string : String) return Time.utc when "yesterday" return Time.utc - 1.day + else nil # Continue end # String matches format "20 hours ago", "4 months ago"... @@ -315,7 +320,7 @@ def get_referer(env, fallback = "/", unroll = true) end referer = referer.full_path - referer = "/" + referer.lstrip("/\\") + referer = "/" + referer.gsub(/[^\/?@&%=\-_.0-9a-zA-Z]/, "").lstrip("/\\") if referer == env.request.path referer = fallback @@ -324,47 +329,10 @@ def get_referer(env, fallback = "/", unroll = true) return referer end -struct VarInt - def self.from_io(io : IO, format = IO::ByteFormat::NetworkEndian) : Int32 - result = 0_u32 - num_read = 0 - - loop do - byte = io.read_byte - raise "Invalid VarInt" if !byte - value = byte & 0x7f - - result |= value.to_u32 << (7 * num_read) - num_read += 1 - - break if byte & 0x80 == 0 - raise "Invalid VarInt" if num_read > 5 - end - - result.to_i32 - end - - def self.to_io(io : IO, value : Int32) - io.write_byte 0x00 if value == 0x00 - value = value.to_u32 - - while value != 0 - byte = (value & 0x7f).to_u8 - value >>= 7 - - if value != 0 - byte |= 0x80 - end - - io.write_byte byte - end - end -end - def sha256(text) digest = OpenSSL::Digest.new("SHA256") digest << text - return digest.hexdigest + return digest.final.hexstring end def subscribe_pubsub(topic, key, config) @@ -383,10 +351,8 @@ def subscribe_pubsub(topic, key, config) nonce = Random::Secure.hex(4) signature = "#{time}:#{nonce}" - host_url = make_host_url(config, Kemal.config) - body = { - "hub.callback" => "#{host_url}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}", + "hub.callback" => "#{HOST_URL}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}", "hub.topic" => "https://www.youtube.com/xml/feeds/videos.xml?#{topic}", "hub.verify" => "async", "hub.mode" => "subscribe", diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 04a37b87..c69eb0c4 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -1,32 +1,32 @@ struct MixVideo - db_mapping({ - title: String, - id: String, - author: String, - ucid: String, - length_seconds: Int32, - index: Int32, - rdid: String, - }) + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property length_seconds : Int32 + property index : Int32 + property rdid : String end struct Mix - db_mapping({ - title: String, - id: String, - videos: Array(MixVideo), - }) + include DB::Serializable + + property title : String + property id : String + property videos : Array(MixVideo) end def fetch_mix(rdid, video_id, cookies = nil, locale = nil) headers = HTTP::Headers.new - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" if cookies headers = cookies.add_request_headers(headers) end - response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en&has_verified=1&bpctr=9999999999", headers) + video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_" + response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers) initial_data = extract_initial_data(response.body) if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]? @@ -49,23 +49,22 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) id = item["videoId"].as_s title = item["title"]?.try &.["simpleText"].as_s - if !title - next - end + next if !title + author = item["longBylineText"]["runs"][0]["text"].as_s ucid = item["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s length_seconds = decode_length_seconds(item["lengthText"]["simpleText"].as_s) index = item["navigationEndpoint"]["watchEndpoint"]["index"].as_i - videos << MixVideo.new( - title, - id, - author, - ucid, - length_seconds, - index, - rdid - ) + videos << MixVideo.new({ + title: title, + id: id, + author: author, + ucid: ucid, + length_seconds: length_seconds, + index: index, + rdid: rdid, + }) end if !cookies @@ -75,7 +74,11 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) videos.uniq! { |video| video.id } videos = videos.first(50) - return Mix.new(mix_title, rdid, videos) + return Mix.new({ + title: mix_title, + id: rdid, + videos: videos, + }) end def template_mix(mix) diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 9c8afd3c..9190e4e6 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -1,26 +1,38 @@ struct PlaylistVideo - def to_xml(host_url, auto_generated, xml : XML::Builder) + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property length_seconds : Int32 + property published : Time + property plid : String + property index : Int64 + property live_now : Bool + + def to_xml(auto_generated, xml : XML::Builder) xml.element("entry") do xml.element("id") { xml.text "yt:video:#{self.id}" } xml.element("yt:videoId") { xml.text self.id } xml.element("yt:channelId") { xml.text self.ucid } xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{self.id}") + xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}") xml.element("author") do if auto_generated xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } else xml.element("name") { xml.text author } - xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } end end xml.element("content", type: "xhtml") do xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{host_url}/watch?v=#{self.id}") do - xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg") + xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do + xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") end end end @@ -29,23 +41,23 @@ struct PlaylistVideo xml.element("media:group") do xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg", + xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", width: "320", height: "180") end end end - def to_xml(host_url, auto_generated, xml : XML::Builder? = nil) + def to_xml(auto_generated, xml : XML::Builder? = nil) if xml - to_xml(host_url, auto_generated, xml) + to_xml(auto_generated, xml) else XML.build do |json| - to_xml(host_url, auto_generated, xml) + to_xml(auto_generated, xml) end end end - def to_json(locale, config, kemal_config, json : JSON::Builder, index : Int32?) + def to_json(locale, json : JSON::Builder, index : Int32?) json.object do json.field "title", self.title json.field "videoId", self.id @@ -55,7 +67,7 @@ struct PlaylistVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "videoThumbnails" do - generate_thumbnails(json, self.id, config, kemal_config) + generate_thumbnails(json, self.id) end if index @@ -69,31 +81,32 @@ struct PlaylistVideo end end - def to_json(locale, config, kemal_config, json : JSON::Builder? = nil, index : Int32? = nil) + def to_json(locale, json : JSON::Builder? = nil, index : Int32? = nil) if json - to_json(locale, config, kemal_config, json, index: index) + to_json(locale, json, index: index) else JSON.build do |json| - to_json(locale, config, kemal_config, json, index: index) + to_json(locale, json, index: index) end end end - - db_mapping({ - title: String, - id: String, - author: String, - ucid: String, - length_seconds: Int32, - published: Time, - plid: String, - index: Int64, - live_now: Bool, - }) end struct Playlist - def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil) + include DB::Serializable + + property title : String + property id : String + property author : String + property author_thumbnail : String + property ucid : String + property description : String + property video_count : Int32 + property views : Int64 + property updated : Time + property thumbnail : String? + + def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) json.object do json.field "type", "playlist" json.field "title", self.title @@ -118,7 +131,7 @@ struct Playlist end end - json.field "description", html_to_content(self.description_html) + json.field "description", self.description json.field "descriptionHtml", self.description_html json.field "videoCount", self.video_count @@ -130,39 +143,30 @@ struct Playlist json.array do videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) videos.each_with_index do |video, index| - video.to_json(locale, config, Kemal.config, json) + video.to_json(locale, json) end end end end end - def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil) if json - to_json(offset, locale, config, kemal_config, json, continuation: continuation) + to_json(offset, locale, json, continuation: continuation) else JSON.build do |json| - to_json(offset, locale, config, kemal_config, json, continuation: continuation) + to_json(offset, locale, json, continuation: continuation) end end end - db_mapping({ - title: String, - id: String, - author: String, - author_thumbnail: String, - ucid: String, - description_html: String, - video_count: Int32, - views: Int64, - updated: Time, - thumbnail: String?, - }) - def privacy PlaylistPrivacy::Public end + + def description_html + HTML.escape(self.description).gsub("\n", "<br>") + end end enum PlaylistPrivacy @@ -172,7 +176,30 @@ enum PlaylistPrivacy end struct InvidiousPlaylist - def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil) + include DB::Serializable + + property title : String + property id : String + property author : String + property description : String = "" + property video_count : Int32 + property created : Time + property updated : Time + + @[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)] + property privacy : PlaylistPrivacy = PlaylistPrivacy::Private + property index : Array(Int64) + + @[DB::Field(ignore: true)] + property thumbnail_id : String? + + module PlaylistPrivacyConverter + def self.from_rs(rs) + return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8)))) + end + end + + def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) json.object do json.field "type", "invidiousPlaylist" json.field "title", self.title @@ -195,43 +222,23 @@ struct InvidiousPlaylist json.array do videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) videos.each_with_index do |video, index| - video.to_json(locale, config, Kemal.config, json, offset + index) + video.to_json(locale, json, offset + index) end end end end end - def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil) + def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil) if json - to_json(offset, locale, config, kemal_config, json, continuation: continuation) + to_json(offset, locale, json, continuation: continuation) else JSON.build do |json| - to_json(offset, locale, config, kemal_config, json, continuation: continuation) + to_json(offset, locale, json, continuation: continuation) end end end - property thumbnail_id - - module PlaylistPrivacyConverter - def self.from_rs(rs) - return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8)))) - end - end - - db_mapping({ - title: String, - id: String, - author: String, - description: {type: String, default: ""}, - video_count: Int32, - created: Time, - updated: Time, - privacy: {type: PlaylistPrivacy, default: PlaylistPrivacy::Private, converter: PlaylistPrivacyConverter}, - index: Array(Int64), - }) - def thumbnail @thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------" "/vi/#{@thumbnail_id}/mqdefault.jpg" @@ -257,17 +264,17 @@ end def create_playlist(db, title, privacy, user) plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}" - playlist = InvidiousPlaylist.new( - title: title.byte_slice(0, 150), - id: plid, - author: user.email, + playlist = InvidiousPlaylist.new({ + title: title.byte_slice(0, 150), + id: plid, + author: user.email, description: "", # Max 5000 characters video_count: 0, - created: Time.utc, - updated: Time.utc, - privacy: privacy, - index: [] of Int64, - ) + created: Time.utc, + updated: Time.utc, + privacy: privacy, + index: [] of Int64, + }) playlist_array = playlist.to_a args = arg_array(playlist_array) @@ -277,50 +284,25 @@ def create_playlist(db, title, privacy, user) return playlist end -def extract_playlist(plid, nodeset, index) - videos = [] of PlaylistVideo - - nodeset.each_with_index do |video, offset| - anchor = video.xpath_node(%q(.//td[@class="pl-video-title"])) - if !anchor - next - end - - title = anchor.xpath_node(%q(.//a)).not_nil!.content.strip(" \n") - id = anchor.xpath_node(%q(.//a)).not_nil!["href"].lchop("/watch?v=")[0, 11] - - anchor = anchor.xpath_node(%q(.//div[@class="pl-video-owner"]/a)) - if anchor - author = anchor.content - ucid = anchor["href"].split("/")[2] - else - author = "" - ucid = "" - end +def subscribe_playlist(db, user, playlist) + playlist = InvidiousPlaylist.new({ + title: playlist.title.byte_slice(0, 150), + id: playlist.id, + author: user.email, + description: "", # Max 5000 characters + video_count: playlist.video_count, + created: Time.utc, + updated: playlist.updated, + privacy: PlaylistPrivacy::Private, + index: [] of Int64, + }) - anchor = video.xpath_node(%q(.//td[@class="pl-video-time"]/div/div[1])) - if anchor && !anchor.content.empty? - length_seconds = decode_length_seconds(anchor.content) - live_now = false - else - length_seconds = 0 - live_now = true - end + playlist_array = playlist.to_a + args = arg_array(playlist_array) - videos << PlaylistVideo.new( - title: title, - id: id, - author: author, - ucid: ucid, - length_seconds: length_seconds, - published: Time.utc, - plid: plid, - index: (index + offset).to_i64, - live_now: live_now - ) - end + db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array) - return videos + return playlist end def produce_playlist_url(id, index) @@ -368,58 +350,64 @@ def fetch_playlist(plid, locale) plid = "UU#{plid.lchop("UC")}" end - response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en&disable_polymer=1") + response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en") if response.status_code != 200 - raise translate(locale, "Not a playlist.") + if response.headers["location"]?.try &.includes? "/sorry/index" + raise "Could not extract playlist info. Instance is likely blocked." + else + raise translate(locale, "Not a playlist.") + end end - body = response.body.gsub(/<button[^>]+><span[^>]+>\s*less\s*<img[^>]+>\n<\/span><\/button>/, "") - document = XML.parse_html(body) + initial_data = extract_initial_data(response.body) + playlist_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[0]["playlistSidebarPrimaryInfoRenderer"]? - title = document.xpath_node(%q(//h1[@class="pl-header-title"])) - if !title - raise translate(locale, "Playlist does not exist.") - end - title = title.content.strip(" \n") + raise "Could not extract playlist info" if !playlist_info + title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || "" - description_html = document.xpath_node(%q(//span[@class="pl-header-description-text"]/div/div[1])).try &.to_s || - document.xpath_node(%q(//span[@class="pl-header-description-text"])).try &.to_s || "" + desc_item = playlist_info["description"]? + description = desc_item.try &.["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || "" - playlist_thumbnail = document.xpath_node(%q(//div[@class="pl-header-thumb"]/img)).try &.["data-thumb"]? || - document.xpath_node(%q(//div[@class="pl-header-thumb"]/img)).try &.["src"] + thumbnail = playlist_info["thumbnailRenderer"]?.try &.["playlistVideoThumbnailRenderer"]? + .try &.["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s - # YouTube allows anonymous playlists, so most of this can be empty or optional - anchor = document.xpath_node(%q(//ul[@class="pl-header-details"])) - author = anchor.try &.xpath_node(%q(.//li[1]/a)).try &.content - author ||= "" - author_thumbnail = document.xpath_node(%q(//img[@class="channel-header-profile-image"])).try &.["src"] - author_thumbnail ||= "" - ucid = anchor.try &.xpath_node(%q(.//li[1]/a)).try &.["href"].split("/")[-1] - ucid ||= "" + views = 0_i64 + updated = Time.utc + video_count = 0 + playlist_info["stats"]?.try &.as_a.each do |stat| + text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s + next if !text - video_count = anchor.try &.xpath_node(%q(.//li[2])).try &.content.gsub(/\D/, "").to_i? - video_count ||= 0 + if text.includes? "video" + video_count = text.gsub(/\D/, "").to_i? || 0 + elsif text.includes? "view" + views = text.gsub(/\D/, "").to_i64? || 0_i64 + else + updated = decode_date(text.lchop("Last updated on ").lchop("Updated ")) + end + end - views = anchor.try &.xpath_node(%q(.//li[3])).try &.content.gsub(/\D/, "").to_i64? - views ||= 0_i64 + author_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[1]["playlistSidebarSecondaryInfoRenderer"]? + .try &.["videoOwner"]["videoOwnerRenderer"]? - updated = anchor.try &.xpath_node(%q(.//li[4])).try &.content.lchop("Last updated on ").lchop("Updated ").try { |date| decode_date(date) } - updated ||= Time.utc + raise "Could not extract author info" if !author_info - playlist = Playlist.new( - title: title, - id: plid, - author: author, - author_thumbnail: author_thumbnail, - ucid: ucid, - description_html: description_html, - video_count: video_count, - views: views, - updated: updated, - thumbnail: playlist_thumbnail, - ) + author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || "" + author = author_info["title"]["runs"][0]["text"]?.try &.as_s || "" + ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || "" - return playlist + return Playlist.new({ + title: title, + id: plid, + author: author, + author_thumbnail: author_thumbnail, + ucid: ucid, + description: description, + video_count: video_count, + views: views, + updated: updated, + thumbnail: thumbnail, + }) end def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) @@ -437,35 +425,26 @@ end def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil) if continuation - html = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") - html = XML.parse_html(html.body) - - index = html.xpath_node(%q(//span[@id="playlist-current-index"])).try &.content.to_i?.try &.- 1 - offset = index || offset + response = YT_POOL.client &.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en") + initial_data = extract_initial_data(response.body) + offset = initial_data["currentVideoEndpoint"]?.try &.["watchEndpoint"]?.try &.["index"]?.try &.as_i64 || offset end if video_count > 100 url = produce_playlist_url(plid, offset) response = YT_POOL.client &.get(url) - response = JSON.parse(response.body) - if !response["content_html"]? || response["content_html"].as_s.empty? - raise translate(locale, "Empty playlist") - end - - document = XML.parse_html(response["content_html"].as_s) - nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")])) - videos = extract_playlist(plid, nodeset, offset) + initial_data = JSON.parse(response.body).as_a.find(&.as_h.["response"]?).try &.as_h elsif offset > 100 return [] of PlaylistVideo else # Extract first page of videos - response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1") - document = XML.parse_html(response.body) - nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")])) - - videos = extract_playlist(plid, nodeset, 0) + response = YT_POOL.client &.get("/playlist?list=#{plid}&gl=US&hl=en") + initial_data = extract_initial_data(response.body) end + return [] of PlaylistVideo if !initial_data + videos = extract_playlist_videos(initial_data) + until videos.empty? || videos[0].index == offset videos.shift end @@ -473,6 +452,45 @@ def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuat return videos end +def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) + videos = [] of PlaylistVideo + + (initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"].as_a || + initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a).try &.each do |item| + if i = item["playlistVideoRenderer"]? + video_id = i["navigationEndpoint"]["watchEndpoint"]["videoId"].as_s + plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s + index = i["navigationEndpoint"]["watchEndpoint"]["index"].as_i64 + + thumbnail = i["thumbnail"]["thumbnails"][0]["url"].as_s + title = i["title"].try { |t| t["simpleText"]? || t["runs"]?.try &.[0]["text"]? }.try &.as_s || "" + author = i["shortBylineText"]?.try &.["runs"][0]["text"].as_s || "" + ucid = i["shortBylineText"]?.try &.["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s || "" + length_seconds = i["lengthSeconds"]?.try &.as_s.to_i + live = false + + if !length_seconds + live = true + length_seconds = 0 + end + + videos << PlaylistVideo.new({ + title: title, + id: video_id, + author: author, + ucid: ucid, + length_seconds: length_seconds, + published: Time.utc, + plid: plid, + live_now: live, + index: index - 1, + }) + end + end + + return videos +end + def template_playlist(playlist) html = <<-END_HTML <h3> diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 92996f75..85fd024a 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -1,5 +1,20 @@ struct SearchVideo - def to_xml(host_url, auto_generated, query_params, xml : XML::Builder) + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property published : Time + property views : Int64 + property description_html : String + property length_seconds : Int32 + property live_now : Bool + property paid : Bool + property premium : Bool + property premiere_timestamp : Time? + + def to_xml(auto_generated, query_params, xml : XML::Builder) query_params["v"] = self.id xml.element("entry") do @@ -7,22 +22,22 @@ struct SearchVideo xml.element("yt:videoId") { xml.text self.id } xml.element("yt:channelId") { xml.text self.ucid } xml.element("title") { xml.text self.title } - xml.element("link", rel: "alternate", href: "#{host_url}/watch?#{query_params}") + xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}") xml.element("author") do if auto_generated xml.element("name") { xml.text self.author } - xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" } else xml.element("name") { xml.text author } - xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" } end end xml.element("content", type: "xhtml") do xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("a", href: "#{host_url}/watch?#{query_params}") do - xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg") + xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do + xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") end xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) } @@ -33,7 +48,7 @@ struct SearchVideo xml.element("media:group") do xml.element("media:title") { xml.text self.title } - xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg", + xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", width: "320", height: "180") xml.element("media:description") { xml.text html_to_content(self.description_html) } end @@ -44,17 +59,17 @@ struct SearchVideo end end - def to_xml(host_url, auto_generated, query_params, xml : XML::Builder | Nil = nil) + def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil) if xml - to_xml(host_url, auto_generated, query_params, xml) + to_xml(HOST_URL, auto_generated, query_params, xml) else XML.build do |json| - to_xml(host_url, auto_generated, query_params, xml) + to_xml(HOST_URL, auto_generated, query_params, xml) end end end - def to_json(locale, config, kemal_config, json : JSON::Builder) + def to_json(locale, json : JSON::Builder) json.object do json.field "type", "video" json.field "title", self.title @@ -65,7 +80,7 @@ struct SearchVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "videoThumbnails" do - generate_thumbnails(json, self.id, config, kemal_config) + generate_thumbnails(json, self.id) end json.field "description", html_to_content(self.description_html) @@ -78,45 +93,49 @@ struct SearchVideo json.field "liveNow", self.live_now json.field "paid", self.paid json.field "premium", self.premium + json.field "isUpcoming", self.is_upcoming + + if self.premiere_timestamp + json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix + end end end - def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil) + def to_json(locale, json : JSON::Builder | Nil = nil) if json - to_json(locale, config, kemal_config, json) + to_json(locale, json) else JSON.build do |json| - to_json(locale, config, kemal_config, json) + to_json(locale, json) end end end - db_mapping({ - title: String, - id: String, - author: String, - ucid: String, - published: Time, - views: Int64, - description_html: String, - length_seconds: Int32, - live_now: Bool, - paid: Bool, - premium: Bool, - premiere_timestamp: Time?, - }) + def is_upcoming + premiere_timestamp ? true : false + end end struct SearchPlaylistVideo - db_mapping({ - title: String, - id: String, - length_seconds: Int32, - }) + include DB::Serializable + + property title : String + property id : String + property length_seconds : Int32 end struct SearchPlaylist - def to_json(locale, config, kemal_config, json : JSON::Builder) + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property video_count : Int32 + property videos : Array(SearchPlaylistVideo) + property thumbnail : String? + + def to_json(locale, json : JSON::Builder) json.object do json.field "type", "playlist" json.field "title", self.title @@ -137,7 +156,7 @@ struct SearchPlaylist json.field "lengthSeconds", video.length_seconds json.field "videoThumbnails" do - generate_thumbnails(json, video.id, config, Kemal.config) + generate_thumbnails(json, video.id) end end end @@ -146,29 +165,29 @@ struct SearchPlaylist end end - def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil) + def to_json(locale, json : JSON::Builder | Nil = nil) if json - to_json(locale, config, kemal_config, json) + to_json(locale, json) else JSON.build do |json| - to_json(locale, config, kemal_config, json) + to_json(locale, json) end end end - - db_mapping({ - title: String, - id: String, - author: String, - ucid: String, - video_count: Int32, - videos: Array(SearchPlaylistVideo), - thumbnail: String?, - }) end struct SearchChannel - def to_json(locale, config, kemal_config, json : JSON::Builder) + include DB::Serializable + + property author : String + property ucid : String + property author_thumbnail : String + property subscriber_count : Int32 + property video_count : Int32 + property description_html : String + property auto_generated : Bool + + def to_json(locale, json : JSON::Builder) json.object do json.field "type", "channel" json.field "author", self.author @@ -198,85 +217,50 @@ struct SearchChannel end end - def to_json(locale, config, kemal_config, json : JSON::Builder | Nil = nil) + def to_json(locale, json : JSON::Builder | Nil = nil) if json - to_json(locale, config, kemal_config, json) + to_json(locale, json) else JSON.build do |json| - to_json(locale, config, kemal_config, json) + to_json(locale, json) end end end - - db_mapping({ - author: String, - ucid: String, - author_thumbnail: String, - subscriber_count: Int32, - video_count: Int32, - description_html: String, - auto_generated: Bool, - }) end alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist def channel_search(query, page, channel) - response = YT_POOL.client &.get("/channel/#{channel}?disable_polymer=1&hl=en&gl=US") - document = XML.parse_html(response.body) - canonical = document.xpath_node(%q(//link[@rel="canonical"])) - - if !canonical - response = YT_POOL.client &.get("/c/#{channel}?disable_polymer=1&hl=en&gl=US") - document = XML.parse_html(response.body) - canonical = document.xpath_node(%q(//link[@rel="canonical"])) - end + response = YT_POOL.client &.get("/channel/#{channel}?hl=en&gl=US") + response = YT_POOL.client &.get("/user/#{channel}?hl=en&gl=US") if response.headers["location"]? + response = YT_POOL.client &.get("/c/#{channel}?hl=en&gl=US") if response.headers["location"]? - if !canonical - response = YT_POOL.client &.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US") - document = XML.parse_html(response.body) - canonical = document.xpath_node(%q(//link[@rel="canonical"])) - end + ucid = response.body.match(/\\"channelId\\":\\"(?<ucid>[^\\]+)\\"/).try &.["ucid"]? - if !canonical - return 0, [] of SearchItem - end - - ucid = canonical["href"].split("/")[-1] + return 0, [] of SearchItem if !ucid url = produce_channel_search_url(ucid, query, page) response = YT_POOL.client &.get(url) - json = JSON.parse(response.body) - - if json["content_html"]? && !json["content_html"].as_s.empty? - document = XML.parse_html(json["content_html"].as_s) - nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])) - - count = nodeset.size - items = extract_items(nodeset) - else - count = 0 - items = [] of SearchItem - end + initial_data = JSON.parse(response.body).as_a.find &.["response"]? + return 0, [] of SearchItem if !initial_data + author = initial_data["response"]?.try &.["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s + items = extract_items(initial_data.as_h, author, ucid) - return count, items + return items.size, items end def search(query, page = 1, search_params = produce_search_params(content_type: "all"), region = nil) - if query.empty? - return {0, [] of SearchItem} - end + return 0, [] of SearchItem if query.empty? - html = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body) - if html.empty? - return {0, [] of SearchItem} - end + body = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en").body) + return 0, [] of SearchItem if body.empty? + + initial_data = extract_initial_data(body) + items = extract_items(initial_data) - html = XML.parse_html(html) - nodeset = html.xpath_nodes(%q(//ol[@class="item-section"]/li)) - items = extract_items(nodeset) + # initial_data["estimatedResults"]?.try &.as_s.to_i64 - return {nodeset.size, items} + return items.size, items end def produce_search_params(sort : String = "relevance", date : String = "", content_type : String = "", @@ -310,6 +294,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte object["2:embedded"].as(Hash)["1:varint"] = 4_i64 when "year" object["2:embedded"].as(Hash)["1:varint"] = 5_i64 + else nil # Ignore end case content_type @@ -334,6 +319,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte object["2:embedded"].as(Hash)["3:varint"] = 1_i64 when "long" object["2:embedded"].as(Hash)["3:varint"] = 2_i64 + else nil # Ignore end features.each do |feature| @@ -358,6 +344,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte object["2:embedded"].as(Hash)["23:varint"] = 1_i64 when "hdr" object["2:embedded"].as(Hash)["25:varint"] = 1_i64 + else nil # Ignore end end @@ -379,12 +366,9 @@ def produce_channel_search_url(ucid, query, page) "2:string" => ucid, "3:base64" => { "2:string" => "search", - "6:varint" => 2_i64, "7:varint" => 1_i64, - "12:varint" => 1_i64, - "13:string" => "", - "23:varint" => 0_i64, "15:string" => "#{page}", + "23:varint" => 0_i64, }, "11:string" => query, }, diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 3a9c6935..8d078387 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -1,7 +1,4 @@ def fetch_trending(trending_type, region, locale) - headers = HTTP::Headers.new - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" - region ||= "US" region = region.upcase @@ -11,7 +8,7 @@ def fetch_trending(trending_type, region, locale) if trending_type && trending_type != "Default" trending_type = trending_type.downcase.capitalize - response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en", headers).body + response = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body initial_data = extract_initial_data(response) @@ -21,51 +18,28 @@ def fetch_trending(trending_type, region, locale) if url url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"] url = url["channelListSubMenuAvatarRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s - url += "&disable_polymer=1&gl=#{region}&hl=en" + url = "#{url}&gl=#{region}&hl=en" trending = YT_POOL.client &.get(url).body plid = extract_plid(url) else - trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body + trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body end else - trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en&disable_polymer=1").body + trending = YT_POOL.client &.get("/feed/trending?gl=#{region}&hl=en").body end - trending = XML.parse_html(trending) - nodeset = trending.xpath_nodes(%q(//ul/li[@class="expanded-shelf-content-item-wrapper"])) - trending = extract_videos(nodeset) + initial_data = extract_initial_data(trending) + trending = extract_videos(initial_data) return {trending, plid} end def extract_plid(url) - wrapper = HTTP::Params.parse(URI.parse(url).query.not_nil!)["bp"] - - wrapper = URI.decode_www_form(wrapper) - wrapper = Base64.decode(wrapper) - - # 0xe2 0x02 0x2e - wrapper += 3 - - # 0x0a - wrapper += 1 - - # Looks like "/m/[a-z0-9]{5}", not sure what it does here - - item_size = wrapper[0] - wrapper += 1 - item = wrapper[0, item_size] - wrapper += item.size - - # 0x12 - wrapper += 1 - - plid_size = wrapper[0] - wrapper += 1 - plid = wrapper[0, plid_size] - wrapper += plid.size - - plid = String.new(plid) - - return plid + return url.try { |i| URI.parse(i).query } + .try { |i| HTTP::Params.parse(i)["bp"] } + .try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + .try &.["44:0:embedded"]?.try &.["2:1:string"]?.try &.as_s end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index afb100f2..46bf8865 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -4,6 +4,20 @@ require "crypto/bcrypt/password" MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" } struct User + include DB::Serializable + + property updated : Time + property notifications : Array(String) + property subscriptions : Array(String) + property email : String + + @[DB::Field(converter: User::PreferencesConverter)] + property preferences : Preferences + property password : String? + property token : String + property watched : Array(String) + property feed_needs_update : Bool? + module PreferencesConverter def self.from_rs(rs) begin @@ -13,31 +27,78 @@ struct User end end end - - db_mapping({ - updated: Time, - notifications: Array(String), - subscriptions: Array(String), - email: String, - preferences: { - type: Preferences, - converter: PreferencesConverter, - }, - password: String?, - token: String, - watched: Array(String), - feed_needs_update: Bool?, - }) end struct Preferences - module ProcessString + include JSON::Serializable + include YAML::Serializable + + property annotations : Bool = CONFIG.default_user_preferences.annotations + property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed + property autoplay : Bool = CONFIG.default_user_preferences.autoplay + + @[JSON::Field(converter: Preferences::StringToArray)] + @[YAML::Field(converter: Preferences::StringToArray)] + property captions : Array(String) = CONFIG.default_user_preferences.captions + + @[JSON::Field(converter: Preferences::StringToArray)] + @[YAML::Field(converter: Preferences::StringToArray)] + property comments : Array(String) = CONFIG.default_user_preferences.comments + property continue : Bool = CONFIG.default_user_preferences.continue + property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay + + @[JSON::Field(converter: Preferences::BoolToString)] + @[YAML::Field(converter: Preferences::BoolToString)] + property dark_mode : String = CONFIG.default_user_preferences.dark_mode + property latest_only : Bool = CONFIG.default_user_preferences.latest_only + property listen : Bool = CONFIG.default_user_preferences.listen + property local : Bool = CONFIG.default_user_preferences.local + + @[JSON::Field(converter: Preferences::ProcessString)] + property locale : String = CONFIG.default_user_preferences.locale + + @[JSON::Field(converter: Preferences::ClampInt)] + property max_results : Int32 = CONFIG.default_user_preferences.max_results + property notifications_only : Bool = CONFIG.default_user_preferences.notifications_only + + @[JSON::Field(converter: Preferences::ProcessString)] + property player_style : String = CONFIG.default_user_preferences.player_style + + @[JSON::Field(converter: Preferences::ProcessString)] + property quality : String = CONFIG.default_user_preferences.quality + property default_home : String = CONFIG.default_user_preferences.default_home + property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu + property related_videos : Bool = CONFIG.default_user_preferences.related_videos + + @[JSON::Field(converter: Preferences::ProcessString)] + property sort : String = CONFIG.default_user_preferences.sort + property speed : Float32 = CONFIG.default_user_preferences.speed + property thin_mode : Bool = CONFIG.default_user_preferences.thin_mode + property unseen_only : Bool = CONFIG.default_user_preferences.unseen_only + property video_loop : Bool = CONFIG.default_user_preferences.video_loop + property volume : Int32 = CONFIG.default_user_preferences.volume + + module BoolToString def self.to_json(value : String, json : JSON::Builder) json.string value end def self.from_json(value : JSON::PullParser) : String - HTML.escape(value.read_string[0, 100]) + begin + result = value.read_string + + if result.empty? + CONFIG.default_user_preferences.dark_mode + else + result + end + rescue ex + if value.read_bool + "dark" + else + "light" + end + end end def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) @@ -45,7 +106,20 @@ struct Preferences end def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String - HTML.escape(node.value[0, 100]) + unless node.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{node.class}" + end + + case node.value + when "true" + "dark" + when "false" + "light" + when "" + CONFIG.default_user_preferences.dark_mode + else + node.value + end end end @@ -67,33 +141,130 @@ struct Preferences end end - json_mapping({ - annotations: {type: Bool, default: CONFIG.default_user_preferences.annotations}, - annotations_subscribed: {type: Bool, default: CONFIG.default_user_preferences.annotations_subscribed}, - autoplay: {type: Bool, default: CONFIG.default_user_preferences.autoplay}, - captions: {type: Array(String), default: CONFIG.default_user_preferences.captions, converter: ConfigPreferences::StringToArray}, - comments: {type: Array(String), default: CONFIG.default_user_preferences.comments, converter: ConfigPreferences::StringToArray}, - continue: {type: Bool, default: CONFIG.default_user_preferences.continue}, - continue_autoplay: {type: Bool, default: CONFIG.default_user_preferences.continue_autoplay}, - dark_mode: {type: String, default: CONFIG.default_user_preferences.dark_mode, converter: ConfigPreferences::BoolToString}, - latest_only: {type: Bool, default: CONFIG.default_user_preferences.latest_only}, - listen: {type: Bool, default: CONFIG.default_user_preferences.listen}, - local: {type: Bool, default: CONFIG.default_user_preferences.local}, - locale: {type: String, default: CONFIG.default_user_preferences.locale, converter: ProcessString}, - max_results: {type: Int32, default: CONFIG.default_user_preferences.max_results, converter: ClampInt}, - notifications_only: {type: Bool, default: CONFIG.default_user_preferences.notifications_only}, - player_style: {type: String, default: CONFIG.default_user_preferences.player_style, converter: ProcessString}, - quality: {type: String, default: CONFIG.default_user_preferences.quality, converter: ProcessString}, - default_home: {type: String, default: CONFIG.default_user_preferences.default_home}, - feed_menu: {type: Array(String), default: CONFIG.default_user_preferences.feed_menu}, - related_videos: {type: Bool, default: CONFIG.default_user_preferences.related_videos}, - sort: {type: String, default: CONFIG.default_user_preferences.sort, converter: ProcessString}, - speed: {type: Float32, default: CONFIG.default_user_preferences.speed}, - thin_mode: {type: Bool, default: CONFIG.default_user_preferences.thin_mode}, - unseen_only: {type: Bool, default: CONFIG.default_user_preferences.unseen_only}, - video_loop: {type: Bool, default: CONFIG.default_user_preferences.video_loop}, - volume: {type: Int32, default: CONFIG.default_user_preferences.volume}, - }) + module FamilyConverter + def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder) + case value + when Socket::Family::UNSPEC + yaml.scalar nil + when Socket::Family::INET + yaml.scalar "ipv4" + when Socket::Family::INET6 + yaml.scalar "ipv6" + when Socket::Family::UNIX + raise "Invalid socket family #{value}" + end + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family + if node.is_a?(YAML::Nodes::Scalar) + case node.value.downcase + when "ipv4" + Socket::Family::INET + when "ipv6" + Socket::Family::INET6 + else + Socket::Family::UNSPEC + end + else + node.raise "Expected scalar, not #{node.class}" + end + end + end + + module ProcessString + def self.to_json(value : String, json : JSON::Builder) + json.string value + end + + def self.from_json(value : JSON::PullParser) : String + HTML.escape(value.read_string[0, 100]) + end + + def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) + yaml.scalar value + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String + HTML.escape(node.value[0, 100]) + end + end + + module StringToArray + def self.to_json(value : Array(String), json : JSON::Builder) + json.array do + value.each do |element| + json.string element + end + end + end + + def self.from_json(value : JSON::PullParser) : Array(String) + begin + result = [] of String + value.read_array do + result << HTML.escape(value.read_string[0, 100]) + end + rescue ex + result = [HTML.escape(value.read_string[0, 100]), ""] + end + + result + end + + def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder) + yaml.sequence do + value.each do |element| + yaml.scalar element + end + end + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String) + begin + unless node.is_a?(YAML::Nodes::Sequence) + node.raise "Expected sequence, not #{node.class}" + end + + result = [] of String + node.nodes.each do |item| + unless item.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{item.class}" + end + + result << HTML.escape(item.value[0, 100]) + end + rescue ex + if node.is_a?(YAML::Nodes::Scalar) + result = [HTML.escape(node.value[0, 100]), ""] + else + result = ["", ""] + end + end + + result + end + end + + module StringToCookies + def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder) + (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml) + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies + unless node.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{node.class}" + end + + cookies = HTTP::Cookies.new + node.value.split(";").each do |cookie| + next if cookie.strip.empty? + name, value = cookie.split("=", 2) + cookies << HTTP::Cookie.new(name.strip, value.strip) + end + + cookies + end + end end def get_user(sid, headers, db, refresh = true) @@ -103,8 +274,7 @@ def get_user(sid, headers, db, refresh = true) if refresh && Time.utc - user.updated > 1.minute user, sid = fetch_user(sid, headers, db) user_array = user.to_a - - user_array[4] = user_array[4].to_json + user_array[4] = user_array[4].to_json # User preferences args = arg_array(user_array) db.exec("INSERT INTO users VALUES (#{args}) \ @@ -122,8 +292,7 @@ def get_user(sid, headers, db, refresh = true) else user, sid = fetch_user(sid, headers, db) user_array = user.to_a - - user_array[4] = user_array[4].to_json + user_array[4] = user_array[4].to_json # User preferences args = arg_array(user.to_a) db.exec("INSERT INTO users VALUES (#{args}) \ @@ -166,7 +335,17 @@ def fetch_user(sid, headers, db) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new(Time.utc, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String, true) + user = User.new({ + updated: Time.utc, + notifications: [] of String, + subscriptions: channels, + email: email, + preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple), + password: nil, + token: token, + watched: [] of String, + feed_needs_update: true, + }) return user, sid end @@ -174,7 +353,17 @@ def create_user(sid, email, password) password = Crypto::Bcrypt::Password.create(password, cost: 10) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new(Time.utc, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String, true) + user = User.new({ + updated: Time.utc, + notifications: [] of String, + subscriptions: [] of String, + email: email, + preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple), + password: password.to_s, + token: token, + watched: [] of String, + feed_needs_update: true, + }) return user, sid end @@ -267,7 +456,7 @@ def subscribe_ajax(channel_id, action, env_headers) end headers = cookies.add_request_headers(headers) - if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/) + if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/) session_token = match["session_token"] headers["content-type"] = "application/x-www-form-urlencoded" @@ -281,48 +470,6 @@ def subscribe_ajax(channel_id, action, env_headers) end end -# TODO: Playlist stub, sync with YouTube for Google accounts -# def playlist_ajax(video_ids, source_playlist_id, name, privacy, action, env_headers) -# headers = HTTP::Headers.new -# headers["Cookie"] = env_headers["Cookie"] -# -# html = YT_POOL.client &.get("/view_all_playlists?disable_polymer=1", headers) -# -# cookies = HTTP::Cookies.from_headers(headers) -# html.cookies.each do |cookie| -# if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name -# if cookies[cookie.name]? -# cookies[cookie.name] = cookie -# else -# cookies << cookie -# end -# end -# end -# headers = cookies.add_request_headers(headers) -# -# if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/) -# session_token = match["session_token"] -# -# headers["content-type"] = "application/x-www-form-urlencoded" -# -# post_req = { -# video_ids: [] of String, -# source_playlist_id: "", -# n: name, -# p: privacy, -# session_token: session_token, -# } -# post_url = "/playlist_ajax?#{action}=1" -# -# response = client.post(post_url, headers, form: post_req) -# if response.status_code == 200 -# return JSON.parse(response.body)["result"]["playlistId"].as_s -# else -# return nil -# end -# end -# end - def get_subscription_feed(db, user, max_results = 40, page = 1) limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE) offset = (page - 1) * limit @@ -350,6 +497,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) notifications.sort_by! { |video| video.author } when "channel name - reverse" notifications.sort_by! { |video| video.author }.reverse! + else nil # Ignore end else if user.preferences.latest_only @@ -398,6 +546,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) videos.sort_by! { |video| video.author } when "channel name - reverse" videos.sort_by! { |video| video.author }.reverse! + else nil # Ignore end notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email, as: Array(String)) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 1c7599f8..e7751fb0 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -222,53 +222,73 @@ VIDEO_FORMATS = { } struct VideoPreferences - json_mapping({ - annotations: Bool, - autoplay: Bool, - comments: Array(String), - continue: Bool, - continue_autoplay: Bool, - controls: Bool, - listen: Bool, - local: Bool, - preferred_captions: Array(String), - player_style: String, - quality: String, - raw: Bool, - region: String?, - related_videos: Bool, - speed: (Float32 | Float64), - video_end: (Float64 | Int32), - video_loop: Bool, - video_start: (Float64 | Int32), - volume: Int32, - }) + include JSON::Serializable + + property annotations : Bool + property autoplay : Bool + property comments : Array(String) + property continue : Bool + property continue_autoplay : Bool + property controls : Bool + property listen : Bool + property local : Bool + property preferred_captions : Array(String) + property player_style : String + property quality : String + property raw : Bool + property region : String? + property related_videos : Bool + property speed : Float32 | Float64 + property video_end : Float64 | Int32 + property video_loop : Bool + property video_start : Float64 | Int32 + property volume : Int32 end struct Video - property player_json : JSON::Any? - property recommended_json : JSON::Any? + include DB::Serializable + + property id : String + + @[DB::Field(converter: Video::JSONConverter)] + property info : Hash(String, JSON::Any) + property updated : Time + + @[DB::Field(ignore: true)] + property captions : Array(Caption)? + + @[DB::Field(ignore: true)] + property adaptive_fmts : Array(Hash(String, JSON::Any))? + + @[DB::Field(ignore: true)] + property fmt_stream : Array(Hash(String, JSON::Any))? + + @[DB::Field(ignore: true)] + property description : String? - module HTTPParamConverter + module JSONConverter def self.from_rs(rs) - HTTP::Params.parse(rs.read(String)) + JSON.parse(rs.read(String)).as_h end end - def to_json(locale, config, kemal_config, decrypt_function, json : JSON::Builder) + def to_json(locale, json : JSON::Builder) json.object do json.field "type", "video" json.field "title", self.title json.field "videoId", self.id + + json.field "error", info["reason"] if info["reason"]? + json.field "videoThumbnails" do - generate_thumbnails(json, self.id, config, kemal_config) + generate_thumbnails(json, self.id) end json.field "storyboards" do - generate_storyboards(json, self.id, self.storyboards, config, kemal_config) + generate_storyboards(json, self.id, self.storyboards) end - json.field "description", html_to_content(self.description_html) + json.field "description", self.description json.field "descriptionHtml", self.description_html json.field "published", self.published.to_unix json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) @@ -307,43 +327,39 @@ struct Video json.field "lengthSeconds", self.length_seconds json.field "allowRatings", self.allow_ratings - json.field "rating", self.info["avg_rating"].to_f32 + json.field "rating", self.average_rating json.field "isListed", self.is_listed json.field "liveNow", self.live_now json.field "isUpcoming", self.is_upcoming if self.premiere_timestamp - json.field "premiereTimestamp", self.premiere_timestamp.not_nil!.to_unix + json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix end - if player_response["streamingData"]?.try &.["hlsManifestUrl"]? - host_url = make_host_url(config, kemal_config) - - hlsvp = player_response["streamingData"]["hlsManifestUrl"].as_s - hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", host_url) - + if hlsvp = self.hls_manifest_url + hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL) json.field "hlsUrl", hlsvp end - json.field "dashUrl", "#{make_host_url(config, kemal_config)}/api/manifest/dash/id/#{id}" + json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}" json.field "adaptiveFormats" do json.array do - self.adaptive_fmts(decrypt_function).each do |fmt| + self.adaptive_fmts.each do |fmt| json.object do - json.field "index", fmt["index"] - json.field "bitrate", fmt["bitrate"] - json.field "init", fmt["init"] + json.field "index", "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}" + json.field "bitrate", fmt["bitrate"].as_i.to_s + json.field "init", "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}" json.field "url", fmt["url"] - json.field "itag", fmt["itag"] - json.field "type", fmt["type"] - json.field "clen", fmt["clen"] - json.field "lmt", fmt["lmt"] - json.field "projectionType", fmt["projection_type"] + json.field "itag", fmt["itag"].as_i.to_s + json.field "type", fmt["mimeType"] + json.field "clen", fmt["contentLength"] + json.field "lmt", fmt["lastModified"] + json.field "projectionType", fmt["projectionType"] fmt_info = itag_to_metadata?(fmt["itag"]) if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.to_i || 30 + fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 json.field "fps", fps json.field "container", fmt_info["ext"] json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] @@ -369,16 +385,16 @@ struct Video json.field "formatStreams" do json.array do - self.fmt_stream(decrypt_function).each do |fmt| + self.fmt_stream.each do |fmt| json.object do json.field "url", fmt["url"] - json.field "itag", fmt["itag"] - json.field "type", fmt["type"] + json.field "itag", fmt["itag"].as_i.to_s + json.field "type", fmt["mimeType"] json.field "quality", fmt["quality"] fmt_info = itag_to_metadata?(fmt["itag"]) if fmt_info - fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.to_i || 30 + fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 json.field "fps", fps json.field "container", fmt_info["ext"] json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] @@ -416,15 +432,13 @@ struct Video json.field "recommendedVideos" do json.array do - self.info["rvs"]?.try &.split(",").each do |rv| - rv = HTTP::Params.parse(rv) - + self.related_videos.each do |rv| if rv["id"]? json.object do json.field "videoId", rv["id"] json.field "title", rv["title"] json.field "videoThumbnails" do - generate_thumbnails(json, rv["id"], config, kemal_config) + generate_thumbnails(json, rv["id"]) end json.field "author", rv["author"] @@ -437,7 +451,7 @@ struct Video qualities.each do |quality| json.object do - json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-") + json.field "url", rv["author_thumbnail"]?.try &.gsub(/s\d+-/, "s#{quality}-") json.field "width", quality json.field "height", quality end @@ -446,9 +460,9 @@ struct Video end end - json.field "lengthSeconds", rv["length_seconds"].to_i - json.field "viewCountText", rv["short_view_count_text"] - json.field "viewCount", rv["view_count"]?.try &.to_i64 + json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i + json.field "viewCountText", rv["short_view_count_text"]? + json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 end end end @@ -457,266 +471,164 @@ struct Video end end - def to_json(locale, config, kemal_config, decrypt_function, json : JSON::Builder | Nil = nil) + def to_json(locale, json : JSON::Builder | Nil = nil) if json - to_json(locale, config, kemal_config, decrypt_function, json) + to_json(locale, json) else JSON.build do |json| - to_json(locale, config, kemal_config, decrypt_function, json) + to_json(locale, json) end end end - # `description_html` is stored in DB as `description`, which can be - # quite confusing. Since it currently isn't very practical to rename - # it, we instead define a getter and setter here. - def description_html - self.description + def title + info["videoDetails"]["title"]?.try &.as_s || "" end - def description_html=(other : String) - self.description = other + def ucid + info["videoDetails"]["channelId"]?.try &.as_s || "" end - def allow_ratings - allow_ratings = player_response["videoDetails"]?.try &.["allowRatings"]?.try &.as_bool - - if allow_ratings.nil? - return true - end + def author + info["videoDetails"]["author"]?.try &.as_s || "" + end - return allow_ratings + def length_seconds : Int32 + info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["lengthSeconds"]?.try &.as_s.to_i || + info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0 end - def live_now - live_now = player_response["videoDetails"]?.try &.["isLive"]?.try &.as_bool + def views : Int64 + info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64 + end - if live_now.nil? - return false - end + def likes : Int64 + info["likes"]?.try &.as_i64 || 0_i64 + end - return live_now + def dislikes : Int64 + info["dislikes"]?.try &.as_i64 || 0_i64 end - def is_listed - is_listed = player_response["videoDetails"]?.try &.["isCrawlable"]?.try &.as_bool + def average_rating : Float64 + # (likes / (likes + dislikes) * 4 + 1) + info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0 + end - if is_listed.nil? - return true - end + def published : Time + info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location.local) } || Time.local + end - return is_listed + def published=(other : Time) + info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d")) end - def is_upcoming - is_upcoming = player_response["videoDetails"]?.try &.["isUpcoming"]?.try &.as_bool + def cookie + info["cookie"]?.try &.as_h.map { |k, v| "#{k}=#{v}" }.join("; ") || "" + end - if is_upcoming.nil? - return false - end + def allow_ratings + r = info["videoDetails"]["allowRatings"]?.try &.as_bool + r.nil? ? false : r + end - return is_upcoming + def live_now + info["videoDetails"]["isLiveContent"]?.try &.as_bool || false end - def premiere_timestamp - if self.is_upcoming - premiere_timestamp = player_response["playabilityStatus"]? - .try &.["liveStreamability"]? - .try &.["liveStreamabilityRenderer"]? - .try &.["offlineSlate"]? - .try &.["liveStreamOfflineSlateRenderer"]? - .try &.["scheduledStartTime"]?.try &.as_s.to_i64 - end + def is_listed + info["videoDetails"]["isCrawlable"]?.try &.as_bool || false + end - if premiere_timestamp - premiere_timestamp = Time.unix(premiere_timestamp) - end + def is_upcoming + info["videoDetails"]["isUpcoming"]?.try &.as_bool || false + end - return premiere_timestamp + def premiere_timestamp : Time? + info["microformat"]?.try &.["playerMicroformatRenderer"]? + .try &.["liveBroadcastDetails"]?.try &.["startTimestamp"]?.try { |t| Time.parse_rfc3339(t.as_s) } end def keywords - keywords = player_response["videoDetails"]?.try &.["keywords"]?.try &.as_a - keywords ||= [] of String - - return keywords + info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String end - def fmt_stream(decrypt_function) - streams = [] of HTTP::Params - - if fmt_streams = player_response["streamingData"]?.try &.["formats"]? - fmt_streams.as_a.each do |fmt_stream| - if !fmt_stream.as_h? - next - end - - fmt = {} of String => String + def related_videos + info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String) + end - fmt["lmt"] = fmt_stream["lastModified"]?.try &.as_s || "0" - fmt["projection_type"] = "1" - fmt["type"] = fmt_stream["mimeType"].as_s - fmt["clen"] = fmt_stream["contentLength"]?.try &.as_s || "0" - fmt["bitrate"] = fmt_stream["bitrate"]?.try &.as_i.to_s || "0" - fmt["itag"] = fmt_stream["itag"].as_i.to_s - if fmt_stream["url"]? - fmt["url"] = fmt_stream["url"].as_s - end - if fmt_stream["cipher"]? - HTTP::Params.parse(fmt_stream["cipher"].as_s).each do |key, value| - fmt[key] = value - end - end - fmt["quality"] = fmt_stream["quality"].as_s + def allowed_regions + info["microformat"]?.try &.["playerMicroformatRenderer"]? + .try &.["availableCountries"]?.try &.as_a.map &.as_s || [] of String + end - if fmt_stream["width"]? - fmt["size"] = "#{fmt_stream["width"]}x#{fmt_stream["height"]}" - fmt["height"] = fmt_stream["height"].as_i.to_s - end + def author_thumbnail : String + info["authorThumbnail"]?.try &.as_s || "" + end - if fmt_stream["fps"]? - fmt["fps"] = fmt_stream["fps"].as_i.to_s - end + def sub_count_text : String + info["subCountText"]?.try &.as_s || "-" + end - if fmt_stream["qualityLabel"]? - fmt["quality_label"] = fmt_stream["qualityLabel"].as_s - end + def fmt_stream + return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream - params = HTTP::Params.new - fmt.each do |key, value| - params[key] = value + fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) + fmt_stream.each do |fmt| + if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } + s.each do |k, v| + fmt[k] = JSON::Any.new(v) end - - streams << params + fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}") end - streams.sort_by! { |stream| stream["height"].to_i }.reverse! - elsif fmt_stream = self.info["url_encoded_fmt_stream_map"]? - fmt_stream.split(",").each do |string| - if !string.empty? - streams << HTTP::Params.parse(string) - end - end + fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") + fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? end - - streams.each { |s| s.add("label", "#{s["quality"]} - #{s["type"].split(";")[0].split("/")[1]}") } - streams = streams.uniq { |s| s["label"] } - - if self.info["region"]? - streams.each do |fmt| - fmt["url"] += "®ion=" + self.info["region"] - end - end - - streams.each do |fmt| - fmt["url"] += "&host=" + (URI.parse(fmt["url"]).host || "") - fmt["url"] += decrypt_signature(fmt, decrypt_function) - end - - return streams + fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } + @fmt_stream = fmt_stream + return @fmt_stream.as(Array(Hash(String, JSON::Any))) end - def adaptive_fmts(decrypt_function) - adaptive_fmts = [] of HTTP::Params - - if fmts = player_response["streamingData"]?.try &.["adaptiveFormats"]? - fmts.as_a.each do |adaptive_fmt| - next if !adaptive_fmt.as_h? - fmt = {} of String => String - - if init = adaptive_fmt["initRange"]? - fmt["init"] = "#{init["start"]}-#{init["end"]}" - end - fmt["init"] ||= "0-0" - - fmt["lmt"] = adaptive_fmt["lastModified"]?.try &.as_s || "0" - fmt["projection_type"] = "1" - fmt["type"] = adaptive_fmt["mimeType"].as_s - fmt["clen"] = adaptive_fmt["contentLength"]?.try &.as_s || "0" - fmt["bitrate"] = adaptive_fmt["bitrate"]?.try &.as_i.to_s || "0" - fmt["itag"] = adaptive_fmt["itag"].as_i.to_s - if adaptive_fmt["url"]? - fmt["url"] = adaptive_fmt["url"].as_s - end - if adaptive_fmt["cipher"]? - HTTP::Params.parse(adaptive_fmt["cipher"].as_s).each do |key, value| - fmt[key] = value - end - end - if index = adaptive_fmt["indexRange"]? - fmt["index"] = "#{index["start"]}-#{index["end"]}" - end - fmt["index"] ||= "0-0" - - if adaptive_fmt["width"]? - fmt["size"] = "#{adaptive_fmt["width"]}x#{adaptive_fmt["height"]}" - end - - if adaptive_fmt["fps"]? - fmt["fps"] = adaptive_fmt["fps"].as_i.to_s - end - - if adaptive_fmt["qualityLabel"]? - fmt["quality_label"] = adaptive_fmt["qualityLabel"].as_s + def adaptive_fmts + return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts + fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) + fmt_stream.each do |fmt| + if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } + s.each do |k, v| + fmt[k] = JSON::Any.new(v) end - - params = HTTP::Params.new - fmt.each do |key, value| - params[key] = value - end - - adaptive_fmts << params - end - elsif fmts = self.info["adaptive_fmts"]? - fmts.split(",") do |string| - adaptive_fmts << HTTP::Params.parse(string) + fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}") end - end - - if self.info["region"]? - adaptive_fmts.each do |fmt| - fmt["url"] += "®ion=" + self.info["region"] - end - end - adaptive_fmts.each do |fmt| - fmt["url"] += "&host=" + (URI.parse(fmt["url"]).host || "") - fmt["url"] += decrypt_signature(fmt, decrypt_function) + fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") + fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? end - - return adaptive_fmts + # See https://github.com/TeamNewPipe/NewPipe/issues/2415 + # Some streams are segmented by URL `sq/` rather than index, for now we just filter them out + fmt_stream.reject! { |f| !f["indexRange"]? } + fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } + @adaptive_fmts = fmt_stream + return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) end - def video_streams(adaptive_fmts) - video_streams = adaptive_fmts.select { |s| s["type"].starts_with? "video" } - - return video_streams + def video_streams + adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("video") end - def audio_streams(adaptive_fmts) - audio_streams = adaptive_fmts.select { |s| s["type"].starts_with? "audio" } - audio_streams.sort_by! { |s| s["bitrate"].to_i }.reverse! - audio_streams.each do |stream| - stream["bitrate"] = (stream["bitrate"].to_f64/1000).to_i.to_s - end - - return audio_streams - end - - def player_response - @player_json = JSON.parse(@info["player_response"]) if !@player_json - @player_json.not_nil! + def audio_streams + adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio") end def storyboards - storyboards = player_response["storyboards"]? + storyboards = info["storyboards"]? .try &.as_h .try &.["playerStoryboardSpecRenderer"]? .try &.["spec"]? .try &.as_s.split("|") if !storyboards - if storyboard = player_response["storyboards"]? + if storyboard = info["storyboards"]? .try &.as_h .try &.["playerLiveStoryboardSpecRenderer"]? .try &.["spec"]? @@ -744,9 +656,7 @@ struct Video storyboard_height: Int32, storyboard_count: Int32) - if !storyboards - return items - end + return items if !storyboards url = URI.parse(storyboards.shift) params = HTTP::Params.parse(url.query || "") @@ -780,220 +690,169 @@ struct Video end def paid - reason = player_response["playabilityStatus"]?.try &.["reason"]? + reason = info["playabilityStatus"]?.try &.["reason"]? paid = reason == "This video requires payment to watch." ? true : false - - return paid + paid end def premium - if info["premium"]? - self.info["premium"] == "true" - else - false + keywords.includes? "YouTube Red" + end + + def captions : Array(Caption) + return @captions.as(Array(Caption)) if @captions + captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| + caption = Caption.from_json(caption.to_json) + caption.name.simpleText = caption.name.simpleText.split(" - ")[0] + caption end + captions ||= [] of Caption + @captions = captions + return @captions.as(Array(Caption)) end - def captions - captions = [] of Caption - if player_response["captions"]? - caption_list = player_response["captions"]["playerCaptionsTracklistRenderer"]["captionTracks"]?.try &.as_a - caption_list ||= [] of JSON::Any + def description + description = info["microformat"]?.try &.["playerMicroformatRenderer"]? + .try &.["description"]?.try &.["simpleText"]?.try &.as_s || "" + end - caption_list.each do |caption| - caption = Caption.from_json(caption.to_json) - caption.name.simpleText = caption.name.simpleText.split(" - ")[0] - captions << caption - end - end + # TODO + def description=(value : String) + @description = value + end - return captions + def description_html + info["descriptionHtml"]?.try &.as_s || "<p></p>" end - def short_description - short_description = self.description_html.gsub(/(<br>)|(<br\/>|"|\n)/, { - "<br>": " ", - "<br/>": " ", - "\"": """, - "\n": " ", - }) - short_description = XML.parse_html(short_description).content[0..200].strip(" ") - - if short_description.empty? - short_description = " " - end + def description_html=(value : String) + info["descriptionHtml"] = JSON::Any.new(value) + end - return short_description - end - - def length_seconds - player_response["videoDetails"]["lengthSeconds"].as_s.to_i - end - - db_mapping({ - id: String, - info: { - type: HTTP::Params, - default: HTTP::Params.parse(""), - converter: Video::HTTPParamConverter, - }, - updated: Time, - title: String, - views: Int64, - likes: Int32, - dislikes: Int32, - wilson_score: Float64, - published: Time, - description: String, - language: String?, - author: String, - ucid: String, - allowed_regions: Array(String), - is_family_friendly: Bool, - genre: String, - genre_url: String, - license: String, - sub_count_text: String, - author_thumbnail: String, - }) -end + def short_description + info["shortDescription"]?.try &.as_s? || "" + end -struct Caption - json_mapping({ - name: CaptionName, - baseUrl: String, - languageCode: String, - }) -end + def hls_manifest_url : String? + info["streamingData"]?.try &.["hlsManifestUrl"]?.try &.as_s + end -struct CaptionName - json_mapping({ - simpleText: String, - }) -end + def dash_manifest_url + info["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s + end -class VideoRedirect < Exception - property video_id : String + def genre : String + info["genre"]?.try &.as_s || "" + end - def initialize(@video_id) + def genre_url : String? + info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil end -end -def get_video(id, db, refresh = true, region = nil, force_refresh = false) - if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region - # If record was last updated over 10 minutes ago, or video has since premiered, - # refresh (expire param in response lasts for 6 hours) - if (refresh && - (Time.utc - video.updated > 10.minutes) || - (video.premiere_timestamp && video.premiere_timestamp.as(Time) < Time.utc)) || - force_refresh - begin - video = fetch_video(id, region) - video_array = video.to_a + def license : String? + info["license"]?.try &.as_s + end - args = arg_array(video_array[1..-1], 2) + def is_family_friendly : Bool + info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false + end - db.exec("UPDATE videos SET (info,updated,title,views,likes,dislikes,wilson_score,\ - published,description,language,author,ucid,allowed_regions,is_family_friendly,\ - genre,genre_url,license,sub_count_text,author_thumbnail)\ - = (#{args}) WHERE id = $1", args: video_array) - rescue ex - db.exec("DELETE FROM videos * WHERE id = $1", id) - raise ex - end - end - else - video = fetch_video(id, region) - video_array = video.to_a + def wilson_score : Float64 + ci_lower_bound(likes, likes + dislikes).round(4) + end - args = arg_array(video_array) + def engagement : Float64 + ((likes + dislikes) / views).round(4) + end - if !region - db.exec("INSERT INTO videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", args: video_array) - end + def reason : String? + info["reason"]?.try &.as_s end - return video + def session_token : String? + info["sessionToken"]?.try &.as_s? + end end -def extract_recommended(recommended_videos) - rvs = [] of HTTP::Params +struct CaptionName + include JSON::Serializable - recommended_videos.try &.each do |compact_renderer| - if compact_renderer["compactRadioRenderer"]? || compact_renderer["compactPlaylistRenderer"]? - # TODO - elsif video_renderer = compact_renderer["compactVideoRenderer"]? - recommended_video = HTTP::Params.new - recommended_video["id"] = video_renderer["videoId"].as_s - recommended_video["title"] = video_renderer["title"]["simpleText"].as_s + property simpleText : String +end - next if !video_renderer["shortBylineText"]? +struct Caption + include JSON::Serializable - recommended_video["author"] = video_renderer["shortBylineText"]["runs"].as_a[0]["text"].as_s - recommended_video["ucid"] = video_renderer["shortBylineText"]["runs"].as_a[0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s - recommended_video["author_thumbnail"] = video_renderer["channelThumbnail"]["thumbnails"][0]["url"].as_s + property name : CaptionName + property baseUrl : String + property languageCode : String +end - if view_count = video_renderer["viewCountText"]?.try { |field| field["simpleText"]?.try &.as_s || field["runs"][0]?.try &.["text"].as_s }.try &.delete(", views watching").to_i64?.try &.to_s - recommended_video["view_count"] = view_count - recommended_video["short_view_count_text"] = "#{number_to_short_text(view_count.to_i64)} views" - end - recommended_video["length_seconds"] = decode_length_seconds(video_renderer["lengthText"]?.try &.["simpleText"]?.try &.as_s || "0:00").to_s +class VideoRedirect < Exception + property video_id : String - rvs << recommended_video - end + def initialize(@video_id) end - - rvs end -def extract_polymer_config(body, html) - params = HTTP::Params.new - - params["session_token"] = body.match(/"XSRF_TOKEN":"(?<session_token>[A-Za-z0-9\_\-\=]+)"/).try &.["session_token"] || "" - - html_info = JSON.parse(body.match(/ytplayer\.config = (?<info>.*?);ytplayer\.load/).try &.["info"] || "{}").try &.["args"]?.try &.as_h +def parse_related(r : JSON::Any) : JSON::Any? + # TODO: r["endScreenPlaylistRenderer"], etc. + return if !r["endScreenVideoRenderer"]? + r = r["endScreenVideoRenderer"].as_h + + return if !r["lengthInSeconds"]? + + rv = {} of String => JSON::Any + rv["author"] = r["shortBylineText"]["runs"][0]?.try &.["text"] || JSON::Any.new("") + rv["ucid"] = r["shortBylineText"]["runs"][0]?.try &.["navigationEndpoint"]["browseEndpoint"]["browseId"] || JSON::Any.new("") + rv["author_url"] = JSON::Any.new("/channel/#{rv["ucid"]}") + rv["length_seconds"] = JSON::Any.new(r["lengthInSeconds"].as_i.to_s) + rv["title"] = r["title"]["simpleText"] + rv["short_view_count_text"] = JSON::Any.new(r["shortViewCountText"]?.try &.["simpleText"]?.try &.as_s || "") + rv["view_count"] = JSON::Any.new(r["title"]["accessibility"]?.try &.["accessibilityData"]["label"].as_s.match(/(?<views>[1-9](\d+,?)*) views/).try &.["views"].gsub(/\D/, "") || "") + rv["id"] = r["videoId"] + JSON::Any.new(rv) +end - if html_info - html_info.each do |key, value| - params[key] = value.to_s - end +def extract_polymer_config(body) + params = {} of String => JSON::Any + player_response = body.match(/window\["ytInitialPlayerResponse"\]\s*=\s*(?<info>.*?);\n/) + .try { |r| JSON.parse(r["info"]).as_h } + + if body.includes?("To continue with your YouTube experience, please fill out the form below.") || + body.includes?("https://www.google.com/sorry/index") + params["reason"] = JSON::Any.new("Could not extract video info. Instance is likely blocked.") + elsif !player_response + params["reason"] = JSON::Any.new("Video unavailable.") + elsif player_response["playabilityStatus"]?.try &.["status"]?.try &.as_s != "OK" + reason = player_response["playabilityStatus"]["errorScreen"]?.try &.["playerErrorMessageRenderer"]?.try &.["subreason"]?.try { |s| s["simpleText"]?.try &.as_s || s["runs"].as_a.map { |r| r["text"] }.join("") } || + player_response["playabilityStatus"]["reason"].as_s + params["reason"] = JSON::Any.new(reason) end - initial_data = extract_initial_data(body) - - primary_results = initial_data["contents"]? - .try &.["twoColumnWatchNextResults"]? - .try &.["results"]? - .try &.["results"]? - .try &.["contents"]? + params["sessionToken"] = JSON::Any.new(body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]?) + params["shortDescription"] = JSON::Any.new(body.match(/"og:description" content="(?<description>[^"]+)"/).try &.["description"]?) - comment_continuation = primary_results.try &.as_a.select { |object| object["itemSectionRenderer"]? }[0]? - .try &.["itemSectionRenderer"]? - .try &.["continuations"]? - .try &.[0]? - .try &.["nextContinuationData"]? + return params if !player_response - params["ctoken"] = comment_continuation.try &.["continuation"]?.try &.as_s || "" - params["itct"] = comment_continuation.try &.["clickTrackingParams"]?.try &.as_s || "" + {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| + params[f] = player_response[f] if player_response[f]? + end - rvs = initial_data["contents"]? - .try &.["twoColumnWatchNextResults"]? - .try &.["secondaryResults"]? - .try &.["secondaryResults"]? - .try &.["results"]? - .try &.as_a + yt_initial_data = body.match(/window\["ytInitialData"\]\s*=\s*(?<info>.*?);\n/) + .try { |r| JSON.parse(r["info"]).as_h } - params["rvs"] = extract_recommended(rvs).join(",") - - # TODO: Watching now - params["views"] = primary_results.try &.as_a.select { |object| object["videoPrimaryInfoRenderer"]? }[0]? - .try &.["videoPrimaryInfoRenderer"]? - .try &.["viewCount"]? - .try &.["videoViewCountRenderer"]? - .try &.["viewCount"]? - .try &.["simpleText"]? - .try &.as_s.gsub(/\D/, "").to_i64.to_s || "0" + params["relatedVideos"] = yt_initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]? + .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r| + parse_related r + }.try { |a| JSON::Any.new(a) } || yt_initial_data.try &.["webWatchNextResponseExtensionData"]?.try &.["relatedVideoArgs"]? + .try &.as_s.split(",").map { |r| + r = HTTP::Params.parse(r).to_h + JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) })) + }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) + primary_results = yt_initial_data.try &.["contents"]?.try &.["twoColumnWatchNextResults"]?.try &.["results"]? + .try &.["results"]?.try &.["contents"]? sentiment_bar = primary_results.try &.as_a.select { |object| object["videoPrimaryInfoRenderer"]? }[0]? .try &.["videoPrimaryInfoRenderer"]? .try &.["sentimentBar"]? @@ -1001,34 +860,13 @@ def extract_polymer_config(body, html) .try &.["tooltip"]? .try &.as_s - likes, dislikes = sentiment_bar.try &.split(" / ").map { |a| a.delete(", ").to_i32 }[0, 2] || {0, 0} - - params["likes"] = "#{likes}" - params["dislikes"] = "#{dislikes}" - - published = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? - .try &.["videoSecondaryInfoRenderer"]? - .try &.["dateText"]? - .try &.["simpleText"]? - .try &.as_s.split(" ")[-3..-1].join(" ") - - if published - params["published"] = Time.parse(published, "%b %-d, %Y", Time::Location.local).to_unix.to_s - else - params["published"] = Time.utc(1990, 1, 1).to_unix.to_s - end - - params["description_html"] = "<p></p>" + likes, dislikes = sentiment_bar.try &.split(" / ", 2).map &.gsub(/\D/, "").to_i64 || {0_i64, 0_i64} + params["likes"] = JSON::Any.new(likes) + params["dislikes"] = JSON::Any.new(dislikes) - description_html = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? - .try &.["videoSecondaryInfoRenderer"]? - .try &.["description"]? - .try &.["runs"]? - .try &.as_a - - if description_html - params["description_html"] = content_to_comment_html(description_html) - end + params["descriptionHtml"] = JSON::Any.new(primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? + .try &.["videoSecondaryInfoRenderer"]?.try &.["description"]?.try &.["runs"]? + .try &.as_a.try { |t| content_to_comment_html(t).gsub("\n", "<br/>") } || "<p></p>") metadata = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? .try &.["videoSecondaryInfoRenderer"]? @@ -1037,9 +875,8 @@ def extract_polymer_config(body, html) .try &.["rows"]? .try &.as_a - params["genre"] = "" - params["genre_ucid"] = "" - params["license"] = "" + params["genre"] = params["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["category"]? || JSON::Any.new("") + params["genreUrl"] = JSON::Any.new(nil) metadata.try &.each do |row| title = row["metadataRowRenderer"]?.try &.["title"]?.try &.["simpleText"]?.try &.as_s @@ -1051,219 +888,130 @@ def extract_polymer_config(body, html) contents = contents.try &.["runs"]? .try &.as_a[0]? - params["genre"] = contents.try &.["text"]? - .try &.as_s || "" - params["genre_ucid"] = contents.try &.["navigationEndpoint"]? - .try &.["browseEndpoint"]? - .try &.["browseId"]?.try &.as_s || "" + params["genre"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "") + params["genreUcid"] = JSON::Any.new(contents.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]? + .try &.["browseId"]?.try &.as_s || "") elsif title.try &.== "License" contents = contents.try &.["runs"]? .try &.as_a[0]? - params["license"] = contents.try &.["text"]? - .try &.as_s || "" + params["license"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "") elsif title.try &.== "Licensed to YouTube by" - params["license"] = contents.try &.["simpleText"]? - .try &.as_s || "" + params["license"] = JSON::Any.new(contents.try &.["simpleText"]?.try &.as_s || "") end end author_info = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? - .try &.["videoSecondaryInfoRenderer"]? - .try &.["owner"]? - .try &.["videoOwnerRenderer"]? + .try &.["videoSecondaryInfoRenderer"]?.try &.["owner"]?.try &.["videoOwnerRenderer"]? - params["author_thumbnail"] = author_info.try &.["thumbnail"]? - .try &.["thumbnails"]? - .try &.as_a[0]? - .try &.["url"]? - .try &.as_s || "" + params["authorThumbnail"] = JSON::Any.new(author_info.try &.["thumbnail"]? + .try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"]? + .try &.as_s || "") - params["sub_count_text"] = author_info.try &.["subscriberCountText"]? - .try &.["simpleText"]? - .try &.as_s.gsub(/\D/, "") || "0" + params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]? + .try { |t| t["simpleText"]? || t["runs"]?.try &.[0]?.try &.["text"]? }.try &.as_s.split(" ", 2)[0] || "-") - return params -end + initial_data = body.match(/ytplayer\.config\s*=\s*(?<info>.*?);ytplayer\.web_player_context_config/) + .try { |r| JSON.parse(r["info"]) }.try &.["args"]["player_response"]? + .try &.as_s?.try &.try { |r| JSON.parse(r).as_h } -def extract_player_config(body, html) - params = HTTP::Params.new + return params if !initial_data - if md = body.match(/'XSRF_TOKEN': "(?<session_token>[A-Za-z0-9\_\-\=]+)"/) - params["session_token"] = md["session_token"] + {"playabilityStatus", "streamingData"}.each do |f| + params[f] = initial_data[f] if initial_data[f]? end - if md = body.match(/'RELATED_PLAYER_ARGS': (?<json>.*?),\n/) - recommended_json = JSON.parse(md["json"]) - rvs_params = recommended_json["rvs"].as_s.split(",").map { |params| HTTP::Params.parse(params) } - - if watch_next_response = recommended_json["watch_next_response"]? - watch_next_json = JSON.parse(watch_next_response.as_s) - rvs = watch_next_json["contents"]? - .try &.["twoColumnWatchNextResults"]? - .try &.["secondaryResults"]? - .try &.["secondaryResults"]? - .try &.["results"]? - .try &.as_a - - rvs = extract_recommended(rvs).compact_map do |rv| - if !rv["short_view_count_text"]? - rv_params = rvs_params.select { |rv_params| rv_params["id"]? == (rv["id"]? || "") }[0]? + params +end - if rv_params.try &.["short_view_count_text"]? - rv["short_view_count_text"] = rv_params.not_nil!["short_view_count_text"] - rv - else - nil - end - else - rv - end +def get_video(id, db, refresh = true, region = nil, force_refresh = false) + if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region + # If record was last updated over 10 minutes ago, or video has since premiered, + # refresh (expire param in response lasts for 6 hours) + if (refresh && + (Time.utc - video.updated > 10.minutes) || + (video.premiere_timestamp.try &.< Time.utc)) || + force_refresh + begin + video = fetch_video(id, region) + db.exec("UPDATE videos SET (id, info, updated) = ($1, $2, $3) WHERE id = $1", video.id, video.info.to_json, video.updated) + rescue ex + db.exec("DELETE FROM videos * WHERE id = $1", id) + raise ex end - params["rvs"] = (rvs.map &.to_s).join(",") - end - end - - html_info = body.match(/ytplayer\.config = (?<info>.*?);ytplayer\.load/).try &.["info"] - - if html_info - JSON.parse(html_info)["args"].as_h.each do |key, value| - params[key] = value.to_s end else - error_message = html.xpath_node(%q(//h1[@id="unavailable-message"])) - if error_message - params["reason"] = error_message.content.strip - elsif body.includes?("To continue with your YouTube experience, please fill out the form below.") || - body.includes?("https://www.google.com/sorry/index") - params["reason"] = "Could not extract video info. Instance is likely blocked." - else - params["reason"] = "Video unavailable." + video = fetch_video(id, region) + if !region + db.exec("INSERT INTO videos VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", video.id, video.info.to_json, video.updated) end end - return params + return video end def fetch_video(id, region) - response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")) + response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999")) if md = response.headers["location"]?.try &.match(/v=(?<id>[a-zA-Z0-9_-]{11})/) raise VideoRedirect.new(video_id: md["id"]) end - html = XML.parse_html(response.body) - info = extract_player_config(response.body, html) - info["cookie"] = response.cookies.to_h.map { |name, cookie| "#{name}=#{cookie.value}" }.join("; ") - - allowed_regions = html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).try &.["content"].split(",") - if !allowed_regions || allowed_regions == [""] - allowed_regions = [] of String - end + info = extract_polymer_config(response.body) + info["cookie"] = JSON::Any.new(response.cookies.to_h.transform_values { |v| JSON::Any.new(v.value) }) + allowed_regions = info["microformat"]?.try &.["playerMicroformatRenderer"]["availableCountries"]?.try &.as_a.map &.as_s || [] of String # Check for region-blocks - if info["reason"]? && info["reason"].includes?("your country") + if info["reason"]?.try &.as_s.includes?("your country") bypass_regions = PROXY_LIST.keys & allowed_regions if !bypass_regions.empty? region = bypass_regions[rand(bypass_regions.size)] - response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")) - - html = XML.parse_html(response.body) - info = extract_player_config(response.body, html) + response = YT_POOL.client(region, &.get("/watch?v=#{id}&gl=US&hl=en&has_verified=1&bpctr=9999999999")) - info["region"] = region if region - info["cookie"] = response.cookies.to_h.map { |name, cookie| "#{name}=#{cookie.value}" }.join("; ") + region_info = extract_polymer_config(response.body) + region_info["region"] = JSON::Any.new(region) if region + region_info["cookie"] = JSON::Any.new(response.cookies.to_h.transform_values { |v| JSON::Any.new(v.value) }) + info = region_info if !region_info["reason"]? end end # Try to pull streams from embed URL if info["reason"]? embed_page = YT_POOL.client &.get("/embed/#{id}").body - sts = embed_page.match(/"sts"\s*:\s*(?<sts>\d+)/).try &.["sts"]? - sts ||= "" - embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&disable_polymer=1&sts=#{sts}").body) + sts = embed_page.match(/"sts"\s*:\s*(?<sts>\d+)/).try &.["sts"]? || "" + embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?html5=1&video_id=#{id}&eurl=https://youtube.googleapis.com/v/#{id}&gl=US&hl=en&sts=#{sts}").body) - if !embed_info["reason"]? - embed_info.each do |key, value| - info[key] = value.to_s + if embed_info["player_response"]? + player_response = JSON.parse(embed_info["player_response"]) + {"captions", "microformat", "playabilityStatus", "streamingData", "videoDetails", "storyboards"}.each do |f| + info[f] = player_response[f] if player_response[f]? end - else - raise info["reason"] end - end - if info["reason"]? && !info["player_response"]? - raise info["reason"] - end + initial_data = JSON.parse(embed_info["watch_next_response"]) if embed_info["watch_next_response"]? - player_json = JSON.parse(info["player_response"]) - if reason = player_json["playabilityStatus"]?.try &.["reason"]?.try &.as_s - raise reason + info["relatedVideos"] = initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]? + .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r| + parse_related r + }.try { |a| JSON::Any.new(a) } || embed_info["rvs"]?.try &.split(",").map { |r| + r = HTTP::Params.parse(r).to_h + JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) })) + }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) end - title = player_json["videoDetails"]["title"].as_s - author = player_json["videoDetails"]["author"]?.try &.as_s || "" - ucid = player_json["videoDetails"]["channelId"]?.try &.as_s || "" - - info["premium"] = html.xpath_node(%q(.//span[text()="Premium"])) ? "true" : "false" - - views = html.xpath_node(%q(//meta[@itemprop="interactionCount"])) - .try &.["content"].to_i64? || 0_i64 + raise info["reason"]?.try &.as_s || "" if !info["videoDetails"]? - likes = html.xpath_node(%q(//button[@title="I like this"]/span)) - .try &.content.delete(",").try &.to_i? || 0 - - dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span)) - .try &.content.delete(",").try &.to_i? || 0 - - avg_rating = (likes.to_f/(likes.to_f + dislikes.to_f) * 4 + 1) - avg_rating = avg_rating.nan? ? 0.0 : avg_rating - info["avg_rating"] = "#{avg_rating}" - - description_html = html.xpath_node(%q(//p[@id="eow-description"])).try &.to_xml(options: XML::SaveOptions::NO_DECL) || "<p></p>" - wilson_score = ci_lower_bound(likes, likes + dislikes) - - published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).try &.["content"] - published ||= Time.utc.to_s("%Y-%m-%d") - published = Time.parse(published, "%Y-%m-%d", Time::Location.local) - - is_family_friendly = html.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).try &.["content"] == "True" - is_family_friendly ||= true - - genre = html.xpath_node(%q(//meta[@itemprop="genre"])).try &.["content"] - genre ||= "" - - genre_url = html.xpath_node(%(//ul[contains(@class, "watch-info-tag-list")]/li/a[text()="#{genre}"])).try &.["href"]? - genre_url ||= "" - - # YouTube provides invalid URLs for some genres, so we fix that here - case genre - when "Comedy" - genre_url = "/channel/UCQZ43c4dAA9eXCQuXWu9aTw" - when "Education" - genre_url = "/channel/UCdxpofrI-dO6oYfsqHDHphw" - when "Gaming" - genre_url = "/channel/UCOpNcN46UbXVtpKMrmU4Abg" - when "Movies" - genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g" - when "Nonprofits & Activism" - genre_url = "/channel/UCfFyYRYslvuhwMDnx6KjUvw" - when "Trailers" - genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g" - end - - license = html.xpath_node(%q(//h4[contains(text(),"License")]/parent::*/ul/li)).try &.content || "" - sub_count_text = html.xpath_node(%q(//span[contains(@class, "subscriber-count")])).try &.["title"]? || "0" - author_thumbnail = html.xpath_node(%(//span[@class="yt-thumb-clip"]/img)).try &.["data-thumb"]?.try &.gsub(/^\/\//, "https://") || "" - - video = Video.new(id, info, Time.utc, title, views, likes, dislikes, wilson_score, published, description_html, - nil, author, ucid, allowed_regions, is_family_friendly, genre, genre_url, license, sub_count_text, author_thumbnail) + video = Video.new({ + id: id, + info: info, + updated: Time.utc, + }) return video end -def itag_to_metadata?(itag : String) - return VIDEO_FORMATS[itag]? +def itag_to_metadata?(itag : JSON::Any) + return VIDEO_FORMATS[itag.to_s]? end def process_continuation(db, query, plid, id) @@ -1365,34 +1113,34 @@ def process_video_params(query, preferences) controls ||= 1 controls = controls >= 1 - params = VideoPreferences.new( - annotations: annotations, - autoplay: autoplay, - comments: comments, - continue: continue, - continue_autoplay: continue_autoplay, - controls: controls, - listen: listen, - local: local, - player_style: player_style, + params = VideoPreferences.new({ + annotations: annotations, + autoplay: autoplay, + comments: comments, + continue: continue, + continue_autoplay: continue_autoplay, + controls: controls, + listen: listen, + local: local, + player_style: player_style, preferred_captions: preferred_captions, - quality: quality, - raw: raw, - region: region, - related_videos: related_videos, - speed: speed, - video_end: video_end, - video_loop: video_loop, - video_start: video_start, - volume: volume, - ) + quality: quality, + raw: raw, + region: region, + related_videos: related_videos, + speed: speed, + video_end: video_end, + video_loop: video_loop, + video_start: video_start, + volume: volume, + }) return params end -def build_thumbnails(id, config, kemal_config) +def build_thumbnails(id) return { - {name: "maxres", host: "#{make_host_url(config, kemal_config)}", url: "maxres", height: 720, width: 1280}, + {name: "maxres", host: "#{HOST_URL}", url: "maxres", height: 720, width: 1280}, {name: "maxresdefault", host: "https://i.ytimg.com", url: "maxresdefault", height: 720, width: 1280}, {name: "sddefault", host: "https://i.ytimg.com", url: "sddefault", height: 480, width: 640}, {name: "high", host: "https://i.ytimg.com", url: "hqdefault", height: 360, width: 480}, @@ -1404,9 +1152,9 @@ def build_thumbnails(id, config, kemal_config) } end -def generate_thumbnails(json, id, config, kemal_config) +def generate_thumbnails(json, id) json.array do - build_thumbnails(id, config, kemal_config).each do |thumbnail| + build_thumbnails(id).each do |thumbnail| json.object do json.field "quality", thumbnail[:name] json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" @@ -1417,7 +1165,7 @@ def generate_thumbnails(json, id, config, kemal_config) end end -def generate_storyboards(json, id, storyboards, config, kemal_config) +def generate_storyboards(json, id, storyboards) json.array do storyboards.each do |storyboard| json.object do diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr index f1899faa..09eacbc8 100644 --- a/src/invidious/views/add_playlist_items.ecr +++ b/src/invidious/views/add_playlist_items.ecr @@ -20,12 +20,14 @@ <div class="pure-u-1 pure-u-lg-1-5"></div> </div> -<script> -var playlist_data = { - csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>', -} +<script id="playlist_data" type="application/json"> +<%= +{ + "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") +}.to_pretty_json +%> </script> -<script src="/js/playlist_widget.js"></script> +<script src="/js/playlist_widget.js?v=<%= ASSET_COMMIT %>"></script> <div class="pure-g"> <% videos.each_slice(4) do |slice| %> diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index b5eb46ea..4e9c7a63 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -92,7 +92,7 @@ <div class="pure-g h-box"> <div class="pure-u-1 pure-u-lg-1-5"> <% if page > 1 %> - <a href="/channel/<%= channel.ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= sort_by %><% end %>"> + <a href="/channel/<%= channel.ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> <%= translate(locale, "Previous page") %> </a> <% end %> @@ -100,7 +100,7 @@ <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if count == 60 %> - <a href="/channel/<%= channel.ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= sort_by %><% end %>"> + <a href="/channel/<%= channel.ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 218cc2d4..69724390 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -71,14 +71,16 @@ </div> <% end %> -<script> -var community_data = { - ucid: '<%= channel.ucid %>', - youtube_comments_text: '<%= HTML.escape(translate(locale, "View YouTube comments")) %>', - comments_text: '<%= HTML.escape(translate(locale, "View `x` comments", "{commentCount}")) %>', - hide_replies_text: '<%= HTML.escape(translate(locale, "Hide replies")) %>', - show_replies_text: '<%= HTML.escape(translate(locale, "Show replies")) %>', - preferences: <%= env.get("preferences").as(Preferences).to_json %>, -} +<script id="community_data" type="application/json"> +<%= +{ + "ucid" => channel.ucid, + "youtube_comments_text" => HTML.escape(translate(locale, "View YouTube comments")), + "comments_text" => HTML.escape(translate(locale, "View `x` comments", "{commentCount}")), + "hide_replies_text" => HTML.escape(translate(locale, "Hide replies")), + "show_replies_text" => HTML.escape(translate(locale, "Show replies")), + "preferences" => env.get("preferences").as(Preferences) +}.to_pretty_json +%> </script> <script src="/js/community.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index f7b9cce6..0c19fc1b 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -44,7 +44,7 @@ <% end %> </div> <% end %> - <p><%= item.title %></p> + <p><%= HTML.escape(item.title) %></p> </a> <p> <b> @@ -57,10 +57,10 @@ <div class="thumbnail"> <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> <% if plid = env.get?("remove_playlist_items") %> - <form onsubmit="return false" action="/playlist_ajax?action_remove_video=1&set_video_id=<%= item.index %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/playlist_ajax?action_remove_video=1&set_video_id=<%= item.index %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <p class="watched"> - <a onclick="remove_playlist_item(this)" data-index="<%= item.index %>" data-plid="<%= plid %>" href="javascript:void(0)"> + <a data-onclick="remove_playlist_item" data-index="<%= item.index %>" data-plid="<%= plid %>" href="javascript:void(0)"> <button type="submit" style="all:unset"> <i class="icon ion-md-trash"></i> </button> @@ -76,7 +76,7 @@ <% end %> </div> <% end %> - <p><a href="/watch?v=<%= item.id %>"><%= item.title %></a></p> + <p><a href="/watch?v=<%= item.id %>"><%= HTML.escape(item.title) %></a></p> </a> <p> <b> @@ -85,7 +85,7 @@ </p> <h5 class="pure-g"> - <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.utc %> + <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %> <div class="pure-u-2-3"><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %></div> <% elsif Time.utc - item.published > 1.minute %> <div class="pure-u-2-3"><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></div> @@ -103,13 +103,12 @@ <div class="thumbnail"> <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> <% if env.get? "show_watched" %> - <form onsubmit="return false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <p class="watched"> - <a onclick="mark_watched(this)" data-id="<%= item.id %>" href="javascript:void(0)"> + <a data-onclick="mark_watched" data-id="<%= item.id %>" href="javascript:void(0)"> <button type="submit" style="all:unset"> - <i onmouseenter='this.setAttribute("class", "icon ion-ios-eye-off")' - onmouseleave='this.setAttribute("class", "icon ion-ios-eye")' + <i data-mouse="switch_classes" data-switch-classes="ion-ios-eye-off,ion-ios-eye" class="icon ion-ios-eye"> </i> </button> @@ -117,10 +116,10 @@ </p> </form> <% elsif plid = env.get? "add_playlist_items" %> - <form onsubmit="return false" action="/playlist_ajax?action_add_video=1&video_id=<%= item.id %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/playlist_ajax?action_add_video=1&video_id=<%= item.id %>&playlist_id=<%= plid %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <p class="watched"> - <a onclick="add_playlist_item(this)" data-id="<%= item.id %>" data-plid="<%= plid %>" href="javascript:void(0)"> + <a data-onclick="add_playlist_item" data-id="<%= item.id %>" data-plid="<%= plid %>" href="javascript:void(0)"> <button type="submit" style="all:unset"> <i class="icon ion-md-add"></i> </button> @@ -137,7 +136,7 @@ </div> </a> <% end %> - <p><a href="/watch?v=<%= item.id %>"><%= item.title %></a></p> + <p><a href="/watch?v=<%= item.id %>"><%= HTML.escape(item.title) %></a></p> <p> <b> <a style="width:100%" href="/channel/<%= item.ucid %>"><%= item.author %></a> @@ -145,7 +144,7 @@ </p> <h5 class="pure-g"> - <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.utc %> + <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %> <div class="pure-u-2-3"><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %></div> <% elsif Time.utc - item.published > 1.minute %> <div class="pure-u-2-3"><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></div> diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index ba6311cb..6b01d25f 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -1,28 +1,25 @@ <video style="outline:none;width:100%;background-color:#000" playsinline poster="<%= thumbnail %>" title="<%= HTML.escape(video.title) %>" - id="player" class="video-js player-style-<%= params.player_style %>" - onmouseenter='this["data-title"]=this["title"];this["title"]=""' - onmouseleave='this["title"]=this["data-title"];this["data-title"]=""' - oncontextmenu='this["title"]=this["data-title"]' + id="player" class="on-video_player video-js player-style-<%= params.player_style %>" <% if params.autoplay %>autoplay<% end %> <% if params.video_loop %>loop<% end %> <% if params.controls %>controls<% end %>> - <% if hlsvp && !CONFIG.disabled?("livestreams") %> - <source src="<%= hlsvp %>?local=true" type="application/x-mpegURL" label="livestream"> + <% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %> + <source src="<%= URI.parse(hlsvp).full_path %>?local=true" type="application/x-mpegURL" label="livestream"> <% else %> <% if params.listen %> <% audio_streams.each_with_index do |fmt, i| %> - <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["bitrate"] %>k" selected="<%= i == 0 ? true : false %>"> + <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["bitrate"] %>k" selected="<%= i == 0 ? true : false %>"> <% end %> <% else %> <% if params.quality == "dash" %> - <source src="/api/manifest/dash/id/<%= video.id %>?local=true" type='application/dash+xml' label="dash"> + <source src="/api/manifest/dash/id/<%= video.id %>?local=true&unique_res=1" type='application/dash+xml' label="dash"> <% end %> <% fmt_stream.each_with_index do |fmt, i| %> <% if params.quality %> - <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["label"] %>" selected="<%= params.quality == fmt["label"].split(" - ")[0] %>"> + <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["quality"] %>" selected="<%= params.quality == fmt["quality"] %>"> <% else %> - <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["type"] %>' label="<%= fmt["label"] %>" selected="<%= i == 0 ? true : false %>"> + <source src="/latest_version?id=<%= video.id %>&itag=<%= fmt["itag"] %><% if params.local %>&local=true<% end %>" type='<%= fmt["mimeType"] %>' label="<%= fmt["quality"] %>" selected="<%= i == 0 ? true : false %>"> <% end %> <% end %> <% end %> @@ -39,12 +36,14 @@ <% end %> </video> -<script> -var player_data = { - aspect_ratio: '<%= aspect_ratio %>', - title: "<%= video.title.dump_unquoted %>", - description: "<%= HTML.escape(video.short_description) %>", - thumbnail: "<%= thumbnail %>" -} +<script id="player_data" type="application/json"> +<%= +{ + "aspect_ratio" => aspect_ratio, + "title" => video.title, + "description" => HTML.escape(video.short_description), + "thumbnail" => thumbnail +}.to_pretty_json +%> </script> <script src="/js/player.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/views/components/player_sources.ecr b/src/invidious/views/components/player_sources.ecr index d950e0da..8162546e 100644 --- a/src/invidious/views/components/player_sources.ecr +++ b/src/invidious/views/components/player_sources.ecr @@ -3,6 +3,7 @@ <link rel="stylesheet" href="/css/videojs.markers.min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/videojs-share.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/videojs-vtt-thumbnails.css?v=<%= ASSET_COMMIT %>"> +<script src="/js/global.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/video.min.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/videojs-contrib-quality-levels.min.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/videojs-http-source-selector.min.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/views/components/subscribe_widget.ecr b/src/invidious/views/components/subscribe_widget.ecr index 471e6c1c..ac2fbf1d 100644 --- a/src/invidious/views/components/subscribe_widget.ecr +++ b/src/invidious/views/components/subscribe_widget.ecr @@ -19,15 +19,17 @@ </p> <% end %> - <script> - var subscribe_data = { - ucid: '<%= ucid %>', - author: '<%= HTML.escape(author) %>', - sub_count_text: '<%= HTML.escape(sub_count_text) %>', - csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>', - subscribe_text: '<%= HTML.escape(translate(locale, "Subscribe")) %>', - unsubscribe_text: '<%= HTML.escape(translate(locale, "Unsubscribe")) %>' - } + <script id="subscribe_data" type="application/json"> + <%= + { + "ucid" => ucid, + "author" => HTML.escape(author), + "sub_count_text" => HTML.escape(sub_count_text), + "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || ""), + "subscribe_text" => HTML.escape(translate(locale, "Subscribe")), + "unsubscribe_text" => HTML.escape(translate(locale, "Unsubscribe")) + }.to_pretty_json + %> </script> <script src="/js/subscribe_widget.js?v=<%= ASSET_COMMIT %>"></script> <% else %> diff --git a/src/invidious/views/embed.ecr b/src/invidious/views/embed.ecr index 6c06bf2e..48dbc55f 100644 --- a/src/invidious/views/embed.ecr +++ b/src/invidious/views/embed.ecr @@ -10,33 +10,24 @@ <script src="/js/videojs-overlay.min.js?v=<%= ASSET_COMMIT %>"></script> <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/darktheme.css?v=<%= ASSET_COMMIT %>"> + <link rel="stylesheet" href="/css/embed.css?v=<%= ASSET_COMMIT %>"> <title><%= HTML.escape(video.title) %> - Invidious</title> - <style> - #player { - position: fixed; - right: 0; - bottom: 0; - min-width: 100%; - min-height: 100%; - width: auto; - height: auto; - z-index: -100; - } - </style> </head> <body> -<script> -var video_data = { - id: '<%= video.id %>', - index: '<%= continuation %>', - plid: '<%= plid %>', - length_seconds: '<%= video.length_seconds.to_f %>', - video_series: <%= video_series.to_json %>, - params: <%= params.to_json %>, - preferences: <%= preferences.to_json %>, - premiere_timestamp: <%= video.premiere_timestamp.try &.to_unix || "null" %> -} +<script id="video_data" type="application/json"> +<%= +{ + "id" => video.id, + "index" => continuation, + "plid" => plid, + "length_seconds" => video.length_seconds.to_f, + "video_series" => video_series, + "params" => params, + "preferences" => preferences, + "premiere_timestamp" => video.premiere_timestamp.try &.to_unix +}.to_pretty_json +%> </script> <%= rendered "components/player" %> diff --git a/src/invidious/views/history.ecr b/src/invidious/views/history.ecr index 7d7ded2c..fe8c70b9 100644 --- a/src/invidious/views/history.ecr +++ b/src/invidious/views/history.ecr @@ -18,10 +18,12 @@ </div> </div> -<script> -var watched_data = { - csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>', -} +<script id="watched_data" type="application/json"> +<%= +{ + "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") +}.to_pretty_json +%> </script> <script src="/js/watched_widget.js"></script> @@ -34,10 +36,10 @@ var watched_data = { <% if !env.get("preferences").as(Preferences).thin_mode %> <div class="thumbnail"> <img class="thumbnail" src="/vi/<%= item %>/mqdefault.jpg"/> - <form onsubmit="return false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <p class="watched"> - <a onclick="mark_unwatched(this)" data-id="<%= item %>" href="javascript:void(0)"> + <a data-onclick="mark_unwatched" data-id="<%= item %>" href="javascript:void(0)"> <button type="submit" style="all:unset"> <i class="icon ion-md-trash"></i> </button> diff --git a/src/invidious/views/login.ecr b/src/invidious/views/login.ecr index 59fa90e5..b6e8117b 100644 --- a/src/invidious/views/login.ecr +++ b/src/invidious/views/login.ecr @@ -22,7 +22,43 @@ <hr> <% case account_type when %> - <% when "invidious" %> + <% when "google" %> + <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=google" method="post"> + <fieldset> + <% if email %> + <input name="email" type="hidden" value="<%= email %>"> + <% else %> + <label for="email"><%= translate(locale, "E-mail") %> :</label> + <input required class="pure-input-1" name="email" type="email" placeholder="<%= translate(locale, "E-mail") %>"> + <% end %> + + <% if password %> + <input name="password" type="hidden" value="<%= HTML.escape(password) %>"> + <% else %> + <label for="password"><%= translate(locale, "Password") %> :</label> + <input required class="pure-input-1" name="password" type="password" placeholder="<%= translate(locale, "Password") %>"> + <% end %> + + <% if prompt %> + <label for="tfa"><%= translate(locale, prompt) %> :</label> + <input required class="pure-input-1" name="tfa" type="text" placeholder="<%= translate(locale, prompt) %>"> + <% end %> + + <% if tfa %> + <input type="hidden" name="tfa" value="<%= tfa %>"> + <% end %> + + <% if captcha %> + <img style="width:50%" src="/Captcha?v=2&ctoken=<%= captcha[:tokens][0] %>"/> + <input type="hidden" name="token" value="<%= captcha[:tokens][0] %>"> + <label for="answer"><%= translate(locale, "Answer") %> :</label> + <input type="text" name="answer" type="text" placeholder="<%= translate(locale, "Answer") %>"> + <% end %> + + <button type="submit" class="pure-button pure-button-primary"><%= translate(locale, "Sign In") %></button> + </fieldset> + </form> + <% else # "invidious" %> <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=invidious" method="post"> <fieldset> <% if email %> @@ -50,7 +86,7 @@ <input type="hidden" name="captcha_type" value="image"> <label for="answer"><%= translate(locale, "Time (h:mm:ss):") %></label> <input type="text" name="answer" type="text" placeholder="h:mm:ss"> - <% when "text" %> + <% else # "text" %> <% captcha = captcha.not_nil! %> <% captcha[:tokens].each_with_index do |token, i| %> <input type="hidden" name="token[<%= i %>]" value="<%= URI.encode_www_form(token) %>"> @@ -71,7 +107,7 @@ <%= translate(locale, "Text CAPTCHA") %> </button> </label> - <% when "text" %> + <% else # "text" %> <label> <button type="submit" name="change_type" class="pure-button pure-button-primary" value="image"> <%= translate(locale, "Image CAPTCHA") %> @@ -85,42 +121,6 @@ <% end %> </fieldset> </form> - <% when "google" %> - <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=google" method="post"> - <fieldset> - <% if email %> - <input name="email" type="hidden" value="<%= email %>"> - <% else %> - <label for="email"><%= translate(locale, "E-mail") %> :</label> - <input required class="pure-input-1" name="email" type="email" placeholder="<%= translate(locale, "E-mail") %>"> - <% end %> - - <% if password %> - <input name="password" type="hidden" value="<%= HTML.escape(password) %>"> - <% else %> - <label for="password"><%= translate(locale, "Password") %> :</label> - <input required class="pure-input-1" name="password" type="password" placeholder="<%= translate(locale, "Password") %>"> - <% end %> - - <% if prompt %> - <label for="tfa"><%= translate(locale, prompt) %> :</label> - <input required class="pure-input-1" name="tfa" type="text" placeholder="<%= translate(locale, prompt) %>"> - <% end %> - - <% if tfa %> - <input type="hidden" name="tfa" value="<%= tfa %>"> - <% end %> - - <% if captcha %> - <img style="width:50%" src="/Captcha?v=2&ctoken=<%= captcha[:tokens][0] %>"/> - <input type="hidden" name="token" value="<%= captcha[:tokens][0] %>"> - <label for="answer"><%= translate(locale, "Answer") %> :</label> - <input type="text" name="answer" type="text" placeholder="<%= translate(locale, "Answer") %>"> - <% end %> - - <button type="submit" class="pure-button pure-button-primary"><%= translate(locale, "Sign In") %></button> - </fieldset> - </form> <% end %> </div> </div> diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index cb643aaa..bb721c3a 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -45,6 +45,12 @@ <% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> <div class="pure-u-1-3"><a href="/edit_playlist?list=<%= plid %>"><i class="icon ion-md-create"></i></a></div> <div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div> + <% else %> + <% if PG_DB.query_one?("SELECT id FROM playlists WHERE id = $1", playlist.id, as: String).nil? %> + <div class="pure-u-1-3"><a href="/subscribe_playlist?list=<%= plid %>"><i class="icon ion-md-add"></i></a></div> + <% else %> + <div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div> + <% end %> <% end %> <div class="pure-u-1-3"><a href="/feed/playlist/<%= plid %>"><i class="icon ion-logo-rss"></i></a></div> </div> @@ -69,12 +75,14 @@ </div> <% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> -<script> -var playlist_data = { - csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>', -} +<script id="playlist_data" type="application/json"> +<%= +{ + "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") +}.to_pretty_json +%> </script> -<script src="/js/playlist_widget.js"></script> +<script src="/js/playlist_widget.js?v=<%= ASSET_COMMIT %>"></script> <% end %> <div class="pure-g"> diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr index a32192b5..0c48be96 100644 --- a/src/invidious/views/playlists.ecr +++ b/src/invidious/views/playlists.ecr @@ -90,7 +90,7 @@ <div class="pure-u-1 pure-u-md-4-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if continuation %> - <a href="/channel/<%= channel.ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= sort_by %><% end %>"> + <a href="/channel/<%= channel.ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= HTML.escape(sort_by) %><% end %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index 17e5804e..fb5bd44b 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -2,12 +2,6 @@ <title><%= translate(locale, "Preferences") %> - Invidious</title> <% end %> -<script> -function update_value(element) { - document.getElementById('volume-value').innerText = element.value; -} -</script> - <div class="h-box"> <form class="pure-form pure-form-aligned" action="/preferences?referer=<%= URI.encode_www_form(referer) %>" method="post"> <fieldset> @@ -65,7 +59,7 @@ function update_value(element) { <div class="pure-control-group"> <label for="volume"><%= translate(locale, "Player volume: ") %></label> - <input name="volume" id="volume" oninput="update_value(this);" type="range" min="0" max="100" step="5" value="<%= preferences.volume %>"> + <input name="volume" id="volume" data-onrange="update_volume_value" type="range" min="0" max="100" step="5" value="<%= preferences.volume %>"> <span class="pure-form-message-inline" id="volume-value"><%= preferences.volume %></span> </div> @@ -205,7 +199,7 @@ function update_value(element) { <% # Web notifications are only supported over HTTPS %> <% if Kemal.config.ssl || config.https_only %> <div class="pure-control-group"> - <a href="#" onclick="Notification.requestPermission()"><%= translate(locale, "Enable web notifications") %></a> + <a href="#" data-onclick="notification_requestPermission"><%= translate(locale, "Enable web notifications") %></a> </div> <% end %> <% end %> @@ -234,11 +228,6 @@ function update_value(element) { </div> <div class="pure-control-group"> - <label for="top_enabled"><%= translate(locale, "Top enabled: ") %></label> - <input name="top_enabled" id="top_enabled" type="checkbox" <% if config.top_enabled %>checked<% end %>> - </div> - - <div class="pure-control-group"> <label for="captcha_enabled"><%= translate(locale, "CAPTCHA enabled: ") %></label> <input name="captcha_enabled" id="captcha_enabled" type="checkbox" <% if config.captcha_enabled %>checked<% end %>> </div> diff --git a/src/invidious/views/subscription_manager.ecr b/src/invidious/views/subscription_manager.ecr index 43d14b37..6cddcd6c 100644 --- a/src/invidious/views/subscription_manager.ecr +++ b/src/invidious/views/subscription_manager.ecr @@ -37,9 +37,9 @@ <div class="pure-u-2-5"></div> <div class="pure-u-1-5" style="text-align:right"> <h3 style="padding-right:0.5em"> - <form onsubmit="return false" action="/subscription_ajax?action_remove_subscriptions=1&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/subscription_ajax?action_remove_subscriptions=1&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> - <a onclick="remove_subscription(this)" data-ucid="<%= channel.id %>" href="#"> + <a data-onclick="remove_subscription" data-ucid="<%= channel.id %>" href="#"> <input style="all:unset" type="submit" value="<%= translate(locale, "unsubscribe") %>"> </a> </form> @@ -52,32 +52,3 @@ <% end %> </div> <% end %> - -<script> -function remove_subscription(target) { - var row = target.parentNode.parentNode.parentNode.parentNode.parentNode; - row.style.display = 'none'; - var count = document.getElementById('count'); - count.innerText = count.innerText - 1; - - var url = '/subscription_ajax?action_remove_subscriptions=1&redirect=false' + - '&referer=<%= env.get("current_page") %>' + - '&c=' + target.getAttribute('data-ucid'); - var xhr = new XMLHttpRequest(); - xhr.responseType = 'json'; - xhr.timeout = 10000; - xhr.open('POST', url, true); - xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded'); - - xhr.onreadystatechange = function() { - if (xhr.readyState == 4) { - if (xhr.status != 200) { - count.innerText = parseInt(count.innerText) + 1; - row.style.display = ''; - } - } - } - - xhr.send('csrf_token=<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>'); -} -</script> diff --git a/src/invidious/views/subscriptions.ecr b/src/invidious/views/subscriptions.ecr index ee31d241..af1d4fbc 100644 --- a/src/invidious/views/subscriptions.ecr +++ b/src/invidious/views/subscriptions.ecr @@ -45,10 +45,12 @@ <hr> </div> -<script> -var watched_data = { - csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>', -} +<script id="watched_data" type="application/json"> +<%= +{ + "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") +}.to_pretty_json +%> </script> <script src="/js/watched_widget.js"></script> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index d2ef9c7e..61cf5c3a 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -111,7 +111,7 @@ <div class="footer"> <div class="pure-g"> <div class="pure-u-1 pure-u-md-1-3"> - <a href="https://github.com/omarroth/invidious"> + <a href="https://github.com/iv-org/invidious"> <%= translate(locale, "Released under the AGPLv3 by Omar Roth.") %> </a> </div> @@ -147,14 +147,17 @@ </div> <div class="pure-u-1 pure-u-md-2-24"></div> </div> + <script src="/js/handlers.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/themes.js?v=<%= ASSET_COMMIT %>"></script> <% if env.get? "user" %> <script src="/js/sse.js?v=<%= ASSET_COMMIT %>"></script> - <script> - var notification_data = { - upload_text: '<%= HTML.escape(translate(locale, "`x` uploaded a video")) %>', - live_upload_text: '<%= HTML.escape(translate(locale, "`x` is live")) %>', - } + <script id="notification_data" type="application/json"> + <%= + { + "upload_text" => HTML.escape(translate(locale, "`x` uploaded a video")), + "live_upload_text" => HTML.escape(translate(locale, "`x` is live")) + }.to_pretty_json + %> </script> <script src="/js/notifications.js?v=<%= ASSET_COMMIT %>"></script> <% end %> diff --git a/src/invidious/views/token_manager.ecr b/src/invidious/views/token_manager.ecr index b626d99c..e48aec2f 100644 --- a/src/invidious/views/token_manager.ecr +++ b/src/invidious/views/token_manager.ecr @@ -29,9 +29,9 @@ </div> <div class="pure-u-1-5" style="text-align:right"> <h3 style="padding-right:0.5em"> - <form onsubmit="return false" action="/token_ajax?action_revoke_token=1&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post"> + <form data-onsubmit="return_false" action="/token_ajax?action_revoke_token=1&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> - <a onclick="revoke_token(this)" data-session="<%= token[:session] %>" href="#"> + <a data-onclick="revoke_token" data-session="<%= token[:session] %>" href="#"> <input style="all:unset" type="submit" value="<%= translate(locale, "revoke") %>"> </a> </form> @@ -44,32 +44,3 @@ <% end %> </div> <% end %> - -<script> -function revoke_token(target) { - var row = target.parentNode.parentNode.parentNode.parentNode.parentNode; - row.style.display = 'none'; - var count = document.getElementById('count'); - count.innerText = count.innerText - 1; - - var url = '/token_ajax?action_revoke_token=1&redirect=false' + - '&referer=<%= env.get("current_page") %>' + - '&session=' + target.getAttribute('data-session'); - var xhr = new XMLHttpRequest(); - xhr.responseType = 'json'; - xhr.timeout = 10000; - xhr.open('POST', url, true); - xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded'); - - xhr.onreadystatechange = function() { - if (xhr.readyState == 4) { - if (xhr.status != 200) { - count.innerText = parseInt(count.innerText) + 1; - row.style.display = ''; - } - } - } - - xhr.send('csrf_token=<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>'); -} -</script> diff --git a/src/invidious/views/top.ecr b/src/invidious/views/top.ecr deleted file mode 100644 index f5db3aaa..00000000 --- a/src/invidious/views/top.ecr +++ /dev/null @@ -1,20 +0,0 @@ -<% content_for "header" do %> -<meta name="description" content="<%= translate(locale, "An alternative front-end to YouTube") %>"> -<title> - <% if env.get("preferences").as(Preferences).default_home != "Top" %> - <%= translate(locale, "Top") %> - Invidious - <% else %> - Invidious - <% end %> -</title> -<% end %> - -<%= rendered "components/feed_menu" %> - -<div class="pure-g"> - <% top_videos.each_slice(4) do |slice| %> - <% slice.each do |item| %> - <%= rendered "components/item" %> - <% end %> - <% end %> -</div> diff --git a/src/invidious/views/view_all_playlists.ecr b/src/invidious/views/view_all_playlists.ecr index 0fa7a325..5ec6aa31 100644 --- a/src/invidious/views/view_all_playlists.ecr +++ b/src/invidious/views/view_all_playlists.ecr @@ -6,7 +6,7 @@ <div class="pure-g h-box"> <div class="pure-u-2-3"> - <h3><%= translate(locale, "`x` playlists", %(<span id="count">#{items.size}</span>)) %></h3> + <h3><%= translate(locale, "`x` created playlists", %(<span id="count">#{items_created.size}</span>)) %></h3> </div> <div class="pure-u-1-3" style="text-align:right"> <h3> @@ -16,7 +16,21 @@ </div> <div class="pure-g"> - <% items.each_slice(4) do |slice| %> + <% items_created.each_slice(4) do |slice| %> + <% slice.each do |item| %> + <%= rendered "components/item" %> + <% end %> + <% end %> +</div> + +<div class="pure-g h-box"> + <div class="pure-u-1"> + <h3><%= translate(locale, "`x` saved playlists", %(<span id="count">#{items_saved.size}</span>)) %></h3> + </div> +</div> + +<div class="pure-g"> + <% items_saved.each_slice(4) do |slice| %> <% slice.each do |item| %> <%= rendered "components/item" %> <% end %> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index df61abc5..9a1e6c32 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -3,47 +3,49 @@ <meta name="description" content="<%= HTML.escape(video.short_description) %>"> <meta name="keywords" content="<%= video.keywords.join(",") %>"> <meta property="og:site_name" content="Invidious"> -<meta property="og:url" content="<%= host_url %>/watch?v=<%= video.id %>"> +<meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta property="og:title" content="<%= HTML.escape(video.title) %>"> <meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg"> -<meta property="og:description" content="<%= HTML.escape(video.short_description) %>"> +<meta property="og:description" content="<%= video.short_description %>"> <meta property="og:type" content="video.other"> -<meta property="og:video:url" content="<%= host_url %>/embed/<%= video.id %>"> -<meta property="og:video:secure_url" content="<%= host_url %>/embed/<%= video.id %>"> +<meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>"> +<meta property="og:video:secure_url" content="<%= HOST_URL %>/embed/<%= video.id %>"> <meta property="og:video:type" content="text/html"> <meta property="og:video:width" content="1280"> <meta property="og:video:height" content="720"> <meta name="twitter:card" content="player"> <meta name="twitter:site" content="@omarroth1"> -<meta name="twitter:url" content="<%= host_url %>/watch?v=<%= video.id %>"> +<meta name="twitter:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>"> <meta name="twitter:title" content="<%= HTML.escape(video.title) %>"> -<meta name="twitter:description" content="<%= HTML.escape(video.short_description) %>"> -<meta name="twitter:image" content="<%= host_url %>/vi/<%= video.id %>/maxres.jpg"> -<meta name="twitter:player" content="<%= host_url %>/embed/<%= video.id %>"> +<meta name="twitter:description" content="<%= video.short_description %>"> +<meta name="twitter:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg"> +<meta name="twitter:player" content="<%= HOST_URL %>/embed/<%= video.id %>"> <meta name="twitter:player:width" content="1280"> <meta name="twitter:player:height" content="720"> <%= rendered "components/player_sources" %> <title><%= HTML.escape(video.title) %> - Invidious</title> <% end %> -<script> -var video_data = { - id: '<%= video.id %>', - index: '<%= continuation %>', - plid: '<%= plid %>', - length_seconds: <%= video.length_seconds.to_f %>, - play_next: <%= !rvs.empty? && !plid && params.continue %>, - next_video: '<%= rvs.select { |rv| rv["id"]? }[0]?.try &.["id"] %>', - youtube_comments_text: '<%= HTML.escape(translate(locale, "View YouTube comments")) %>', - reddit_comments_text: '<%= HTML.escape(translate(locale, "View Reddit comments")) %>', - reddit_permalink_text: '<%= HTML.escape(translate(locale, "View more comments on Reddit")) %>', - comments_text: '<%= HTML.escape(translate(locale, "View `x` comments", "{commentCount}")) %>', - hide_replies_text: '<%= HTML.escape(translate(locale, "Hide replies")) %>', - show_replies_text: '<%= HTML.escape(translate(locale, "Show replies")) %>', - params: <%= params.to_json %>, - preferences: <%= preferences.to_json %>, - premiere_timestamp: <%= video.premiere_timestamp.try &.to_unix || "null" %> -} +<script id="video_data" type="application/json"> +<%= +{ + "id" => video.id, + "index" => continuation, + "plid" => plid, + "length_seconds" => video.length_seconds.to_f, + "play_next" => !video.related_videos.empty? && !plid && params.continue, + "next_video" => video.related_videos.select { |rv| rv["id"]? }[0]?.try &.["id"], + "youtube_comments_text" => HTML.escape(translate(locale, "View YouTube comments")), + "reddit_comments_text" => HTML.escape(translate(locale, "View Reddit comments")), + "reddit_permalink_text" => HTML.escape(translate(locale, "View more comments on Reddit")), + "comments_text" => HTML.escape(translate(locale, "View `x` comments", "{commentCount}")), + "hide_replies_text" => HTML.escape(translate(locale, "Hide replies")), + "show_replies_text" => HTML.escape(translate(locale, "Show replies")), + "params" => params, + "preferences" => preferences, + "premiere_timestamp" => video.premiere_timestamp.try &.to_unix +}.to_pretty_json +%> </script> <div id="player-container" class="h-box"> @@ -70,13 +72,13 @@ var video_data = { </h3> <% end %> - <% if !reason.empty? %> + <% if video.reason %> <h3> - <%= reason %> + <%= video.reason %> </h3> - <% elsif video.premiere_timestamp %> + <% elsif video.premiere_timestamp.try &.> Time.utc %> <h3> - <%= translate(locale, "Premieres in `x`", recode_date((video.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %> + <%= video.premiere_timestamp.try { |t| translate(locale, "Premieres in `x`", recode_date((t - Time.utc).ago, locale)) } %> </h3> <% end %> </div> @@ -84,10 +86,10 @@ var video_data = { <div class="pure-g"> <div class="pure-u-1 pure-u-lg-1-5"> <div class="h-box"> - <span> + <span id="watch-on-youtube"> <a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "Watch on YouTube") %></a> </span> - <p> + <p id="annotations"> <% if params.annotations %> <a href="/watch?<%= env.params.query %>&iv_load_policy=3"> <%= translate(locale, "Hide annotations") %> @@ -99,26 +101,54 @@ var video_data = { <% end %> </p> + <% if user %> + <% playlists = PG_DB.query_all("SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%'", user.email, as: {String, String}) %> + <% if !playlists.empty? %> + <form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax" method="post"> + <div class="pure-control-group"> + <label for="playlist_id"><%= translate(locale, "Add to playlist: ") %></label> + <select style="width:100%" name="playlist_id" id="playlist_id"> + <% playlists.each do |plid, title| %> + <option data-plid="<%= plid %>" value="<%= plid %>"><%= title %></option> + <% end %> + </select> + </div> + + <button data-onclick="add_playlist_video" data-id="<%= video.id %>" type="submit" class="pure-button pure-button-primary"> + <b><%= translate(locale, "Add to playlist") %></b> + </button> + </form> + <script id="playlist_data" type="application/json"> + <%= + { + "csrf_token" => URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") + }.to_pretty_json + %> + </script> + <script src="/js/playlist_widget.js?v=<%= Time.utc.to_unix_ms %>"></script> + <% end %> + <% end %> + <% if CONFIG.dmca_content.includes?(video.id) || CONFIG.disabled?("downloads") %> - <p><%= translate(locale, "Download is disabled.") %></p> + <p id="download"><%= translate(locale, "Download is disabled.") %></p> <% else %> <form class="pure-form pure-form-stacked" action="/latest_version" method="get" rel="noopener" target="_blank"> <div class="pure-control-group"> <label for="download_widget"><%= translate(locale, "Download as: ") %></label> <select style="width:100%" name="download_widget" id="download_widget"> <% fmt_stream.each do |option| %> - <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'> - <%= itag_to_metadata?(option["itag"]).try &.["height"]? || "~240" %>p - <%= option["type"].split(";")[0] %> + <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'> + <%= itag_to_metadata?(option["itag"]).try &.["height"]? || "~240" %>p - <%= option["mimeType"].as_s.split(";")[0] %> </option> <% end %> <% video_streams.each do |option| %> - <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'> - <%= option["quality_label"] %> - <%= option["type"].split(";")[0] %> @ <%= option["fps"] %>fps - video only + <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'> + <%= option["qualityLabel"] %> - <%= option["mimeType"].as_s.split(";")[0] %> @ <%= option["fps"] %>fps - video only </option> <% end %> <% audio_streams.each do |option| %> - <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'> - <%= option["type"].split(";")[0] %> @ <%= option["bitrate"] %>k - audio only + <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'> + <%= option["mimeType"].as_s.split(";")[0] %> @ <%= option["bitrate"]?.try &.as_i./ 1000 %>k - audio only </option> <% end %> <% captions.each do |caption| %> @@ -135,23 +165,23 @@ var video_data = { </form> <% end %> - <p><i class="icon ion-ios-eye"></i> <%= number_with_separator(video.views) %></p> - <p><i class="icon ion-ios-thumbs-up"></i> <%= number_with_separator(video.likes) %></p> - <p><i class="icon ion-ios-thumbs-down"></i> <%= number_with_separator(video.dislikes) %></p> + <p id="views"><i class="icon ion-ios-eye"></i> <%= number_with_separator(video.views) %></p> + <p id="likes"><i class="icon ion-ios-thumbs-up"></i> <%= number_with_separator(video.likes) %></p> + <p id="dislikes"><i class="icon ion-ios-thumbs-down"></i> <%= number_with_separator(video.dislikes) %></p> <p id="genre"><%= translate(locale, "Genre: ") %> - <% if video.genre_url.empty? %> + <% if !video.genre_url %> <%= video.genre %> <% else %> <a href="<%= video.genre_url %>"><%= video.genre %></a> <% end %> </p> - <% if !video.license.empty? %> + <% if video.license %> <p id="license"><%= translate(locale, "License: ") %><%= video.license %></p> <% end %> <p id="family_friendly"><%= translate(locale, "Family friendly? ") %><%= translate_bool(locale, video.is_family_friendly) %></p> - <p id="wilson"><%= translate(locale, "Wilson score: ") %><%= video.wilson_score.round(4) %></p> - <p id="rating"><%= translate(locale, "Rating: ") %><%= rating.round(4) %> / 5</p> - <p id="engagement"><%= translate(locale, "Engagement: ") %><%= engagement.round(2) %>%</p> + <p id="wilson"><%= translate(locale, "Wilson score: ") %><%= video.wilson_score %></p> + <p id="rating"><%= translate(locale, "Rating: ") %><%= video.average_rating %> / 5</p> + <p id="engagement"><%= translate(locale, "Engagement: ") %><%= video.engagement %>%</p> <% if video.allowed_regions.size != REGIONS.size %> <p id="allowed_regions"> <% if video.allowed_regions.size < REGIONS.size // 2 %> @@ -168,8 +198,10 @@ var video_data = { <div class="h-box"> <a href="/channel/<%= video.ucid %>" style="display:block;width:fit-content;width:-moz-fit-content"> <div class="channel-profile"> - <img src="/ggpht<%= URI.parse(video.author_thumbnail).full_path %>"> - <span><%= video.author %></span> + <% if !video.author_thumbnail.empty? %> + <img src="/ggpht<%= URI.parse(video.author_thumbnail).full_path %>"> + <% end %> + <span id="channel-name"><%= video.author %></span> </div> </a> @@ -178,9 +210,9 @@ var video_data = { <% sub_count_text = video.sub_count_text %> <%= rendered "components/subscribe_widget" %> - <p> - <% if video.premiere_timestamp %> - <b><%= translate(locale, "Premieres `x`", video.premiere_timestamp.not_nil!.to_s("%B %-d, %R UTC")) %></b> + <p id="published-date"> + <% if video.premiere_timestamp.try &.> Time.utc %> + <b><%= video.premiere_timestamp.try { |t| translate(locale, "Premieres `x`", t.to_s("%B %-d, %R UTC")) } %></b> <% else %> <b><%= translate(locale, "Shared `x`", video.published.to_s("%B %-d, %Y")) %></b> <% end %> @@ -214,7 +246,7 @@ var video_data = { <% if params.related_videos %> <div class="h-box"> - <% if !rvs.empty? %> + <% if !video.related_videos.empty? %> <div <% if plid %>style="display:none"<% end %>> <div class="pure-control-group"> <label for="continue"><%= translate(locale, "Play next by default: ") %></label> @@ -224,7 +256,7 @@ var video_data = { </div> <% end %> - <% rvs.each do |rv| %> + <% video.related_videos.each do |rv| %> <% if rv["id"]? %> <a href="/watch?v=<%= rv["id"] %>"> <% if !env.get("preferences").as(Preferences).thin_mode %> @@ -237,15 +269,17 @@ var video_data = { <h5 class="pure-g"> <div class="pure-u-14-24"> <% if rv["ucid"]? %> - <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"] %></a></b> + <b style="width:100%"><a href="/channel/<%= rv["ucid"] %>"><%= rv["author"]? %></a></b> <% else %> - <b style="width:100%"><%= rv["author"] %></b> + <b style="width:100%"><%= rv["author"]? %></b> <% end %> </div> <div class="pure-u-10-24" style="text-align:right"> <% if views = rv["short_view_count_text"]?.try &.delete(", views watching") %> - <b class="width:100%"><%= translate(locale, "`x` views", views) %></b> + <% if !views.empty? %> + <b class="width:100%"><%= translate(locale, "`x` views", views) %></b> + <% end %> <% end %> </div> </h5> |