diff options
Diffstat (limited to 'src')
36 files changed, 1865 insertions, 1868 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 284b238c..8d4c2e58 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -64,7 +64,7 @@ HTTP_CHUNK_SIZE = 10485760 # ~10MB CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} -CURRENT_VERSION = {{ "#{`git describe --tags --abbrev=0`.strip}" }} +CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }} # This is used to determine the `?v=` on the end of file URLs (for cache busting). We # only need to expire modified assets, so we can use this to find the last commit that changes @@ -104,10 +104,11 @@ LOCALES = { "zh-TW" => load_locale("zh-TW"), } -YT_POOL = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.1) +YT_POOL = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 2.0) config = CONFIG -logger = Invidious::LogHandler.new +output = STDOUT +loglvl = LogLevel::Debug Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" @@ -127,11 +128,14 @@ Kemal.config.extra_options do |parser| exit end end - parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: STDOUT)") do |output| - FileUtils.mkdir_p(File.dirname(output)) - logger = Invidious::LogHandler.new(File.open(output, mode: "a")) + parser.on("-o OUTPUT", "--output=OUTPUT", "Redirect output (default: STDOUT)") do |output_arg| + FileUtils.mkdir_p(File.dirname(output_arg)) + output = File.open(output_arg, mode: "a") end - parser.on("-v", "--version", "Print version") do |output| + parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{loglvl})") do |loglvl_arg| + loglvl = LogLevel.parse(loglvl_arg) + end + parser.on("-v", "--version", "Print version") do puts SOFTWARE.to_pretty_json exit end @@ -139,6 +143,8 @@ end Kemal::CLI.new ARGV +logger = Invidious::LogHandler.new(output, loglvl) + # Check table integrity if CONFIG.check_tables check_enum(PG_DB, logger, "privacy", PlaylistPrivacy) @@ -162,13 +168,16 @@ end Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB, logger, config) Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB, logger, config) Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, logger, config, HMAC_KEY) -Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new if config.statistics_enabled Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, config, SOFTWARE) end +if config.popular_enabled + Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) +end + if config.captcha_key Invidious::Jobs.register Invidious::Jobs::BypassCaptchaJob.new(logger, config) end @@ -197,6 +206,7 @@ before_all do |env| extra_media_csp = "" if CONFIG.disabled?("local") || !preferences.local extra_media_csp += " https://*.googlevideo.com:443" + extra_media_csp += " https://*.youtube.com:443" end # TODO: Remove style-src's 'unsafe-inline', requires to remove all inline styles (<style> [..] </style>, style=" [..] ") env.response.headers["Content-Security-Policy"] = "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; connect-src 'self'; manifest-src 'self'; media-src 'self' blob:#{extra_media_csp}" @@ -249,7 +259,7 @@ before_all do |env| headers["Cookie"] = env.request.headers["Cookie"] begin - user, sid = get_user(sid, headers, PG_DB, false) + user, sid = get_user(sid, headers, PG_DB, logger, false) csrf_token = generate_response(sid, { ":authorize_token", ":playlist_ajax", @@ -299,1394 +309,30 @@ Invidious::Routing.get "/licenses", Invidious::Routes::Licenses Invidious::Routing.get "/watch", Invidious::Routes::Watch Invidious::Routing.get "/embed/", Invidious::Routes::Embed::Index Invidious::Routing.get "/embed/:id", Invidious::Routes::Embed::Show - -# Playlists - -get "/feed/playlists" do |env| - env.redirect "/view_all_playlists" -end - -get "/view_all_playlists" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - - items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) - items_created.map! do |item| - item.author = "" - item - end - - items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) - items_saved.map! do |item| - item.author = "" - item - end - - templated "view_all_playlists" -end - -get "/create_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - sid = sid.as(String) - csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY, PG_DB) - - templated "create_playlist" -end - -post "/create_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" - end - - title = env.params.body["title"]?.try &.as(String) - if !title || title.empty? - error_message = "Title cannot be empty." - next templated "error" - end - - privacy = PlaylistPrivacy.parse?(env.params.body["privacy"]?.try &.as(String) || "") - if !privacy - error_message = "Invalid privacy setting." - next templated "error" - end - - if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 - error_message = "User cannot have more than 100 playlists." - next templated "error" - end - - playlist = create_playlist(PG_DB, title, privacy, user) - - env.redirect "/playlist?list=#{playlist.id}" -end - -get "/subscribe_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - - playlist_id = env.params.query["list"] - playlist = get_playlist(PG_DB, playlist_id, locale) - subscribe_playlist(PG_DB, user, playlist) - - env.redirect "/playlist?list=#{playlist.id}" -end - -get "/delete_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - sid = sid.as(String) - - plid = env.params.query["list"]? - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email - next env.redirect referer - end - - csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY, PG_DB) - - templated "delete_playlist" -end - -post "/delete_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - plid = env.params.query["list"]? - if !plid - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" - end - - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email - next env.redirect referer - end - - PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) - PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) - - env.redirect "/view_all_playlists" -end - -get "/edit_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - sid = sid.as(String) - - plid = env.params.query["list"]? - if !plid || !plid.starts_with?("IV") - next env.redirect referer - end - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email - next env.redirect referer - end - rescue ex - next env.redirect referer - end - - begin - videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) - rescue ex - videos = [] of PlaylistVideo - end - - csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB) - - templated "edit_playlist" -end - -post "/edit_playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - plid = env.params.query["list"]? - if !plid - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" - end - - playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email - next env.redirect referer - end - - title = env.params.body["title"]?.try &.delete("<>") || "" - privacy = PlaylistPrivacy.parse(env.params.body["privacy"]? || "Public") - description = env.params.body["description"]?.try &.delete("\r") || "" - - if title != playlist.title || - privacy != playlist.privacy || - description != playlist.description - updated = Time.utc - else - updated = playlist.updated - end - - PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) - - env.redirect "/playlist?list=#{plid}" -end - -get "/add_playlist_items" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect "/" - end - - user = user.as(User) - sid = sid.as(String) - - plid = env.params.query["list"]? - if !plid || !plid.starts_with?("IV") - next env.redirect referer - end - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - begin - playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - if !playlist || playlist.author != user.email - next env.redirect referer - end - rescue ex - next env.redirect referer - end - - query = env.params.query["q"]? - if query - begin - search_query, count, items = process_search_query(query, page, user, region: nil) - videos = items.select { |item| item.is_a? SearchVideo }.map { |item| item.as(SearchVideo) } - rescue ex - videos = [] of SearchVideo - count = 0 - end - else - videos = [] of SearchVideo - count = 0 - end - - env.set "add_playlist_items", plid - templated "add_playlist_items" -end - -post "/playlist_ajax" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env, "/") - - redirect = env.params.query["redirect"]? - redirect ||= "true" - redirect = redirect == "true" - - if !user - if redirect - next env.redirect referer - else - error_message = {"error" => "No such user"}.to_json - env.response.status_code = 403 - next error_message - end - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - if redirect - error_message = ex.message - env.response.status_code = 400 - next templated "error" - else - error_message = {"error" => ex.message}.to_json - env.response.status_code = 400 - next error_message - end - end - - if env.params.query["action_create_playlist"]? - action = "action_create_playlist" - elsif env.params.query["action_delete_playlist"]? - action = "action_delete_playlist" - elsif env.params.query["action_edit_playlist"]? - action = "action_edit_playlist" - elsif env.params.query["action_add_video"]? - action = "action_add_video" - video_id = env.params.query["video_id"] - elsif env.params.query["action_remove_video"]? - action = "action_remove_video" - elsif env.params.query["action_move_video_before"]? - action = "action_move_video_before" - else - next env.redirect referer - end - - begin - playlist_id = env.params.query["playlist_id"] - playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist) - raise "Invalid user" if playlist.author != user.email - rescue ex - if redirect - error_message = ex.message - env.response.status_code = 400 - next templated "error" - else - error_message = {"error" => ex.message}.to_json - env.response.status_code = 400 - next error_message - end - end - - if !user.password - # TODO: Playlist stub, sync with YouTube for Google accounts - # playlist_ajax(playlist_id, action, env.request.headers) - end - email = user.email - - case action - when "action_edit_playlist" - # TODO: Playlist stub - when "action_add_video" - if playlist.index.size >= 500 - env.response.status_code = 400 - if redirect - error_message = "Playlist cannot have more than 500 videos" - next templated "error" - else - error_message = {"error" => "Playlist cannot have more than 500 videos"}.to_json - next error_message - end - end - - video_id = env.params.query["video_id"] - - begin - video = get_video(video_id, PG_DB) - rescue ex - env.response.status_code = 500 - if redirect - error_message = ex.message - next templated "error" - else - error_message = {"error" => ex.message}.to_json - next error_message - end - end - - playlist_video = PlaylistVideo.new({ - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, - length_seconds: video.length_seconds, - published: video.published, - plid: playlist_id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX), - }) - - video_array = playlist_video.to_a - args = arg_array(video_array) - - PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) - PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id) - when "action_remove_video" - index = env.params.query["set_video_id"] - PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) - PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id) - when "action_move_video_before" - # TODO: Playlist stub - else - error_message = {"error" => "Unsupported action #{action}"}.to_json - env.response.status_code = 400 - next error_message - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - -get "/playlist" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get?("user").try &.as(User) - referer = get_referer(env) - - plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") - if !plid - next env.redirect "/" - end - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - if plid.starts_with? "RD" - next env.redirect "/mix?list=#{plid}" - end - - begin - playlist = get_playlist(PG_DB, plid, locale) - rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" - end - - if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email - error_message = "This playlist is private." - env.response.status_code = 403 - next templated "error" - end - - begin - videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) - rescue ex - videos = [] of PlaylistVideo - end - - if playlist.author == user.try &.email - env.set "remove_playlist_items", plid - end - - templated "playlist" -end - -get "/mix" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - rdid = env.params.query["list"]? - if !rdid - next env.redirect "/" - end - - continuation = env.params.query["continuation"]? - continuation ||= rdid.lchop("RD") - - begin - mix = fetch_mix(rdid, continuation, locale: locale) - rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" - end - - templated "mix" -end - -# Search - -get "/opensearch.xml" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.response.content_type = "application/opensearchdescription+xml" - - XML.build(indent: " ", encoding: "UTF-8") do |xml| - xml.element("OpenSearchDescription", xmlns: "http://a9.com/-/spec/opensearch/1.1/") do - xml.element("ShortName") { xml.text "Invidious" } - xml.element("LongName") { xml.text "Invidious Search" } - xml.element("Description") { xml.text "Search for videos, channels, and playlists on Invidious" } - xml.element("InputEncoding") { xml.text "UTF-8" } - xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "#{HOST_URL}/favicon.ico" } - xml.element("Url", type: "text/html", method: "get", template: "#{HOST_URL}/search?q={searchTerms}") - end - end -end - -get "/results" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - query = env.params.query["search_query"]? - query ||= env.params.query["q"]? - query ||= "" - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - if query - env.redirect "/search?q=#{URI.encode_www_form(query)}&page=#{page}" - else - env.redirect "/" - end -end - -get "/search" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - region = env.params.query["region"]? - - query = env.params.query["search_query"]? - query ||= env.params.query["q"]? - query ||= "" - - if query.empty? - next env.redirect "/" - end - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - user = env.get? "user" - - begin - search_query, count, videos = process_search_query(query, page, user, region: nil) - rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" - end - - env.set "search", query - templated "search" -end +Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Playlists, :index +Invidious::Routing.get "/create_playlist", Invidious::Routes::Playlists, :new +Invidious::Routing.post "/create_playlist", Invidious::Routes::Playlists, :create +Invidious::Routing.get "/subscribe_playlist", Invidious::Routes::Playlists, :subscribe +Invidious::Routing.get "/delete_playlist", Invidious::Routes::Playlists, :delete_page +Invidious::Routing.post "/delete_playlist", Invidious::Routes::Playlists, :delete +Invidious::Routing.get "/edit_playlist", Invidious::Routes::Playlists, :edit +Invidious::Routing.post "/edit_playlist", Invidious::Routes::Playlists, :update +Invidious::Routing.get "/add_playlist_items", Invidious::Routes::Playlists, :add_playlist_items_page +Invidious::Routing.post "/playlist_ajax", Invidious::Routes::Playlists, :playlist_ajax +Invidious::Routing.get "/playlist", Invidious::Routes::Playlists, :show +Invidious::Routing.get "/mix", Invidious::Routes::Playlists, :mix +Invidious::Routing.get "/opensearch.xml", Invidious::Routes::Search, :opensearch +Invidious::Routing.get "/results", Invidious::Routes::Search, :results +Invidious::Routing.get "/search", Invidious::Routes::Search, :search +Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page +Invidious::Routing.post "/login", Invidious::Routes::Login, :login +Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout +Invidious::Routing.get "/preferences", Invidious::Routes::UserPreferences, :show +Invidious::Routing.post "/preferences", Invidious::Routes::UserPreferences, :update +Invidious::Routing.get "/toggle_theme", Invidious::Routes::UserPreferences, :toggle_theme # Users -get "/login" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - if user - next env.redirect "/feed/subscriptions" - end - - if !config.login_enabled - error_message = "Login has been disabled by administrator." - env.response.status_code = 400 - next templated "error" - end - - referer = get_referer(env, "/feed/subscriptions") - - email = nil - password = nil - captcha = nil - - account_type = env.params.query["type"]? - account_type ||= "invidious" - - captcha_type = env.params.query["captcha"]? - captcha_type ||= "image" - - tfa = env.params.query["tfa"]? - prompt = nil - - templated "login" -end - -post "/login" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - referer = get_referer(env, "/feed/subscriptions") - - if !config.login_enabled - error_message = "Login has been disabled by administrator." - env.response.status_code = 403 - next templated "error" - end - - # https://stackoverflow.com/a/574698 - email = env.params.body["email"]?.try &.downcase.byte_slice(0, 254) - password = env.params.body["password"]? - - account_type = env.params.query["type"]? - account_type ||= "invidious" - - case account_type - when "google" - tfa_code = env.params.body["tfa"]?.try &.lchop("G-") - traceback = IO::Memory.new - - # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82 - begin - client = QUIC::Client.new(LOGIN_URL) - headers = HTTP::Headers.new - - login_page = client.get("/ServiceLogin") - headers = login_page.cookies.add_request_headers(headers) - - lookup_req = { - email, nil, [] of String, nil, "US", nil, nil, 2, false, true, - {nil, nil, - {2, 1, nil, 1, - "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn", - nil, [] of String, 4}, - 1, - {nil, nil, [] of String}, - nil, nil, nil, true, - }, - email, - }.to_json - - traceback << "Getting lookup..." - - headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8" - headers["Google-Accounts-XSRF"] = "1" - - response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req)) - lookup_results = JSON.parse(response.body[5..-1]) - - traceback << "done, returned #{response.status_code}.<br/>" - - user_hash = lookup_results[0][2] - - if token = env.params.body["token"]? - answer = env.params.body["answer"]? - captcha = {token, answer} - else - captcha = nil - end - - challenge_req = { - user_hash, nil, 1, nil, - {1, nil, nil, nil, - {password, captcha, true}, - }, - {nil, nil, - {2, 1, nil, 1, - "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn", - nil, [] of String, 4}, - 1, - {nil, nil, [] of String}, - nil, nil, nil, true, - }, - }.to_json - - traceback << "Getting challenge..." - - response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req)) - headers = response.cookies.add_request_headers(headers) - challenge_results = JSON.parse(response.body[5..-1]) - - traceback << "done, returned #{response.status_code}.<br/>" - - headers["Cookie"] = URI.decode_www_form(headers["Cookie"]) - - if challenge_results[0][3]?.try &.== 7 - error_message = translate(locale, "Account has temporarily been disabled") - env.response.status_code = 423 - next templated "error" - end - - if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s - account_type = "google" - captcha_type = "image" - prompt = nil - tfa = tfa_code - captcha = {tokens: [token], question: ""} - - next templated "login" - end - - if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED" - error_message = translate(locale, "Incorrect password") - env.response.status_code = 401 - next templated "error" - end - - prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]? - if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type - traceback << "Handling prompt #{prompt_type}.<br/>" - case prompt_type - when "TWO_STEP_VERIFICATION" - prompt_type = 2 - else # "LOGIN_CHALLENGE" - prompt_type = 4 - end - - # Prefer Authenticator app and SMS over unsupported protocols - if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2 - tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0] - - traceback << "Selecting challenge #{tfa[8]}..." - select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json - - tl = challenge_results[1][2] - - tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body - tfa = tfa[5..-1] - tfa = JSON.parse(tfa)[0][-1] - - traceback << "done.<br/>" - else - traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.<br/>" - tfa = challenge_results[0][-1][0][0] - end - - if tfa[5] == "QUOTA_EXCEEDED" - error_message = translate(locale, "Quota exceeded, try again in a few hours") - env.response.status_code = 423 - next templated "error" - end - - if !tfa_code - account_type = "google" - captcha_type = "image" - - case tfa[8] - when 6, 9 - prompt = "Google verification code" - when 12 - prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" - when 15 - prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" - else - prompt = "Google verification code" - end - - tfa = nil - captcha = nil - next templated "login" - end - - tl = challenge_results[1][2] - - request_type = tfa[8] - case request_type - when 6 # Authenticator app - tfa_req = { - user_hash, nil, 2, nil, - {6, nil, nil, nil, nil, - {tfa_code, false}, - }, - }.to_json - when 9 # Voice or text message - tfa_req = { - user_hash, nil, 2, nil, - {9, nil, nil, nil, nil, nil, nil, nil, - {nil, tfa_code, false, 2}, - }, - }.to_json - when 12 # Recovery email - tfa_req = { - user_hash, nil, 4, nil, - {12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - {tfa_code}, - }, - }.to_json - when 15 # Security question - tfa_req = { - user_hash, nil, 5, nil, - {15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - {tfa_code}, - }, - }.to_json - else - error_message = translate(locale, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.") - env.response.status_code = 500 - next templated "error" - end - - traceback << "Submitting challenge..." - - response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req)) - headers = response.cookies.add_request_headers(headers) - challenge_results = JSON.parse(response.body[5..-1]) - - if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") || - (challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT") - error_message = translate(locale, "Invalid TFA code") - env.response.status_code = 401 - next templated "error" - end - - traceback << "done.<br/>" - end - - traceback << "Logging in..." - - location = URI.parse(challenge_results[0][-1][2].to_s) - cookies = HTTP::Cookies.from_headers(headers) - - headers.delete("Content-Type") - headers.delete("Google-Accounts-XSRF") - - loop do - if !location || location.path == "/ManageAccount" - break - end - - # Occasionally there will be a second page after login confirming - # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle. - - if location.path.starts_with? "/b/0/SmsAuthInterstitial" - traceback << "Unhandled dialog /b/0/SmsAuthInterstitial." - end - - login = client.get(location.full_path, headers) - - headers = login.cookies.add_request_headers(headers) - location = login.headers["Location"]?.try { |u| URI.parse(u) } - end - - cookies = HTTP::Cookies.from_headers(headers) - sid = cookies["SID"]?.try &.value - if !sid - raise "Couldn't get SID." - end - - user, sid = get_user(sid, headers, PG_DB) - - # We are now logged in - traceback << "done.<br/>" - - host = URI.parse(env.request.headers["Host"]).host - - if Kemal.config.ssl || config.https_only - secure = true - else - secure = false - end - - cookies.each do |cookie| - if Kemal.config.ssl || config.https_only - cookie.secure = secure - else - cookie.secure = secure - end - - if cookie.extension - cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host) - cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "") - end - env.response.cookies << cookie - end - - if env.request.cookies["PREFS"]? - preferences = env.get("preferences").as(Preferences) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) - - cookie = env.request.cookies["PREFS"] - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - - env.redirect referer - rescue ex - traceback.rewind - # error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.") - error_message = %(#{ex.message}<br/>Traceback:<br/><div style="padding-left:2em" id="traceback">#{traceback.gets_to_end}</div>) - env.response.status_code = 500 - next templated "error" - end - when "invidious" - if !email - error_message = translate(locale, "User ID is a required field") - env.response.status_code = 401 - next templated "error" - end - - if !password - error_message = translate(locale, "Password is a required field") - env.response.status_code = 401 - next templated "error" - end - - user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1", email, as: User) - - if user - if !user.password - error_message = translate(locale, "Please sign in using 'Log in with Google'") - env.response.status_code = 400 - next templated "error" - end - - if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) - sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) - - if Kemal.config.ssl || config.https_only - secure = true - else - secure = false - end - - if config.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end - else - error_message = translate(locale, "Wrong username or password") - env.response.status_code = 401 - next templated "error" - end - - # Since this user has already registered, we don't want to overwrite their preferences - if env.request.cookies["PREFS"]? - cookie = env.request.cookies["PREFS"] - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - else - if !config.registration_enabled - error_message = "Registration has been disabled by administrator." - env.response.status_code = 400 - next templated "error" - end - - if password.empty? - error_message = translate(locale, "Password cannot be empty") - env.response.status_code = 401 - next templated "error" - end - - # See https://security.stackexchange.com/a/39851 - if password.bytesize > 55 - error_message = translate(locale, "Password should not be longer than 55 characters") - env.response.status_code = 400 - next templated "error" - end - - password = password.byte_slice(0, 55) - - if config.captcha_enabled - captcha_type = env.params.body["captcha_type"]? - answer = env.params.body["answer"]? - change_type = env.params.body["change_type"]? - - if !captcha_type || change_type - if change_type - captcha_type = change_type - end - captcha_type ||= "image" - - account_type = "invidious" - tfa = false - prompt = "" - - if captcha_type == "image" - captcha = generate_captcha(HMAC_KEY, PG_DB) - else - captcha = generate_text_captcha(HMAC_KEY, PG_DB) - end - - next templated "login" - end - - tokens = env.params.body.select { |k, v| k.match(/^token\[\d+\]$/) }.map { |k, v| v } - - answer ||= "" - captcha_type ||= "image" - - case captcha_type - when "image" - answer = answer.lstrip('0') - answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) - - begin - validate_request(tokens[0], answer, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" - end - else # "text" - answer = Digest::MD5.hexdigest(answer.downcase.strip) - - found_valid_captcha = false - - error_message = translate(locale, "Erroneous CAPTCHA") - tokens.each_with_index do |token, i| - begin - validate_request(token, answer, env.request, HMAC_KEY, PG_DB, locale) - found_valid_captcha = true - rescue ex - error_message = ex.message - end - end - - if !found_valid_captcha - env.response.status_code = 500 - next templated "error" - end - end - end - - sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user, sid = create_user(sid, email, password) - user_array = user.to_a - user_array[4] = user_array[4].to_json # User preferences - - args = arg_array(user_array) - - PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) - PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) - - view_name = "subscriptions_#{sha256(user.email)}" - PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") - - if Kemal.config.ssl || config.https_only - secure = true - else - secure = false - end - - if config.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end - - if env.request.cookies["PREFS"]? - preferences = env.get("preferences").as(Preferences) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) - - cookie = env.request.cookies["PREFS"] - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - end - - env.redirect referer - else - env.redirect referer - end -end - -post "/signout" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) - rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" - end - - PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", sid) - - env.request.cookies.each do |cookie| - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - - env.redirect referer -end - -get "/preferences" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - referer = get_referer(env) - - preferences = env.get("preferences").as(Preferences) - - templated "preferences" -end - -post "/preferences" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - referer = get_referer(env) - - video_loop = env.params.body["video_loop"]?.try &.as(String) - video_loop ||= "off" - video_loop = video_loop == "on" - - annotations = env.params.body["annotations"]?.try &.as(String) - annotations ||= "off" - annotations = annotations == "on" - - annotations_subscribed = env.params.body["annotations_subscribed"]?.try &.as(String) - annotations_subscribed ||= "off" - annotations_subscribed = annotations_subscribed == "on" - - autoplay = env.params.body["autoplay"]?.try &.as(String) - autoplay ||= "off" - autoplay = autoplay == "on" - - continue = env.params.body["continue"]?.try &.as(String) - continue ||= "off" - continue = continue == "on" - - continue_autoplay = env.params.body["continue_autoplay"]?.try &.as(String) - continue_autoplay ||= "off" - continue_autoplay = continue_autoplay == "on" - - listen = env.params.body["listen"]?.try &.as(String) - listen ||= "off" - listen = listen == "on" - - local = env.params.body["local"]?.try &.as(String) - local ||= "off" - local = local == "on" - - speed = env.params.body["speed"]?.try &.as(String).to_f32? - speed ||= CONFIG.default_user_preferences.speed - - player_style = env.params.body["player_style"]?.try &.as(String) - player_style ||= CONFIG.default_user_preferences.player_style - - quality = env.params.body["quality"]?.try &.as(String) - quality ||= CONFIG.default_user_preferences.quality - - volume = env.params.body["volume"]?.try &.as(String).to_i? - volume ||= CONFIG.default_user_preferences.volume - - comments = [] of String - 2.times do |i| - comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i]) - end - - captions = [] of String - 3.times do |i| - captions << (env.params.body["captions[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.captions[i]) - end - - related_videos = env.params.body["related_videos"]?.try &.as(String) - related_videos ||= "off" - related_videos = related_videos == "on" - - default_home = env.params.body["default_home"]?.try &.as(String) || CONFIG.default_user_preferences.default_home - - feed_menu = [] of String - 5.times do |index| - option = env.params.body["feed_menu[#{index}]"]?.try &.as(String) || "" - if !option.empty? - feed_menu << option - end - end - - locale = env.params.body["locale"]?.try &.as(String) - locale ||= CONFIG.default_user_preferences.locale - - dark_mode = env.params.body["dark_mode"]?.try &.as(String) - dark_mode ||= CONFIG.default_user_preferences.dark_mode - - thin_mode = env.params.body["thin_mode"]?.try &.as(String) - thin_mode ||= "off" - thin_mode = thin_mode == "on" - - max_results = env.params.body["max_results"]?.try &.as(String).to_i? - max_results ||= CONFIG.default_user_preferences.max_results - - sort = env.params.body["sort"]?.try &.as(String) - sort ||= CONFIG.default_user_preferences.sort - - latest_only = env.params.body["latest_only"]?.try &.as(String) - latest_only ||= "off" - latest_only = latest_only == "on" - - unseen_only = env.params.body["unseen_only"]?.try &.as(String) - unseen_only ||= "off" - unseen_only = unseen_only == "on" - - notifications_only = env.params.body["notifications_only"]?.try &.as(String) - notifications_only ||= "off" - notifications_only = notifications_only == "on" - - # Convert to JSON and back again to take advantage of converters used for compatability - preferences = Preferences.from_json({ - annotations: annotations, - annotations_subscribed: annotations_subscribed, - autoplay: autoplay, - captions: captions, - comments: comments, - continue: continue, - continue_autoplay: continue_autoplay, - dark_mode: dark_mode, - latest_only: latest_only, - listen: listen, - local: local, - locale: locale, - max_results: max_results, - notifications_only: notifications_only, - player_style: player_style, - quality: quality, - default_home: default_home, - feed_menu: feed_menu, - related_videos: related_videos, - sort: sort, - speed: speed, - thin_mode: thin_mode, - unseen_only: unseen_only, - video_loop: video_loop, - volume: volume, - }.to_json).to_json - - if user = env.get? "user" - user = user.as(User) - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) - - if config.admins.includes? user.email - config.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || config.default_user_preferences.default_home - - admin_feed_menu = [] of String - 5.times do |index| - option = env.params.body["admin_feed_menu[#{index}]"]?.try &.as(String) || "" - if !option.empty? - admin_feed_menu << option - end - end - config.default_user_preferences.feed_menu = admin_feed_menu - - captcha_enabled = env.params.body["captcha_enabled"]?.try &.as(String) - captcha_enabled ||= "off" - config.captcha_enabled = captcha_enabled == "on" - - login_enabled = env.params.body["login_enabled"]?.try &.as(String) - login_enabled ||= "off" - config.login_enabled = login_enabled == "on" - - registration_enabled = env.params.body["registration_enabled"]?.try &.as(String) - registration_enabled ||= "off" - config.registration_enabled = registration_enabled == "on" - - statistics_enabled = env.params.body["statistics_enabled"]?.try &.as(String) - statistics_enabled ||= "off" - config.statistics_enabled = statistics_enabled == "on" - - CONFIG.default_user_preferences = config.default_user_preferences - File.write("config/config.yml", config.to_yaml) - end - else - if Kemal.config.ssl || config.https_only - secure = true - else - secure = false - end - - if config.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end - end - - env.redirect referer -end - -get "/toggle_theme" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - referer = get_referer(env, unroll: false) - - redirect = env.params.query["redirect"]? - redirect ||= "true" - redirect = redirect == "true" - - if user = env.get? "user" - user = user.as(User) - preferences = user.preferences - - case preferences.dark_mode - when "dark" - preferences.dark_mode = "light" - else - preferences.dark_mode = "dark" - end - - preferences = preferences.to_json - - PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) - else - preferences = env.get("preferences").as(Preferences) - - case preferences.dark_mode - when "dark" - preferences.dark_mode = "light" - else - preferences.dark_mode = "dark" - end - - preferences = preferences.to_json - - if Kemal.config.ssl || config.https_only - secure = true - else - secure = false - end - - if config.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - post "/watch_ajax" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -1702,9 +348,7 @@ post "/watch_ajax" do |env| if redirect next env.redirect referer else - error_message = {"error" => "No such user"}.to_json - env.response.status_code = 403 - next error_message + next error_json(403, "No such user") end end @@ -1721,13 +365,10 @@ post "/watch_ajax" do |env| begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex - env.response.status_code = 400 if redirect - error_message = ex.message - next templated "error" + next error_template(400, ex) else - error_message = {"error" => ex.message}.to_json - next error_message + next error_json(400, ex) end end @@ -1747,9 +388,7 @@ post "/watch_ajax" do |env| when "action_mark_unwatched" PG_DB.exec("UPDATE users SET watched = array_remove(watched, $1) WHERE email = $2", id, user.email) else - error_message = {"error" => "Unsupported action #{action}"}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Unsupported action #{action}") end if redirect @@ -1779,9 +418,7 @@ get "/modify_notifications" do |env| if redirect next env.redirect referer else - error_message = {"error" => "No such user"}.to_json - env.response.status_code = 403 - next error_message + next error_json(403, "No such user") end end @@ -1854,9 +491,7 @@ post "/subscription_ajax" do |env| if redirect next env.redirect referer else - error_message = {"error" => "No such user"}.to_json - env.response.status_code = 403 - next error_message + next error_json(403, "No such user") end end @@ -1868,13 +503,9 @@ post "/subscription_ajax" do |env| validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex if redirect - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) else - error_message = {"error" => ex.message}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, ex) end end @@ -1898,15 +529,13 @@ post "/subscription_ajax" do |env| case action when "action_create_subscription_to_channel" if !user.subscriptions.includes? channel_id - get_channel(channel_id, PG_DB, false, false) + get_channel(channel_id, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions, $1) WHERE email = $2", channel_id, email) end when "action_remove_subscriptions" PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_remove(subscriptions, $1) WHERE email = $2", channel_id, email) else - error_message = {"error" => "Unsupported action #{action}"}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Unsupported action #{action}") end if redirect @@ -1935,7 +564,7 @@ get "/subscription_manager" do |env| headers = HTTP::Headers.new headers["Cookie"] = env.request.headers["Cookie"] - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers, PG_DB, logger) end action_takeout = env.params.query["action_takeout"]?.try &.to_i? @@ -2059,7 +688,7 @@ post "/data_control" do |env| user.subscriptions += body["subscriptions"].as_a.map { |a| a.as_s } user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) end @@ -2089,7 +718,7 @@ post "/data_control" do |env| PG_DB.exec("UPDATE playlists SET description = $1 WHERE id = $2", description, playlist.id) videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| - raise "Playlist cannot have more than 500 videos" if idx > 500 + raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 video_id = video_id.try &.as_s? next if !video_id @@ -2121,13 +750,14 @@ post "/data_control" do |env| end end when "import_youtube" - subscriptions = XML.parse(body) - user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + subscriptions = JSON.parse(body) + + user.subscriptions += subscriptions.as_a.compact_map do |entry| + entry["snippet"]["resourceId"]["channelId"].as_s end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_freetube" @@ -2136,7 +766,7 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_newpipe_subscriptions" @@ -2155,7 +785,7 @@ post "/data_control" do |env| end user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_newpipe" @@ -2174,7 +804,7 @@ post "/data_control" do |env| user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map { |url| url.lchop("https://www.youtube.com/channel/") } user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, false, false) + user.subscriptions = get_batch_channels(user.subscriptions, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) @@ -2226,51 +856,37 @@ post "/change_password" do |env| # We don't store passwords for Google accounts if !user.password - error_message = "Cannot change password for Google accounts" - env.response.status_code = 400 - next templated "error" + next error_template(400, "Cannot change password for Google accounts") end begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) end password = env.params.body["password"]? if !password - error_message = translate(locale, "Password is a required field") - env.response.status_code = 401 - next templated "error" + next error_template(401, "Password is a required field") end new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } if new_passwords.size <= 1 || new_passwords.uniq.size != 1 - error_message = translate(locale, "New passwords must match") - env.response.status_code = 400 - next templated "error" + next error_template(400, "New passwords must match") end new_password = new_passwords.uniq[0] if new_password.empty? - error_message = translate(locale, "Password cannot be empty") - env.response.status_code = 401 - next templated "error" + next error_template(401, "Password cannot be empty") end if new_password.bytesize > 55 - error_message = translate(locale, "Password should not be longer than 55 characters") - env.response.status_code = 400 - next templated "error" + next error_template(400, "Password cannot be longer than 55 characters") end if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) - error_message = translate(locale, "Incorrect password") - env.response.status_code = 401 - next templated "error" + next error_template(401, "Incorrect password") end new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) @@ -2315,9 +931,7 @@ post "/delete_account" do |env| begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) end view_name = "subscriptions_#{sha256(user.email)}" @@ -2369,9 +983,7 @@ post "/clear_watch_history" do |env| begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) end PG_DB.exec("UPDATE users SET watched = '{}' WHERE email = $1", user.email) @@ -2424,9 +1036,7 @@ post "/authorize_token" do |env| begin validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) end scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } @@ -2489,9 +1099,7 @@ post "/token_ajax" do |env| if redirect next env.redirect referer else - error_message = {"error" => "No such user"}.to_json - env.response.status_code = 403 - next error_message + next error_json(403, "No such user") end end @@ -2503,13 +1111,9 @@ post "/token_ajax" do |env| validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) rescue ex if redirect - error_message = ex.message - env.response.status_code = 400 - next templated "error" + next error_template(400, ex) else - error_message = {"error" => ex.message}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, ex) end end @@ -2526,9 +1130,7 @@ post "/token_ajax" do |env| when .starts_with? "action_revoke_token" PG_DB.exec("DELETE FROM session_ids * WHERE id = $1 AND email = $2", session, user.email) else - error_message = {"error" => "Unsupported action #{action}"}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Unsupported action #{action}") end if redirect @@ -2541,15 +1143,26 @@ end # Feeds +get "/feed/playlists" do |env| + env.redirect "/view_all_playlists" +end + get "/feed/top" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - env.redirect "/" + + message = translate(locale, "The Top feed has been removed from Invidious.") + templated "message" end get "/feed/popular" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - templated "popular" + if config.popular_enabled + templated "popular" + else + message = translate(locale, "The Popular feed has been disabled by the administrator.") + templated "message" + end end get "/feed/trending" do |env| @@ -2564,9 +1177,7 @@ get "/feed/trending" do |env| begin trending, plid = fetch_trending(trending_type, region, locale) rescue ex - error_message = "#{ex.message}" - env.response.status_code = 500 - next templated "error" + next error_template(500, ex) end templated "trending" @@ -2596,7 +1207,7 @@ get "/feed/subscriptions" do |env| headers["Cookie"] = env.request.headers["Cookie"] if !user.password - user, sid = get_user(sid, headers, PG_DB) + user, sid = get_user(sid, headers, PG_DB, logger) end max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) @@ -2661,9 +1272,7 @@ get "/feed/channel/:ucid" do |env| rescue ex : ChannelRedirect next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - next error_message + next error_atom(500, ex) end response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}") @@ -2704,6 +1313,7 @@ get "/feed/channel/:ucid" do |env| xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") xml.element("id") { xml.text "yt:channel:#{channel.ucid}" } xml.element("yt:channelId") { xml.text channel.ucid } + xml.element("icon") { xml.text channel.author_thumbnail } xml.element("title") { xml.text channel.author } xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}") @@ -2901,7 +1511,7 @@ post "/feed/webhook/:token" do |env| signature = env.request.headers["X-Hub-Signature"].lchop("sha1=") if signature != OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, body) - logger.puts("#{token} : Invalid signature") + logger.error("/feed/webhook/#{token} : Invalid signature") env.response.status_code = 200 next end @@ -3074,9 +1684,7 @@ get "/channel/:ucid" do |env| rescue ex : ChannelRedirect next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" + next error_template(500, ex) end if channel.auto_generated @@ -3143,9 +1751,7 @@ get "/channel/:ucid/playlists" do |env| rescue ex : ChannelRedirect next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" + next error_template(500, ex) end if channel.auto_generated @@ -3183,9 +1789,7 @@ get "/channel/:ucid/community" do |env| rescue ex : ChannelRedirect next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - next templated "error" + next error_template(500, ex) end if !channel.tabs.includes? "community" @@ -3194,9 +1798,11 @@ get "/channel/:ucid/community" do |env| begin items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) - rescue ex + rescue ex : InfoException env.response.status_code = 500 error_message = ex.message + rescue ex + next error_template(500, ex) end env.set "search", "channel:#{channel.ucid} " @@ -3206,12 +1812,11 @@ end # API Endpoints get "/api/v1/stats" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? env.response.content_type = "application/json" if !config.statistics_enabled - error_message = {"error" => "Statistics are not enabled."}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Statistics are not enabled.") end Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json @@ -3231,10 +1836,8 @@ get "/api/v1/storyboards/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - next error_message + next error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex env.response.status_code = 500 next @@ -3280,14 +1883,14 @@ get "/api/v1/storyboards/:id" do |env| storyboard[:storyboard_count].times do |i| url = storyboard[:url] authority = /(i\d?).ytimg.com/.match(url).not_nil![1]? - url = storyboard[:url].gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") + url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") url = "#{HOST_URL}/sb/#{authority}/#{url}" storyboard[:storyboard_height].times do |j| storyboard[:storyboard_width].times do |k| str << <<-END_CUE #{start_time}.000 --> #{end_time}.000 - #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width]},#{storyboard[:height]} + #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]} END_CUE @@ -3319,10 +1922,8 @@ get "/api/v1/captions/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - next error_message + next error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex env.response.status_code = 500 next @@ -3454,9 +2055,7 @@ get "/api/v1/comments/:id" do |env| begin comments = fetch_youtube_comments(id, PG_DB, continuation, format, locale, thin_mode, region, sort_by: sort_by) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end next comments @@ -3499,13 +2098,7 @@ end get "/api/v1/insights/:id" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - id = env.params.url["id"] - env.response.content_type = "application/json" - - error_message = {"error" => "YouTube has removed publicly available analytics."}.to_json - env.response.status_code = 410 - error_message + next error_json(410, "YouTube has removed publicly available analytics.") end get "/api/v1/annotations/:id" do |env| @@ -3540,14 +2133,13 @@ get "/api/v1/annotations/:id" do |env| file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") - client = make_client(ARCHIVE_URL) - location = client.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}") + location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) if !location.headers["Location"]? env.response.status_code = location.status_code end - response = make_client(URI.parse(location.headers["Location"])).get(location.headers["Location"]) + response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"])) if response.body.empty? env.response.status_code = 404 @@ -3594,14 +2186,10 @@ get "/api/v1/videos/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) - next error_message + next error_json(302, "Video is unavailable", {"videoId" => ex.video_id}) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end video.to_json(locale) @@ -3618,9 +2206,7 @@ get "/api/v1/trending" do |env| begin trending, plid = fetch_trending(trending_type, region, locale) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end videos = JSON.build do |json| @@ -3639,6 +2225,12 @@ get "/api/v1/popular" do |env| env.response.content_type = "application/json" + if !config.popular_enabled + error_message = {"error" => "Administrator has disabled this endpoint."}.to_json + env.response.status_code = 400 + next error_message + end + JSON.build do |json| json.array do popular_videos.each do |video| @@ -3652,7 +2244,8 @@ get "/api/v1/top" do |env| locale = LOCALES[env.get("preferences").as(Preferences).locale]? env.response.content_type = "application/json" - "[]" + env.response.status_code = 400 + {"error" => "The Top feed has been removed from Invidious."}.to_json end get "/api/v1/channels/:ucid" do |env| @@ -3667,14 +2260,10 @@ get "/api/v1/channels/:ucid" do |env| begin channel = get_about_info(ucid, locale) rescue ex : ChannelRedirect - error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - next error_message + next error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end page = 1 @@ -3685,9 +2274,7 @@ get "/api/v1/channels/:ucid" do |env| begin count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end end @@ -3802,22 +2389,16 @@ end begin channel = get_about_info(ucid, locale) rescue ex : ChannelRedirect - error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - next error_message + next error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end begin count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end JSON.build do |json| @@ -3841,9 +2422,7 @@ end begin videos = get_latest_videos(ucid) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end JSON.build do |json| @@ -3871,14 +2450,10 @@ end begin channel = get_about_info(ucid, locale) rescue ex : ChannelRedirect - error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json - env.response.status_code = 302 env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) - next error_message + next error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end items, continuation = fetch_channel_playlists(channel.ucid, channel.author, channel.auto_generated, continuation, sort_by) @@ -3919,9 +2494,7 @@ end begin fetch_channel_community(ucid, continuation, locale, format, thin_mode) rescue ex - env.response.status_code = 400 - error_message = {"error" => ex.message}.to_json - next error_message + next error_json(500, ex) end end end @@ -3979,9 +2552,7 @@ get "/api/v1/search" do |env| begin search_params = produce_search_params(sort_by, date, content_type, duration, features) rescue ex - env.response.status_code = 400 - error_message = {"error" => ex.message}.to_json - next error_message + next error_json(400, ex) end count, search_results = search(query, page, search_params, region).as(Tuple) @@ -4024,9 +2595,7 @@ get "/api/v1/search/suggestions" do |env| end end rescue ex - env.response.status_code = 500 - error_message = {"error" => ex.message}.to_json - next error_message + next error_json(500, ex) end end @@ -4053,16 +2622,12 @@ end begin playlist = get_playlist(PG_DB, plid, locale) rescue ex - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end user = env.get?("user").try &.as(User) if !playlist || playlist.privacy.private? && playlist.author != user.try &.email - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end response = playlist.to_json(offset, locale, continuation: continuation) @@ -4106,9 +2671,7 @@ get "/api/v1/mixes/:rdid" do |env| mix.videos = mix.videos[index..-1] rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end response = JSON.build do |json| @@ -4264,7 +2827,7 @@ post "/api/v1/auth/subscriptions/:ucid" do |env| ucid = env.params.url["ucid"] if !user.subscriptions.includes? ucid - get_channel(ucid, PG_DB, false, false) + get_channel(ucid, PG_DB, logger, false, false) PG_DB.exec("UPDATE users SET feed_needs_update = true, subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) end @@ -4310,22 +2873,16 @@ post "/api/v1/auth/playlists" do |env| title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150) if !title - error_message = {"error" => "Invalid title."}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Invalid title.") end privacy = env.params.json["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String).downcase) } if !privacy - error_message = {"error" => "Invalid privacy setting."}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Invalid privacy setting.") end if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 - error_message = {"error" => "User cannot have more than 100 playlists."}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "User cannot have more than 100 playlists.") end playlist = create_playlist(PG_DB, title, privacy, user) @@ -4347,15 +2904,11 @@ patch "/api/v1/auth/playlists/:plid" do |env| playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if !playlist || playlist.author != user.email && playlist.privacy.private? - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end if playlist.author != user.email - env.response.status_code = 403 - error_message = {"error" => "Invalid user"}.to_json - next error_message + next error_json(403, "Invalid user") end title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title @@ -4375,6 +2928,8 @@ patch "/api/v1/auth/playlists/:plid" do |env| end delete "/api/v1/auth/playlists/:plid" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + env.response.content_type = "application/json" user = env.get("user").as(User) @@ -4382,15 +2937,11 @@ delete "/api/v1/auth/playlists/:plid" do |env| playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if !playlist || playlist.author != user.email && playlist.privacy.private? - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end if playlist.author != user.email - env.response.status_code = 403 - error_message = {"error" => "Invalid user"}.to_json - next error_message + next error_json(403, "Invalid user") end PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) @@ -4409,36 +2960,26 @@ post "/api/v1/auth/playlists/:plid/videos" do |env| playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if !playlist || playlist.author != user.email && playlist.privacy.private? - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end if playlist.author != user.email - env.response.status_code = 403 - error_message = {"error" => "Invalid user"}.to_json - next error_message + next error_json(403, "Invalid user") end if playlist.index.size >= 500 - env.response.status_code = 400 - error_message = {"error" => "Playlist cannot have more than 500 videos"}.to_json - next error_message + next error_json(400, "Playlist cannot have more than 500 videos") end video_id = env.params.json["videoId"].try &.as(String) if !video_id - env.response.status_code = 403 - error_message = {"error" => "Invalid videoId"}.to_json - next error_message + next error_json(403, "Invalid videoId") end begin video = get_video(video_id, PG_DB) rescue ex - error_message = {"error" => ex.message}.to_json - env.response.status_code = 500 - next error_message + next error_json(500, ex) end playlist_video = PlaylistVideo.new({ @@ -4465,6 +3006,8 @@ post "/api/v1/auth/playlists/:plid/videos" do |env| end delete "/api/v1/auth/playlists/:plid/videos/:index" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + env.response.content_type = "application/json" user = env.get("user").as(User) @@ -4473,21 +3016,15 @@ delete "/api/v1/auth/playlists/:plid/videos/:index" do |env| playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if !playlist || playlist.author != user.email && playlist.privacy.private? - env.response.status_code = 404 - error_message = {"error" => "Playlist does not exist."}.to_json - next error_message + next error_json(404, "Playlist does not exist.") end if playlist.author != user.email - env.response.status_code = 403 - error_message = {"error" => "Invalid user"}.to_json - next error_message + next error_json(403, "Invalid user") end if !playlist.index.includes? index - env.response.status_code = 404 - error_message = {"error" => "Playlist does not contain index"}.to_json - next error_message + next error_json(404, "Playlist does not contain index") end PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) @@ -4533,9 +3070,7 @@ post "/api/v1/auth/tokens/register" do |env| callback_url = env.params.json["callbackUrl"]?.try &.as(String) expire = env.params.json["expire"]?.try &.as(Int64) else - error_message = {"error" => "Invalid or missing header 'Content-Type'"}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Invalid or missing header 'Content-Type'") end if callback_url && callback_url.empty? @@ -4585,6 +3120,7 @@ post "/api/v1/auth/tokens/register" do |env| end post "/api/v1/auth/tokens/unregister" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? env.response.content_type = "application/json" user = env.get("user").as(User) scopes = env.get("scopes").as(Array(String)) @@ -4598,9 +3134,7 @@ post "/api/v1/auth/tokens/unregister" do |env| elsif scopes_include_scope(scopes, "GET:tokens") PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", session) else - error_message = {"error" => "Cannot revoke session #{session}"}.to_json - env.response.status_code = 400 - next error_message + next error_json(400, "Cannot revoke session #{session}") end env.response.status_code = 204 @@ -4924,6 +3458,7 @@ get "/videoplayback/*" do |env| end get "/videoplayback" do |env| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? query_params = env.params.query fvip = query_params["fvip"]? || "3" @@ -4962,8 +3497,12 @@ get "/videoplayback" do |env| location = URI.parse(response.headers["Location"]) env.response.headers["Access-Control-Allow-Origin"] = "*" - host = "#{location.scheme}://#{location.host}" - client = make_client(URI.parse(host), region) + new_host = "#{location.scheme}://#{location.host}" + if new_host != host + host = new_host + client.close + client = make_client(URI.parse(new_host), region) + end url = "#{location.full_path}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" else @@ -4990,13 +3529,10 @@ get "/videoplayback" do |env| if url.includes? "&file=seg.ts" if CONFIG.disabled?("livestreams") - env.response.status_code = 403 - error_message = "Administrator has disabled this endpoint." - next templated "error" + next error_template(403, "Administrator has disabled this endpoint.") end begin - client = make_client(URI.parse(host), region) client.get(url, headers) do |response| response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -5024,9 +3560,7 @@ get "/videoplayback" do |env| else if query_params["title"]? && CONFIG.disabled?("downloads") || CONFIG.disabled?("dash") - env.response.status_code = 403 - error_message = "Administrator has disabled this endpoint." - next templated "error" + next error_template(403, "Administrator has disabled this endpoint.") end content_length = nil @@ -5039,8 +3573,6 @@ get "/videoplayback" do |env| chunk_end = chunk_start + HTTP_CHUNK_SIZE - 1 end - client = make_client(URI.parse(host), region) - # TODO: Record bytes written so we can restart after a chunk fails while true if !range_end && content_length @@ -5104,6 +3636,7 @@ get "/videoplayback" do |env| if ex.message != "Error reading socket: Connection reset by peer" break else + client.close client = make_client(URI.parse(host), region) end end @@ -5113,6 +3646,7 @@ get "/videoplayback" do |env| first_chunk = false end end + client.close end get "/ggpht/*" do |env| @@ -5367,14 +3901,9 @@ error 404 do |env| halt env, status_code: 302 end -error 500 do |env| - error_message = <<-END_HTML - Looks like you've found a bug in Invidious. Feel free to open a new issue - <a href="https://github.com/iv-org/invidious/issues">here</a> - or send an email to - <a href="mailto:#{CONFIG.admin_email}">#{CONFIG.admin_email}</a>. - END_HTML - templated "error" +error 500 do |env, ex| + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + error_template(500, ex) end static_headers do |response, filepath, filestat| diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 656b9953..6907ff3d 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -144,7 +144,7 @@ class ChannelRedirect < Exception end end -def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) +def get_batch_channels(channels, db, logger, refresh = false, pull_all_videos = true, max_threads = 10) finished_channel = Channel(String | Nil).new spawn do @@ -160,7 +160,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma active_threads += 1 spawn do begin - get_channel(ucid, db, refresh, pull_all_videos) + get_channel(ucid, db, logger, refresh, pull_all_videos) finished_channel.send(ucid) rescue ex finished_channel.send(nil) @@ -181,10 +181,10 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma return final end -def get_channel(id, db, refresh = true, pull_all_videos = true) +def get_channel(id, db, logger, refresh = true, pull_all_videos = true) if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) if refresh && Time.utc - channel.updated > 10.minutes - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) + channel = fetch_channel(id, db, logger, pull_all_videos: pull_all_videos) channel_array = channel.to_a args = arg_array(channel_array) @@ -192,7 +192,7 @@ def get_channel(id, db, refresh = true, pull_all_videos = true) ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array) end else - channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) + channel = fetch_channel(id, db, logger, pull_all_videos: pull_all_videos) channel_array = channel.to_a args = arg_array(channel_array) @@ -202,13 +202,17 @@ def get_channel(id, db, refresh = true, pull_all_videos = true) return channel end -def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) +def fetch_channel(ucid, db, logger, pull_all_videos = true, locale = nil) + logger.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}") + + logger.trace("fetch_channel: #{ucid} : Downloading RSS feed") rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body + logger.trace("fetch_channel: #{ucid} : Parsing RSS feed") rss = XML.parse_html(rss) author = rss.xpath_node(%q(//feed/title)) if !author - raise translate(locale, "Deleted or invalid channel") + raise InfoException.new("Deleted or invalid channel") end author = author.content @@ -219,22 +223,29 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) auto_generated = true end + logger.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") + page = 1 + logger.trace("fetch_channel: #{ucid} : Downloading channel videos page") response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) videos = [] of SearchVideo begin initial_data = JSON.parse(response.body).as_a.find &.["response"]? - raise "Could not extract JSON" if !initial_data + raise InfoException.new("Could not extract channel JSON") if !initial_data + + logger.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data") videos = extract_videos(initial_data.as_h, author, ucid) rescue ex if response.body.includes?("To continue with your YouTube experience, please fill out the form below.") || response.body.includes?("https://www.google.com/sorry/index") - raise "Could not extract channel info. Instance is likely blocked." + raise InfoException.new("Could not extract channel info. Instance is likely blocked.") end + raise ex end + logger.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") rss.xpath_nodes("//feed/entry").each do |entry| video_id = entry.xpath_node("videoid").not_nil!.content title = entry.xpath_node("title").not_nil!.content @@ -268,6 +279,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) views: views, }) + logger.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") + # We don't include the 'premiere_timestamp' here because channel pages don't include them, # meaning the above timestamp is always null was_insert = db.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ @@ -275,8 +288,13 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) updated = $4, ucid = $5, author = $6, length_seconds = $7, \ live_now = $8, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + if was_insert + logger.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") + db.exec("UPDATE users SET notifications = array_append(notifications, $1), \ + feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) + else + logger.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") + end end if pull_all_videos @@ -287,7 +305,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) loop do response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) initial_data = JSON.parse(response.body).as_a.find &.["response"]? - raise "Could not extract JSON" if !initial_data + raise InfoException.new("Could not extract channel JSON") if !initial_data videos = extract_videos(initial_data.as_h, author, ucid) count = videos.size @@ -507,8 +525,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) end if response.status_code != 200 - error_message = translate(locale, "This channel does not exist.") - raise error_message + raise InfoException.new("This channel does not exist.") end ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"] @@ -518,7 +535,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) body = initial_data["contents"]?.try &.["twoColumnBrowseResultsRenderer"]["tabs"].as_a.select { |tab| tab["tabRenderer"]?.try &.["selected"].as_bool.== true }[0]? if !body - raise "Could not extract community tab." + raise InfoException.new("Could not extract community tab.") end body = body["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"] @@ -540,7 +557,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) body["response"]["continuationContents"]["backstageCommentsContinuation"]? if !body - raise "Could not extract continuation." + raise InfoException.new("Could not extract continuation.") end end @@ -551,7 +568,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) error_message = (message["text"]["simpleText"]? || message["text"]["runs"]?.try &.[0]?.try &.["text"]?) .try &.as_s || "" - raise error_message + raise InfoException.new(error_message) end response = JSON.build do |json| @@ -634,7 +651,8 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) else video_id = attachment["videoId"].as_s - json.field "title", attachment["title"]["simpleText"].as_s + video_title = attachment["title"]["simpleText"]? || attachment["title"]["runs"]?.try &.[0]?.try &.["text"]? + json.field "title", video_title json.field "videoId", video_id json.field "videoThumbnails" do generate_thumbnails(json, video_id) @@ -656,7 +674,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) json.field "published", published.to_unix json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) - view_count = attachment["viewCountText"]["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64 + view_count = attachment["viewCountText"]?.try &.["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64 json.field "viewCount", view_count json.field "viewCountText", translate(locale, "`x` views", number_to_short_text(view_count)) @@ -775,46 +793,41 @@ def extract_channel_community_cursor(continuation) cursor end -INITDATA_PREQUERY = "window[\"ytInitialData\"] = {" - def get_about_info(ucid, locale) - about = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en") - if about.status_code != 200 - about = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en") + result = YT_POOL.client &.get("/channel/#{ucid}/about?gl=US&hl=en") + if result.status_code != 200 + result = YT_POOL.client &.get("/user/#{ucid}/about?gl=US&hl=en") end - if md = about.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/) + if md = result.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/) raise ChannelRedirect.new(channel_id: md["ucid"]) end - if about.status_code != 200 - error_message = translate(locale, "This channel does not exist.") - raise error_message + if result.status_code != 200 + raise InfoException.new("This channel does not exist.") + end + + about = XML.parse_html(result.body) + if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")])) + raise InfoException.new("This channel does not exist.") end - initdata_pre = about.body.index(INITDATA_PREQUERY) - initdata_post = initdata_pre.nil? ? nil : about.body.index("};", initdata_pre) - if initdata_post.nil? - about = XML.parse_html(about.body) + initdata = extract_initial_data(result.body) + if initdata.empty? error_message = about.xpath_node(%q(//div[@class="yt-alert-content"])).try &.content.strip error_message ||= translate(locale, "Could not get channel info.") - raise error_message + raise InfoException.new(error_message) end - initdata_pre = initdata_pre.not_nil! + INITDATA_PREQUERY.size - 1 - - initdata = JSON.parse(about.body[initdata_pre, initdata_post - initdata_pre + 1]) - about = XML.parse_html(about.body) - if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")])) - error_message = translate(locale, "This channel does not exist.") - raise error_message + if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]? + raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s) end - author = about.xpath_node(%q(//meta[@name="title"])).not_nil!["content"] - author_url = about.xpath_node(%q(//link[@rel="canonical"])).not_nil!["href"] - author_thumbnail = about.xpath_node(%q(//link[@rel="image_src"])).not_nil!["href"] + author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s + author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s + author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - ucid = about.xpath_node(%q(//meta[@itemprop="channelId"])).not_nil!["content"] + ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s # Raises a KeyError on failure. banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 407cef78..8849c87f 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -88,11 +88,11 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so "cookie" => video.cookie, } - response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US", headers, form: post_req)) + response = YT_POOL.client(region, &.post("/comment_service_ajax?action_get_comments=1&hl=en&gl=US&pbj=1", headers, form: post_req)) response = JSON.parse(response.body) if !response["response"]["continuationContents"]? - raise translate(locale, "Could not fetch comments") + raise InfoException.new("Could not fetch comments") end response = response["response"]["continuationContents"] @@ -266,9 +266,11 @@ def fetch_reddit_comments(id, sort_by = "confidence") thread = result[0].data.as(RedditListing).children[0].data.as(RedditLink) else - raise "Got error code #{search_results.status_code}" + raise InfoException.new("Could not fetch comments") end + client.close + comments = result[1].data.as(RedditListing).children return comments, thread end @@ -581,13 +583,17 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top") object = { "2:embedded" => { "2:string" => video_id, - "24:varint" => 1_i64, - "25:varint" => 1_i64, + "25:varint" => 0_i64, "28:varint" => 1_i64, "36:embedded" => { "5:varint" => -1_i64, "8:varint" => 0_i64, }, + "40:embedded" => { + "1:varint" => 4_i64, + "3:string" => "https://www.youtube.com", + "4:string" => "", + }, }, "3:varint" => 6_i64, "6:embedded" => { diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr new file mode 100644 index 00000000..4487ff8c --- /dev/null +++ b/src/invidious/helpers/errors.cr @@ -0,0 +1,103 @@ +# InfoExceptions are for displaying information to the user. +# +# An InfoException might or might not indicate that something went wrong. +# Historically Invidious didn't differentiate between these two options, so to +# maintain previous functionality InfoExceptions do not print backtraces. +class InfoException < Exception +end + +macro error_template(*args) + error_template_helper(env, config, locale, {{*args}}) +end + +def github_details(summary : String, content : String) + details = %(\n<details>) + details += %(\n<summary>#{summary}</summary>) + details += %(\n<p>) + details += %(\n \n```\n) + details += content.strip + details += %(\n```) + details += %(\n</p>) + details += %(\n</details>) + return HTML.escape(details) +end + +def error_template_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) + if exception.is_a?(InfoException) + return error_template_helper(env, config, locale, status_code, exception.message || "") + end + env.response.status_code = status_code + issue_template = %(Title: `#{exception.message} (#{exception.class})`) + issue_template += %(\nDate: `#{Time::Format::ISO_8601_DATE_TIME.format(Time.utc)}`) + issue_template += %(\nRoute: `#{env.request.resource}`) + issue_template += %(\nVersion: `#{SOFTWARE["version"]} @ #{SOFTWARE["branch"]}`) + # issue_template += github_details("Preferences", env.get("preferences").as(Preferences).to_pretty_json) + issue_template += github_details("Backtrace", exception.inspect_with_backtrace) + error_message = <<-END_HTML + Looks like you've found a bug in Invidious. Please open a new issue + <a href="https://github.com/iv-org/invidious/issues">on GitHub</a> + and include the following text in your message: + <pre style="padding: 20px; background: rgba(0, 0, 0, 0.12345);">#{issue_template}</pre> + END_HTML + return templated "error" +end + +def error_template_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) + env.response.status_code = status_code + error_message = translate(locale, message) + return templated "error" +end + +macro error_atom(*args) + error_atom_helper(env, config, locale, {{*args}}) +end + +def error_atom_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) + if exception.is_a?(InfoException) + return error_atom_helper(env, config, locale, status_code, exception.message || "") + end + env.response.content_type = "application/atom+xml" + env.response.status_code = status_code + return "<error>#{exception.inspect_with_backtrace}</error>" +end + +def error_atom_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) + env.response.content_type = "application/atom+xml" + env.response.status_code = status_code + return "<error>#{message}</error>" +end + +macro error_json(*args) + error_json_helper(env, config, locale, {{*args}}) +end + +def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil) + if exception.is_a?(InfoException) + return error_json_helper(env, config, locale, status_code, exception.message || "", additional_fields) + end + env.response.content_type = "application/json" + env.response.status_code = status_code + error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace} + if additional_fields + error_message = error_message.merge(additional_fields) + end + return error_message.to_json +end + +def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, exception : Exception) + return error_json_helper(env, config, locale, status_code, exception, nil) +end + +def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil) + env.response.content_type = "application/json" + env.response.status_code = status_code + error_message = {"error" => message} + if additional_fields + error_message = error_message.merge(additional_fields) + end + return error_message.to_json +end + +def error_json_helper(env : HTTP::Server::Context, config : Config, locale : Hash(String, JSON::Any) | Nil, status_code : Int32, message : String) + error_json_helper(env, config, locale, status_code, message, nil) +end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 62c24f3e..2da49abb 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -41,6 +41,7 @@ struct ConfigPreferences property notifications_only : Bool = false property player_style : String = "invidious" property quality : String = "hd720" + property quality_dash : String = "auto" property default_home : String = "Popular" property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"] property related_videos : Bool = true @@ -60,7 +61,7 @@ struct ConfigPreferences end end -struct Config +class Config include YAML::Serializable property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions) @@ -71,6 +72,7 @@ struct Config property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property popular_enabled : Bool = true property captcha_enabled : Bool = true property login_enabled : Bool = true property registration_enabled : Bool = true @@ -93,8 +95,9 @@ struct Config property admin_email : String = "omarroth@protonmail.com" # Email for bug reports @[YAML::Field(converter: Preferences::StringToCookies)] - property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format - property captcha_key : String? = nil # Key for Anti-Captcha + property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format + property captcha_key : String? = nil # Key for Anti-Captcha + property captcha_api_url : String = "https://api.anti-captcha.com" # API URL for Anti-Captcha def disabled?(option) case disabled = CONFIG.disable_proxy @@ -334,7 +337,7 @@ def check_enum(db, logger, enum_name, struct_type = nil) return # TODO if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) - logger.puts("CREATE TYPE #{enum_name}") + logger.info("check_enum: CREATE TYPE #{enum_name}") db.using_connection do |conn| conn.as(PG::Connection).exec_all(File.read("config/sql/#{enum_name}.sql")) @@ -347,7 +350,7 @@ def check_table(db, logger, table_name, struct_type = nil) begin db.exec("SELECT * FROM #{table_name} LIMIT 0") rescue ex - logger.puts("CREATE TABLE #{table_name}") + logger.info("check_table: check_table: CREATE TABLE #{table_name}") db.using_connection do |conn| conn.as(PG::Connection).exec_all(File.read("config/sql/#{table_name}.sql")) @@ -367,7 +370,7 @@ def check_table(db, logger, table_name, struct_type = nil) if name != column_array[i]? if !column_array[i]? new_column = column_types.select { |line| line.starts_with? name }[0] - logger.puts("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + logger.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") next end @@ -385,29 +388,29 @@ def check_table(db, logger, table_name, struct_type = nil) # There's a column we didn't expect if !new_column - logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") + logger.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]}") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") column_array = get_column_array(db, table_name) next end - logger.puts("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") + logger.info("check_table: ALTER TABLE #{table_name} ADD COLUMN #{new_column}") db.exec("ALTER TABLE #{table_name} ADD COLUMN #{new_column}") - logger.puts("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") + logger.info("check_table: UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") db.exec("UPDATE #{table_name} SET #{column_array[i]}_new=#{column_array[i]}") - logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + logger.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") - logger.puts("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") + logger.info("check_table: ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") db.exec("ALTER TABLE #{table_name} RENAME COLUMN #{column_array[i]}_new TO #{column_array[i]}") column_array = get_column_array(db, table_name) end else - logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") + logger.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column_array[i]} CASCADE") end end @@ -417,7 +420,7 @@ def check_table(db, logger, table_name, struct_type = nil) column_array.each do |column| if !struct_array.includes? column - logger.puts("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") + logger.info("check_table: ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") db.exec("ALTER TABLE #{table_name} DROP COLUMN #{column} CASCADE") end end @@ -597,12 +600,7 @@ def create_notification_stream(env, topics, connection_channel) end def extract_initial_data(body) : Hash(String, JSON::Any) - initial_data = body.match(/(window\["ytInitialData"\]|var\s+ytInitialData)\s*=\s*(?<info>.*?);+\s*\n/).try &.["info"] || "{}" - if initial_data.starts_with?("JSON.parse(\"") - return JSON.parse(JSON.parse(%({"initial_data":"#{initial_data[12..-3]}"}))["initial_data"].as_s).as_h - else - return JSON.parse(initial_data).as_h - end + return JSON.parse(body.match(/(window\["ytInitialData"\]|var\s*ytInitialData)\s*=\s*(?<info>\{.*?\});/mx).try &.["info"] || "{}").as_h end def proxy_file(response, env) diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index 52f0a22c..4e4d7306 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -1,66 +1,52 @@ require "logger" enum LogLevel + All + Trace Debug Info Warn Error + Fatal + Off end class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Warn) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug) end def call(context : HTTP::Server::Context) - time = Time.utc - call_next(context) - elapsed_text = elapsed_text(Time.utc - time) + elapsed_time = Time.measure { call_next(context) } + elapsed_text = elapsed_text(elapsed_time) - @io << time << ' ' << context.response.status_code << ' ' << context.request.method << ' ' << context.request.resource << ' ' << elapsed_text << '\n' - - if @io.is_a? File - @io.flush - end + info("#{context.response.status_code} #{context.request.method} #{context.request.resource} #{elapsed_text}") context end def puts(message : String) @io << message << '\n' - - if @io.is_a? File - @io.flush - end + @io.flush end - def write(message : String, level = @level) + def write(message : String) @io << message - - if @io.is_a? File - @io.flush - end + @io.flush end def set_log_level(level : String) - case level.downcase - when "debug" - set_log_level(LogLevel::Debug) - when "info" - set_log_level(LogLevel::Info) - when "warn" - set_log_level(LogLevel::Warn) - when "error" - set_log_level(LogLevel::Error) - end + @level = LogLevel.parse(level) end def set_log_level(level : LogLevel) @level = level end - {% for level in %w(debug info warn error) %} + {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) - puts(message, LogLevel::{{level.id.capitalize}}) + if LogLevel::{{level.id.capitalize}} >= @level + puts("#{Time.utc} [{{level.id}}] #{message}") + end end {% end %} diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/helpers/proxy.cr index 4f415ba0..7a42ef41 100644 --- a/src/invidious/helpers/proxy.cr +++ b/src/invidious/helpers/proxy.cr @@ -108,7 +108,9 @@ def filter_proxies(proxies) proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) client.set_proxy(proxy) - client.head("/").status_code == 200 + status_ok = client.head("/").status_code == 200 + client.close + status_ok rescue ex false end @@ -132,6 +134,7 @@ def get_nova_proxies(country_code = "US") headers["Referer"] = "https://www.proxynova.com/proxy-server-list/country-#{country_code}/" response = client.get("/proxy-server-list/country-#{country_code}/", headers) + client.close document = XML.parse_html(response.body) proxies = [] of {ip: String, port: Int32, score: Float64} @@ -177,6 +180,7 @@ def get_spys_proxies(country_code = "US") } response = client.post("/free-proxy-list/#{country_code}/", headers, form: body) + client.close 20.times do if response.status_code == 200 break diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index 39aae367..a09ce90b 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -70,33 +70,33 @@ def validate_request(token, session, request, key, db, locale = nil) when JSON::Any token = token.as_h when Nil - raise translate(locale, "Hidden field \"token\" is a required field") + raise InfoException.new("Hidden field \"token\" is a required field") end expire = token["expire"]?.try &.as_i if expire.try &.< Time.utc.to_unix - raise translate(locale, "Token is expired, please try again") + raise InfoException.new("Token is expired, please try again") end if token["session"] != session - raise translate(locale, "Erroneous token") + raise InfoException.new("Erroneous token") end scopes = token["scopes"].as_a.map { |v| v.as_s } scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}" if !scopes_include_scope(scopes, scope) - raise translate(locale, "Invalid scope") + raise InfoException.new("Invalid scope") end if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token)) - raise translate(locale, "Invalid signature") + raise InfoException.new("Invalid signature") end if token["nonce"]? && (nonce = db.query_one?("SELECT * FROM nonces WHERE nonce = $1", token["nonce"], as: {String, Time})) if nonce[1] > Time.utc db.exec("UPDATE nonces SET expire = $1 WHERE nonce = $2", Time.utc(1990, 1, 1), nonce[0]) else - raise translate(locale, "Erroneous token") + raise InfoException.new("Erroneous token") end end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index a51f15ce..f068b5f2 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -83,6 +83,7 @@ def make_client(url : URI, region = nil) # TODO: Migrate any applicable endpoints to QUIC client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" client.read_timeout = 10.seconds client.connect_timeout = 10.seconds @@ -100,6 +101,15 @@ def make_client(url : URI, region = nil) return client end +def make_client(url : URI, region = nil, &block) + client = make_client(url, region) + begin + yield client + ensure + client.close + end +end + def decode_length_seconds(string) length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i length_seconds = [0] * (3 - length_seconds.size) + length_seconds @@ -360,7 +370,7 @@ def subscribe_pubsub(topic, key, config) "hub.secret" => key.to_s, } - return make_client(PUBSUB_URL).post("/subscribe", form: body) + return make_client(PUBSUB_URL, &.post("/subscribe", form: body)) end def parse_range(range) diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr index 8b69e01a..61f8eaf3 100644 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ b/src/invidious/jobs/bypass_captcha_job.cr @@ -23,7 +23,8 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob headers = response.cookies.add_request_headers(HTTP::Headers.new) - response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: { + response = JSON.parse(HTTP::Client.post(config.captcha_api_url + "/createTask", + headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { "clientKey" => config.captcha_key, "task" => { "type" => "NoCaptchaTaskProxyless", @@ -39,7 +40,8 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob loop do sleep 10.seconds - response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: { + response = JSON.parse(HTTP::Client.post(config.captcha_api_url + "/getTaskResult", + headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { "clientKey" => config.captcha_key, "taskId" => task_id, }.to_json).body) @@ -76,9 +78,10 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob inputs[node["name"]] = node["value"] end - captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com")) + captcha_client = HTTPClient.new(URI.parse(config.captcha_api_url)) captcha_client.family = config.force_resolve || Socket::Family::INET - response = JSON.parse(captcha_client.post("/createTask", body: { + response = JSON.parse(captcha_client.post("/createTask", + headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { "clientKey" => config.captcha_key, "task" => { "type" => "NoCaptchaTaskProxyless", @@ -88,13 +91,16 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob }, }.to_json).body) + captcha_client.close + raise response["error"].as_s if response["error"]? task_id = response["taskId"].as_i loop do sleep 10.seconds - response = JSON.parse(captcha_client.post("/getTaskResult", body: { + response = JSON.parse(captcha_client.post("/getTaskResult", + headers: HTTP::Headers{"Content-Type" => "application/json"}, body: { "clientKey" => config.captcha_key, "taskId" => task_id, }.to_json).body) @@ -121,7 +127,7 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob end end rescue ex - logger.puts("Exception: #{ex.message}") + logger.error("BypassCaptchaJob: #{ex.message}") ensure sleep 1.minute Fiber.yield diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 75fc474d..6c858afa 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -7,37 +7,44 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob end def begin - max_threads = config.channel_threads - lim_threads = max_threads - active_threads = 0 + max_fibers = config.channel_threads + lim_fibers = max_fibers + active_fibers = 0 active_channel = Channel(Bool).new backoff = 1.seconds loop do + logger.debug("RefreshChannelsJob: Refreshing all channels") db.query("SELECT id FROM channels ORDER BY updated") do |rs| rs.each do id = rs.read(String) - if active_threads >= lim_threads + if active_fibers >= lim_fibers + logger.trace("RefreshChannelsJob: Fiber limit reached, waiting...") if active_channel.receive - active_threads -= 1 + logger.trace("RefreshChannelsJob: Fiber limit ok, continuing") + active_fibers -= 1 end end - active_threads += 1 + logger.trace("RefreshChannelsJob: #{id} : Spawning fiber") + active_fibers += 1 spawn do begin - channel = fetch_channel(id, db, config.full_refresh) + logger.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") + channel = fetch_channel(id, db, logger, config.full_refresh) - lim_threads = max_threads + lim_fibers = max_fibers + + logger.trace("RefreshChannelsJob: #{id} fiber : Updating DB") db.exec("UPDATE channels SET updated = $1, author = $2, deleted = false WHERE id = $3", Time.utc, channel.author, id) rescue ex - logger.puts("#{id} : #{ex.message}") + logger.error("RefreshChannelsJob: #{id} : #{ex.message}") if ex.message == "Deleted or invalid channel" db.exec("UPDATE channels SET updated = $1, deleted = true WHERE id = $2", Time.utc, id) else - lim_threads = 1 - logger.puts("#{id} : backing off for #{backoff}s") + lim_fibers = 1 + logger.error("RefreshChannelsJob: #{id} fiber : backing off for #{backoff}s") sleep backoff if backoff < 1.days backoff += backoff @@ -45,13 +52,15 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob backoff = 1.days end end + ensure + logger.trace("RefreshChannelsJob: #{id} fiber : Done") + active_channel.send(true) end - - active_channel.send(true) end end end + logger.debug("RefreshChannelsJob: Done, sleeping for one minute") sleep 1.minute Fiber.yield end diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr index eebdf0f3..208569b8 100644 --- a/src/invidious/jobs/refresh_feeds_job.cr +++ b/src/invidious/jobs/refresh_feeds_job.cr @@ -7,8 +7,8 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob end def begin - max_threads = config.feed_threads - active_threads = 0 + max_fibers = config.feed_threads + active_fibers = 0 active_channel = Channel(Bool).new loop do @@ -17,27 +17,27 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob email = rs.read(String) view_name = "subscriptions_#{sha256(email)}" - if active_threads >= max_threads + if active_fibers >= max_fibers if active_channel.receive - active_threads -= 1 + active_fibers -= 1 end end - active_threads += 1 + active_fibers += 1 spawn do begin # Drop outdated views column_array = get_column_array(db, view_name) ChannelVideo.type_array.each_with_index do |name, i| if name != column_array[i]? - logger.puts("DROP MATERIALIZED VIEW #{view_name}") + logger.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}") db.exec("DROP MATERIALIZED VIEW #{view_name}") raise "view does not exist" end end if !db.query_one("SELECT pg_get_viewdef('#{view_name}')", as: String).includes? "WHERE ((cv.ucid = ANY (u.subscriptions))" - logger.puts("Materialized view #{view_name} is out-of-date, recreating...") + logger.info("RefreshFeedsJob: Materialized view #{view_name} is out-of-date, recreating...") db.exec("DROP MATERIALIZED VIEW #{view_name}") end @@ -49,18 +49,18 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob legacy_view_name = "subscriptions_#{sha256(email)[0..7]}" db.exec("SELECT * FROM #{legacy_view_name} LIMIT 0") - logger.puts("RENAME MATERIALIZED VIEW #{legacy_view_name}") + logger.info("RefreshFeedsJob: RENAME MATERIALIZED VIEW #{legacy_view_name}") db.exec("ALTER MATERIALIZED VIEW #{legacy_view_name} RENAME TO #{view_name}") rescue ex begin # While iterating through, we may have an email stored from a deleted account if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool) - logger.puts("CREATE #{view_name}") + logger.info("RefreshFeedsJob: CREATE #{view_name}") db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}") db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) end rescue ex - logger.puts("REFRESH #{email} : #{ex.message}") + logger.error("RefreshFeedJobs: REFRESH #{email} : #{ex.message}") end end end diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr index 3d3b2218..2255730d 100644 --- a/src/invidious/jobs/subscribe_to_feeds_job.cr +++ b/src/invidious/jobs/subscribe_to_feeds_job.cr @@ -8,12 +8,12 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob end def begin - max_threads = 1 + max_fibers = 1 if config.use_pubsub_feeds.is_a?(Int32) - max_threads = config.use_pubsub_feeds.as(Int32) + max_fibers = config.use_pubsub_feeds.as(Int32) end - active_threads = 0 + active_fibers = 0 active_channel = Channel(Bool).new loop do @@ -21,23 +21,23 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob rs.each do ucid = rs.read(String) - if active_threads >= max_threads.as(Int32) + if active_fibers >= max_fibers.as(Int32) if active_channel.receive - active_threads -= 1 + active_fibers -= 1 end end - active_threads += 1 + active_fibers += 1 spawn do begin response = subscribe_pubsub(ucid, hmac_key, config) if response.status_code >= 400 - logger.puts("#{ucid} : #{response.body}") + logger.error("SubscribeToFeedsJob: #{ucid} : #{response.body}") end rescue ex - logger.puts("#{ucid} : #{ex.message}") + logger.error("SubscribeToFeedsJob: #{ucid} : #{ex.message}") end active_channel.send(true) diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index c69eb0c4..55b01174 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -30,7 +30,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) initial_data = extract_initial_data(response.body) if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]? - raise translate(locale, "Could not create mix.") + raise InfoException.new("Could not create mix.") end playlist = initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]["playlist"] diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index c984a12a..d5b41caa 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -220,6 +220,11 @@ struct InvidiousPlaylist json.field "videos" do json.array do + if !offset || offset == 0 + index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, continuation, as: Int64) + offset = self.index.index(index) || 0 + end + videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) videos.each_with_index do |video, index| video.to_json(locale, json, offset + index) @@ -338,7 +343,7 @@ def get_playlist(db, plid, locale, refresh = true, force_refresh = false) if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) return playlist else - raise "Playlist does not exist." + raise InfoException.new("Playlist does not exist.") end else return fetch_playlist(plid, locale) @@ -353,16 +358,16 @@ def fetch_playlist(plid, locale) response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en") if response.status_code != 200 if response.headers["location"]?.try &.includes? "/sorry/index" - raise "Could not extract playlist info. Instance is likely blocked." + raise InfoException.new("Could not extract playlist info. Instance is likely blocked.") else - raise translate(locale, "Not a playlist.") + raise InfoException.new("Not a playlist.") end end initial_data = extract_initial_data(response.body) playlist_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[0]["playlistSidebarPrimaryInfoRenderer"]? - raise "Could not extract playlist info" if !playlist_info + raise InfoException.new("Could not extract playlist info") if !playlist_info title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || "" desc_item = playlist_info["description"]? @@ -390,7 +395,7 @@ def fetch_playlist(plid, locale) author_info = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?.try &.[1]["playlistSidebarSecondaryInfoRenderer"]? .try &.["videoOwner"]["videoOwnerRenderer"]? - raise "Could not extract author info" if !author_info + raise InfoException.new("Could not extract author info") if !author_info author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || "" author = author_info["title"]["runs"][0]["text"]?.try &.as_s || "" @@ -412,11 +417,6 @@ end def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) if playlist.is_a? InvidiousPlaylist - if !offset - index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", playlist.id, continuation, as: Int64) - offset = playlist.index.index(index) || 0 - end - db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", playlist.id, playlist.index, offset, as: PlaylistVideo) else fetch_playlist_videos(playlist.id, playlist.video_count, offset, locale, continuation) diff --git a/src/invidious/routes/base_route.cr b/src/invidious/routes/base_route.cr index c6e6667e..2852cb04 100644 --- a/src/invidious/routes/base_route.cr +++ b/src/invidious/routes/base_route.cr @@ -4,6 +4,4 @@ abstract class Invidious::Routes::BaseRoute def initialize(@config, @logger) end - - abstract def handle(env) end diff --git a/src/invidious/routes/embed/index.cr b/src/invidious/routes/embed/index.cr index 79c91d86..32a4966b 100644 --- a/src/invidious/routes/embed/index.cr +++ b/src/invidious/routes/embed/index.cr @@ -8,9 +8,7 @@ class Invidious::Routes::Embed::Index < Invidious::Routes::BaseRoute offset = env.params.query["index"]?.try &.to_i? || 0 videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) rescue ex - error_message = ex.message - env.response.status_code = 500 - return templated "error" + return error_template(500, ex) end url = "/embed/#{videos[0].id}?#{env.params.query}" diff --git a/src/invidious/routes/embed/show.cr b/src/invidious/routes/embed/show.cr index 23c2b86f..8a655556 100644 --- a/src/invidious/routes/embed/show.cr +++ b/src/invidious/routes/embed/show.cr @@ -38,9 +38,7 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute offset = env.params.query["index"]?.try &.to_i? || 0 videos = get_playlist_videos(PG_DB, playlist, offset: offset, locale: locale) rescue ex - error_message = ex.message - env.response.status_code = 500 - return templated "error" + return error_template(500, ex) end url = "/embed/#{videos[0].id}" @@ -63,8 +61,7 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute env.params.query.delete_all("channel") if !video_id || video_id == "live_stream" - error_message = "Video is unavailable." - return templated "error" + return error_template(500, "Video is unavailable.") end url = "/embed/#{video_id}" @@ -100,9 +97,7 @@ class Invidious::Routes::Embed::Show < Invidious::Routes::BaseRoute rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - return templated "error" + return error_template(500, ex) end if preferences.annotations_subscribed && diff --git a/src/invidious/routes/home.cr b/src/invidious/routes/home.cr index 9b1bf61b..486a7344 100644 --- a/src/invidious/routes/home.cr +++ b/src/invidious/routes/home.cr @@ -5,30 +5,24 @@ class Invidious::Routes::Home < Invidious::Routes::BaseRoute user = env.get? "user" case preferences.default_home - when "" - templated "empty" when "Popular" - templated "popular" + env.redirect "/feed/popular" when "Trending" env.redirect "/feed/trending" when "Subscriptions" if user env.redirect "/feed/subscriptions" else - templated "popular" + env.redirect "/feed/popular" end when "Playlists" if user env.redirect "/view_all_playlists" else - templated "popular" + env.redirect "/feed/popular" end else templated "empty" end end - - private def popular_videos - Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get - end end diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr new file mode 100644 index 00000000..42fb4676 --- /dev/null +++ b/src/invidious/routes/login.cr @@ -0,0 +1,508 @@ +class Invidious::Routes::Login < Invidious::Routes::BaseRoute + def login_page(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + + return env.redirect "/feed/subscriptions" if user + + if !config.login_enabled + return error_template(400, "Login has been disabled by administrator.") + end + + referer = get_referer(env, "/feed/subscriptions") + + email = nil + password = nil + captcha = nil + + account_type = env.params.query["type"]? + account_type ||= "invidious" + + captcha_type = env.params.query["captcha"]? + captcha_type ||= "image" + + tfa = env.params.query["tfa"]? + prompt = nil + + templated "login" + end + + def login(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + referer = get_referer(env, "/feed/subscriptions") + + if !config.login_enabled + return error_template(403, "Login has been disabled by administrator.") + end + + # https://stackoverflow.com/a/574698 + email = env.params.body["email"]?.try &.downcase.byte_slice(0, 254) + password = env.params.body["password"]? + + account_type = env.params.query["type"]? + account_type ||= "invidious" + + case account_type + when "google" + tfa_code = env.params.body["tfa"]?.try &.lchop("G-") + traceback = IO::Memory.new + + # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82 + begin + client = QUIC::Client.new(LOGIN_URL) + headers = HTTP::Headers.new + + login_page = client.get("/ServiceLogin") + headers = login_page.cookies.add_request_headers(headers) + + lookup_req = { + email, nil, [] of String, nil, "US", nil, nil, 2, false, true, + {nil, nil, + {2, 1, nil, 1, + "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn", + nil, [] of String, 4}, + 1, + {nil, nil, [] of String}, + nil, nil, nil, true, + }, + email, + }.to_json + + traceback << "Getting lookup..." + + headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8" + headers["Google-Accounts-XSRF"] = "1" + + response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req)) + lookup_results = JSON.parse(response.body[5..-1]) + + traceback << "done, returned #{response.status_code}.<br/>" + + user_hash = lookup_results[0][2] + + if token = env.params.body["token"]? + answer = env.params.body["answer"]? + captcha = {token, answer} + else + captcha = nil + end + + challenge_req = { + user_hash, nil, 1, nil, + {1, nil, nil, nil, + {password, captcha, true}, + }, + {nil, nil, + {2, 1, nil, 1, + "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn", + nil, [] of String, 4}, + 1, + {nil, nil, [] of String}, + nil, nil, nil, true, + }, + }.to_json + + traceback << "Getting challenge..." + + response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req)) + headers = response.cookies.add_request_headers(headers) + challenge_results = JSON.parse(response.body[5..-1]) + + traceback << "done, returned #{response.status_code}.<br/>" + + headers["Cookie"] = URI.decode_www_form(headers["Cookie"]) + + if challenge_results[0][3]?.try &.== 7 + return error_template(423, "Account has temporarily been disabled") + end + + if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s + account_type = "google" + captcha_type = "image" + prompt = nil + tfa = tfa_code + captcha = {tokens: [token], question: ""} + + return templated "login" + end + + if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED" + return error_template(401, "Incorrect password") + end + + prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]? + if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type + traceback << "Handling prompt #{prompt_type}.<br/>" + case prompt_type + when "TWO_STEP_VERIFICATION" + prompt_type = 2 + else # "LOGIN_CHALLENGE" + prompt_type = 4 + end + + # Prefer Authenticator app and SMS over unsupported protocols + if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2 + tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0] + + traceback << "Selecting challenge #{tfa[8]}..." + select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json + + tl = challenge_results[1][2] + + tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body + tfa = tfa[5..-1] + tfa = JSON.parse(tfa)[0][-1] + + traceback << "done.<br/>" + else + traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.<br/>" + tfa = challenge_results[0][-1][0][0] + end + + if tfa[5] == "QUOTA_EXCEEDED" + return error_template(423, "Quota exceeded, try again in a few hours") + end + + if !tfa_code + account_type = "google" + captcha_type = "image" + + case tfa[8] + when 6, 9 + prompt = "Google verification code" + when 12 + prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" + when 15 + prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" + else + prompt = "Google verification code" + end + + tfa = nil + captcha = nil + return templated "login" + end + + tl = challenge_results[1][2] + + request_type = tfa[8] + case request_type + when 6 # Authenticator app + tfa_req = { + user_hash, nil, 2, nil, + {6, nil, nil, nil, nil, + {tfa_code, false}, + }, + }.to_json + when 9 # Voice or text message + tfa_req = { + user_hash, nil, 2, nil, + {9, nil, nil, nil, nil, nil, nil, nil, + {nil, tfa_code, false, 2}, + }, + }.to_json + when 12 # Recovery email + tfa_req = { + user_hash, nil, 4, nil, + {12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + {tfa_code}, + }, + }.to_json + when 15 # Security question + tfa_req = { + user_hash, nil, 5, nil, + {15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + {tfa_code}, + }, + }.to_json + else + return error_template(500, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.") + end + + traceback << "Submitting challenge..." + + response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req)) + headers = response.cookies.add_request_headers(headers) + challenge_results = JSON.parse(response.body[5..-1]) + + if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") || + (challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT") + return error_template(401, "Invalid TFA code") + end + + traceback << "done.<br/>" + end + + traceback << "Logging in..." + + location = URI.parse(challenge_results[0][-1][2].to_s) + cookies = HTTP::Cookies.from_headers(headers) + + headers.delete("Content-Type") + headers.delete("Google-Accounts-XSRF") + + loop do + if !location || location.path == "/ManageAccount" + break + end + + # Occasionally there will be a second page after login confirming + # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle. + + if location.path.starts_with? "/b/0/SmsAuthInterstitial" + traceback << "Unhandled dialog /b/0/SmsAuthInterstitial." + end + + login = client.get(location.full_path, headers) + + headers = login.cookies.add_request_headers(headers) + location = login.headers["Location"]?.try { |u| URI.parse(u) } + end + + cookies = HTTP::Cookies.from_headers(headers) + sid = cookies["SID"]?.try &.value + if !sid + raise "Couldn't get SID." + end + + user, sid = get_user(sid, headers, PG_DB, logger) + + # We are now logged in + traceback << "done.<br/>" + + host = URI.parse(env.request.headers["Host"]).host + + if Kemal.config.ssl || config.https_only + secure = true + else + secure = false + end + + cookies.each do |cookie| + if Kemal.config.ssl || config.https_only + cookie.secure = secure + else + cookie.secure = secure + end + + if cookie.extension + cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host) + cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "") + end + env.response.cookies << cookie + end + + if env.request.cookies["PREFS"]? + preferences = env.get("preferences").as(Preferences) + PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + + cookie = env.request.cookies["PREFS"] + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + + env.redirect referer + rescue ex + traceback.rewind + # error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.") + error_message = %(#{ex.message}<br/>Traceback:<br/><div style="padding-left:2em" id="traceback">#{traceback.gets_to_end}</div>) + return error_template(500, error_message) + end + when "invidious" + if !email + return error_template(401, "User ID is a required field") + end + + if !password + return error_template(401, "Password is a required field") + end + + user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1", email, as: User) + + if user + if !user.password + return error_template(400, "Please sign in using 'Log in with Google'") + end + + if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) + sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) + PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + + if Kemal.config.ssl || config.https_only + secure = true + else + secure = false + end + + if config.domain + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, + secure: secure, http_only: true) + else + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, + secure: secure, http_only: true) + end + else + return error_template(401, "Wrong username or password") + end + + # Since this user has already registered, we don't want to overwrite their preferences + if env.request.cookies["PREFS"]? + cookie = env.request.cookies["PREFS"] + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + else + if !config.registration_enabled + return error_template(400, "Registration has been disabled by administrator.") + end + + if password.empty? + return error_template(401, "Password cannot be empty") + end + + # See https://security.stackexchange.com/a/39851 + if password.bytesize > 55 + return error_template(400, "Password cannot be longer than 55 characters") + end + + password = password.byte_slice(0, 55) + + if config.captcha_enabled + captcha_type = env.params.body["captcha_type"]? + answer = env.params.body["answer"]? + change_type = env.params.body["change_type"]? + + if !captcha_type || change_type + if change_type + captcha_type = change_type + end + captcha_type ||= "image" + + account_type = "invidious" + tfa = false + prompt = "" + + if captcha_type == "image" + captcha = generate_captcha(HMAC_KEY, PG_DB) + else + captcha = generate_text_captcha(HMAC_KEY, PG_DB) + end + + return templated "login" + end + + tokens = env.params.body.select { |k, v| k.match(/^token\[\d+\]$/) }.map { |k, v| v } + + answer ||= "" + captcha_type ||= "image" + + case captcha_type + when "image" + answer = answer.lstrip('0') + answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) + + begin + validate_request(tokens[0], answer, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + return error_template(400, ex) + end + else # "text" + answer = Digest::MD5.hexdigest(answer.downcase.strip) + + if tokens.empty? + return error_template(500, "Erroneous CAPTCHA") + end + + found_valid_captcha = false + error_exception = Exception.new + tokens.each_with_index do |token, i| + begin + validate_request(token, answer, env.request, HMAC_KEY, PG_DB, locale) + found_valid_captcha = true + rescue ex + error_exception = ex + end + end + + if !found_valid_captcha + return error_template(500, error_exception) + end + end + end + + sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) + user, sid = create_user(sid, email, password) + user_array = user.to_a + user_array[4] = user_array[4].to_json # User preferences + + args = arg_array(user_array) + + PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) + PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) + + view_name = "subscriptions_#{sha256(user.email)}" + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") + + if Kemal.config.ssl || config.https_only + secure = true + else + secure = false + end + + if config.domain + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{config.domain}", value: sid, expires: Time.utc + 2.years, + secure: secure, http_only: true) + else + env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, + secure: secure, http_only: true) + end + + if env.request.cookies["PREFS"]? + preferences = env.get("preferences").as(Preferences) + PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences.to_json, user.email) + + cookie = env.request.cookies["PREFS"] + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + end + + env.redirect referer + else + env.redirect referer + end + end + + def signout(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + return error_template(400, ex) + end + + PG_DB.exec("DELETE FROM session_ids * WHERE id = $1", sid) + + env.request.cookies.each do |cookie| + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + + env.redirect referer + end +end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr new file mode 100644 index 00000000..6c899054 --- /dev/null +++ b/src/invidious/routes/playlists.cr @@ -0,0 +1,472 @@ +class Invidious::Routes::Playlists < Invidious::Routes::BaseRoute + def index(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + + items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_created.map! do |item| + item.author = "" + item + end + + items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_saved.map! do |item| + item.author = "" + item + end + + templated "view_all_playlists" + end + + def new(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + sid = sid.as(String) + csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY, PG_DB) + + templated "create_playlist" + end + + def create(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + return error_template(400, ex) + end + + title = env.params.body["title"]?.try &.as(String) + if !title || title.empty? + return error_template(400, "Title cannot be empty.") + end + + privacy = PlaylistPrivacy.parse?(env.params.body["privacy"]?.try &.as(String) || "") + if !privacy + return error_template(400, "Invalid privacy setting.") + end + + if PG_DB.query_one("SELECT count(*) FROM playlists WHERE author = $1", user.email, as: Int64) >= 100 + return error_template(400, "User cannot have more than 100 playlists.") + end + + playlist = create_playlist(PG_DB, title, privacy, user) + + env.redirect "/playlist?list=#{playlist.id}" + end + + def subscribe(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + + playlist_id = env.params.query["list"] + playlist = get_playlist(PG_DB, playlist_id, locale) + subscribe_playlist(PG_DB, user, playlist) + + env.redirect "/playlist?list=#{playlist.id}" + end + + def delete_page(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + sid = sid.as(String) + + plid = env.params.query["list"]? + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email + return env.redirect referer + end + + csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY, PG_DB) + + templated "delete_playlist" + end + + def delete(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + plid = env.params.query["list"]? + return env.redirect referer if plid.nil? + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + return error_template(400, ex) + end + + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email + return env.redirect referer + end + + PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) + PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) + + env.redirect "/view_all_playlists" + end + + def edit(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + sid = sid.as(String) + + plid = env.params.query["list"]? + if !plid || !plid.starts_with?("IV") + return env.redirect referer + end + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + begin + playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email + return env.redirect referer + end + rescue ex + return env.redirect referer + end + + begin + videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) + rescue ex + videos = [] of PlaylistVideo + end + + csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB) + + templated "edit_playlist" + end + + def update(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + plid = env.params.query["list"]? + return env.redirect referer if plid.nil? + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + return error_template(400, ex) + end + + playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email + return env.redirect referer + end + + title = env.params.body["title"]?.try &.delete("<>") || "" + privacy = PlaylistPrivacy.parse(env.params.body["privacy"]? || "Public") + description = env.params.body["description"]?.try &.delete("\r") || "" + + if title != playlist.title || + privacy != playlist.privacy || + description != playlist.description + updated = Time.utc + else + updated = playlist.updated + end + + PG_DB.exec("UPDATE playlists SET title = $1, privacy = $2, description = $3, updated = $4 WHERE id = $5", title, privacy, description, updated, plid) + + env.redirect "/playlist?list=#{plid}" + end + + def add_playlist_items_page(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + sid = sid.as(String) + + plid = env.params.query["list"]? + if !plid || !plid.starts_with?("IV") + return env.redirect referer + end + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + begin + playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + if !playlist || playlist.author != user.email + return env.redirect referer + end + rescue ex + return env.redirect referer + end + + query = env.params.query["q"]? + if query + begin + search_query, count, items = process_search_query(query, page, user, region: nil) + videos = items.select { |item| item.is_a? SearchVideo }.map { |item| item.as(SearchVideo) } + rescue ex + videos = [] of SearchVideo + count = 0 + end + else + videos = [] of SearchVideo + count = 0 + end + + env.set "add_playlist_items", plid + templated "add_playlist_items" + end + + def playlist_ajax(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env, "/") + + redirect = env.params.query["redirect"]? + redirect ||= "true" + redirect = redirect == "true" + + if !user + if redirect + return env.redirect referer + else + return error_json(403, "No such user") + end + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, PG_DB, locale) + rescue ex + if redirect + return error_template(400, ex) + else + return error_json(400, ex) + end + end + + if env.params.query["action_create_playlist"]? + action = "action_create_playlist" + elsif env.params.query["action_delete_playlist"]? + action = "action_delete_playlist" + elsif env.params.query["action_edit_playlist"]? + action = "action_edit_playlist" + elsif env.params.query["action_add_video"]? + action = "action_add_video" + video_id = env.params.query["video_id"] + elsif env.params.query["action_remove_video"]? + action = "action_remove_video" + elsif env.params.query["action_move_video_before"]? + action = "action_move_video_before" + else + return env.redirect referer + end + + begin + playlist_id = env.params.query["playlist_id"] + playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist) + raise "Invalid user" if playlist.author != user.email + rescue ex + if redirect + return error_template(400, ex) + else + return error_json(400, ex) + end + end + + if !user.password + # TODO: Playlist stub, sync with YouTube for Google accounts + # playlist_ajax(playlist_id, action, env.request.headers) + end + email = user.email + + case action + when "action_edit_playlist" + # TODO: Playlist stub + when "action_add_video" + if playlist.index.size >= 500 + if redirect + return error_template(400, "Playlist cannot have more than 500 videos") + else + return error_json(400, "Playlist cannot have more than 500 videos") + end + end + + video_id = env.params.query["video_id"] + + begin + video = get_video(video_id, PG_DB) + rescue ex + if redirect + return error_template(500, ex) + else + return error_json(500, ex) + end + end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: playlist_id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + video_array = playlist_video.to_a + args = arg_array(video_array) + + PG_DB.exec("INSERT INTO playlist_videos VALUES (#{args})", args: video_array) + PG_DB.exec("UPDATE playlists SET index = array_append(index, $1), video_count = cardinality(index) + 1, updated = $2 WHERE id = $3", playlist_video.index, Time.utc, playlist_id) + when "action_remove_video" + index = env.params.query["set_video_id"] + PG_DB.exec("DELETE FROM playlist_videos * WHERE index = $1", index) + PG_DB.exec("UPDATE playlists SET index = array_remove(index, $1), video_count = cardinality(index) - 1, updated = $2 WHERE id = $3", index, Time.utc, playlist_id) + when "action_move_video_before" + # TODO: Playlist stub + else + return error_json(400, "Unsupported action #{action}") + end + + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + "{}" + end + end + + def show(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get?("user").try &.as(User) + referer = get_referer(env) + + plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") + if !plid + return env.redirect "/" + end + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + if plid.starts_with? "RD" + return env.redirect "/mix?list=#{plid}" + end + + begin + playlist = get_playlist(PG_DB, plid, locale) + rescue ex + return error_template(500, ex) + end + + if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email + return error_template(403, "This playlist is private.") + end + + begin + videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) + rescue ex + videos = [] of PlaylistVideo + end + + if playlist.author == user.try &.email + env.set "remove_playlist_items", plid + end + + templated "playlist" + end + + def mix(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + rdid = env.params.query["list"]? + if !rdid + return env.redirect "/" + end + + continuation = env.params.query["continuation"]? + continuation ||= rdid.lchop("RD") + + begin + mix = fetch_mix(rdid, continuation, locale: locale) + rescue ex + return error_template(500, ex) + end + + templated "mix" + end +end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr new file mode 100644 index 00000000..48446161 --- /dev/null +++ b/src/invidious/routes/search.cr @@ -0,0 +1,59 @@ +class Invidious::Routes::Search < Invidious::Routes::BaseRoute + def opensearch(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + env.response.content_type = "application/opensearchdescription+xml" + + XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("OpenSearchDescription", xmlns: "http://a9.com/-/spec/opensearch/1.1/") do + xml.element("ShortName") { xml.text "Invidious" } + xml.element("LongName") { xml.text "Invidious Search" } + xml.element("Description") { xml.text "Search for videos, channels, and playlists on Invidious" } + xml.element("InputEncoding") { xml.text "UTF-8" } + xml.element("Image", width: 48, height: 48, type: "image/x-icon") { xml.text "#{HOST_URL}/favicon.ico" } + xml.element("Url", type: "text/html", method: "get", template: "#{HOST_URL}/search?q={searchTerms}") + end + end + end + + def results(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + query = env.params.query["search_query"]? + query ||= env.params.query["q"]? + query ||= "" + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + if query + env.redirect "/search?q=#{URI.encode_www_form(query)}&page=#{page}" + else + env.redirect "/" + end + end + + def search(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + region = env.params.query["region"]? + + query = env.params.query["search_query"]? + query ||= env.params.query["q"]? + query ||= "" + + return env.redirect "/" if query.empty? + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + user = env.get? "user" + + begin + search_query, count, videos = process_search_query(query, page, user, region: nil) + rescue ex + return error_template(500, ex) + end + + env.set "search", query + templated "search" + end +end diff --git a/src/invidious/routes/user_preferences.cr b/src/invidious/routes/user_preferences.cr new file mode 100644 index 00000000..7f334115 --- /dev/null +++ b/src/invidious/routes/user_preferences.cr @@ -0,0 +1,259 @@ +class Invidious::Routes::UserPreferences < Invidious::Routes::BaseRoute + def show(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + referer = get_referer(env) + + preferences = env.get("preferences").as(Preferences) + + templated "preferences" + end + + def update(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + referer = get_referer(env) + + video_loop = env.params.body["video_loop"]?.try &.as(String) + video_loop ||= "off" + video_loop = video_loop == "on" + + annotations = env.params.body["annotations"]?.try &.as(String) + annotations ||= "off" + annotations = annotations == "on" + + annotations_subscribed = env.params.body["annotations_subscribed"]?.try &.as(String) + annotations_subscribed ||= "off" + annotations_subscribed = annotations_subscribed == "on" + + autoplay = env.params.body["autoplay"]?.try &.as(String) + autoplay ||= "off" + autoplay = autoplay == "on" + + continue = env.params.body["continue"]?.try &.as(String) + continue ||= "off" + continue = continue == "on" + + continue_autoplay = env.params.body["continue_autoplay"]?.try &.as(String) + continue_autoplay ||= "off" + continue_autoplay = continue_autoplay == "on" + + listen = env.params.body["listen"]?.try &.as(String) + listen ||= "off" + listen = listen == "on" + + local = env.params.body["local"]?.try &.as(String) + local ||= "off" + local = local == "on" + + speed = env.params.body["speed"]?.try &.as(String).to_f32? + speed ||= CONFIG.default_user_preferences.speed + + player_style = env.params.body["player_style"]?.try &.as(String) + player_style ||= CONFIG.default_user_preferences.player_style + + quality = env.params.body["quality"]?.try &.as(String) + quality ||= CONFIG.default_user_preferences.quality + + quality_dash = env.params.body["quality_dash"]?.try &.as(String) + quality_dash ||= CONFIG.default_user_preferences.quality_dash + + volume = env.params.body["volume"]?.try &.as(String).to_i? + volume ||= CONFIG.default_user_preferences.volume + + comments = [] of String + 2.times do |i| + comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i]) + end + + captions = [] of String + 3.times do |i| + captions << (env.params.body["captions[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.captions[i]) + end + + related_videos = env.params.body["related_videos"]?.try &.as(String) + related_videos ||= "off" + related_videos = related_videos == "on" + + default_home = env.params.body["default_home"]?.try &.as(String) || CONFIG.default_user_preferences.default_home + + feed_menu = [] of String + 4.times do |index| + option = env.params.body["feed_menu[#{index}]"]?.try &.as(String) || "" + if !option.empty? + feed_menu << option + end + end + + locale = env.params.body["locale"]?.try &.as(String) + locale ||= CONFIG.default_user_preferences.locale + + dark_mode = env.params.body["dark_mode"]?.try &.as(String) + dark_mode ||= CONFIG.default_user_preferences.dark_mode + + thin_mode = env.params.body["thin_mode"]?.try &.as(String) + thin_mode ||= "off" + thin_mode = thin_mode == "on" + + max_results = env.params.body["max_results"]?.try &.as(String).to_i? + max_results ||= CONFIG.default_user_preferences.max_results + + sort = env.params.body["sort"]?.try &.as(String) + sort ||= CONFIG.default_user_preferences.sort + + latest_only = env.params.body["latest_only"]?.try &.as(String) + latest_only ||= "off" + latest_only = latest_only == "on" + + unseen_only = env.params.body["unseen_only"]?.try &.as(String) + unseen_only ||= "off" + unseen_only = unseen_only == "on" + + notifications_only = env.params.body["notifications_only"]?.try &.as(String) + notifications_only ||= "off" + notifications_only = notifications_only == "on" + + # Convert to JSON and back again to take advantage of converters used for compatability + preferences = Preferences.from_json({ + annotations: annotations, + annotations_subscribed: annotations_subscribed, + autoplay: autoplay, + captions: captions, + comments: comments, + continue: continue, + continue_autoplay: continue_autoplay, + dark_mode: dark_mode, + latest_only: latest_only, + listen: listen, + local: local, + locale: locale, + max_results: max_results, + notifications_only: notifications_only, + player_style: player_style, + quality: quality, + quality_dash: quality_dash, + default_home: default_home, + feed_menu: feed_menu, + related_videos: related_videos, + sort: sort, + speed: speed, + thin_mode: thin_mode, + unseen_only: unseen_only, + video_loop: video_loop, + volume: volume, + }.to_json).to_json + + if user = env.get? "user" + user = user.as(User) + PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) + + if config.admins.includes? user.email + config.default_user_preferences.default_home = env.params.body["admin_default_home"]?.try &.as(String) || config.default_user_preferences.default_home + + admin_feed_menu = [] of String + 4.times do |index| + option = env.params.body["admin_feed_menu[#{index}]"]?.try &.as(String) || "" + if !option.empty? + admin_feed_menu << option + end + end + config.default_user_preferences.feed_menu = admin_feed_menu + + popular_enabled = env.params.body["popular_enabled"]?.try &.as(String) + popular_enabled ||= "off" + config.popular_enabled = popular_enabled == "on" + + captcha_enabled = env.params.body["captcha_enabled"]?.try &.as(String) + captcha_enabled ||= "off" + config.captcha_enabled = captcha_enabled == "on" + + login_enabled = env.params.body["login_enabled"]?.try &.as(String) + login_enabled ||= "off" + config.login_enabled = login_enabled == "on" + + registration_enabled = env.params.body["registration_enabled"]?.try &.as(String) + registration_enabled ||= "off" + config.registration_enabled = registration_enabled == "on" + + statistics_enabled = env.params.body["statistics_enabled"]?.try &.as(String) + statistics_enabled ||= "off" + config.statistics_enabled = statistics_enabled == "on" + + CONFIG.default_user_preferences = config.default_user_preferences + File.write("config/config.yml", config.to_yaml) + end + else + if Kemal.config.ssl || config.https_only + secure = true + else + secure = false + end + + if config.domain + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, + secure: secure, http_only: true) + else + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, + secure: secure, http_only: true) + end + end + + env.redirect referer + end + + def toggle_theme(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + referer = get_referer(env, unroll: false) + + redirect = env.params.query["redirect"]? + redirect ||= "true" + redirect = redirect == "true" + + if user = env.get? "user" + user = user.as(User) + preferences = user.preferences + + case preferences.dark_mode + when "dark" + preferences.dark_mode = "light" + else + preferences.dark_mode = "dark" + end + + preferences = preferences.to_json + + PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", preferences, user.email) + else + preferences = env.get("preferences").as(Preferences) + + case preferences.dark_mode + when "dark" + preferences.dark_mode = "light" + else + preferences.dark_mode = "dark" + end + + preferences = preferences.to_json + + if Kemal.config.ssl || config.https_only + secure = true + else + secure = false + end + + if config.domain + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{config.domain}", value: preferences, expires: Time.utc + 2.years, + secure: secure, http_only: true) + else + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, + secure: secure, http_only: true) + end + end + + if redirect + env.redirect referer + else + env.response.content_type = "application/json" + "{}" + end + end +end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 4eee7793..a5c05c00 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -12,9 +12,7 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute id = env.params.query["v"] if env.params.query["v"].empty? - error_message = "Invalid parameters." - env.response.status_code = 400 - return templated "error" + return error_template(400, "Invalid parameters.") end if id.size > 11 @@ -30,6 +28,14 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute return env.redirect "/" end + embed_link = "/embed/#{id}" + if env.params.query.size > 1 + embed_params = HTTP::Params.parse(env.params.query.to_s) + embed_params.delete_all("v") + embed_link += "?" + embed_link += embed_params.to_s + end + plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") continuation = process_continuation(PG_DB, env.params.query, plid, id) @@ -56,10 +62,8 @@ class Invidious::Routes::Watch < Invidious::Routes::BaseRoute rescue ex : VideoRedirect return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex - error_message = ex.message - env.response.status_code = 500 - logger.puts("#{id} : #{ex.message}") - return templated "error" + logger.error("get_video: #{id} : #{ex.message}") + return error_template(500, ex) end if preferences.annotations_subscribed && diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index c09dda38..602e6ae5 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -1,8 +1,15 @@ module Invidious::Routing - macro get(path, controller) + macro get(path, controller, method = :handle) get {{ path }} do |env| controller_instance = {{ controller }}.new(config, logger) - controller_instance.handle(env) + controller_instance.{{ method.id }}(env) + end + end + + macro post(path, controller, method = :handle) + post {{ path }} do |env| + controller_instance = {{ controller }}.new(config, logger) + controller_instance.{{ method.id }}(env) end end end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 46bf8865..5dc16edd 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -66,6 +66,8 @@ struct Preferences @[JSON::Field(converter: Preferences::ProcessString)] property quality : String = CONFIG.default_user_preferences.quality + @[JSON::Field(converter: Preferences::ProcessString)] + property quality_dash : String = CONFIG.default_user_preferences.quality_dash property default_home : String = CONFIG.default_user_preferences.default_home property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu property related_videos : Bool = CONFIG.default_user_preferences.related_videos @@ -267,12 +269,12 @@ struct Preferences end end -def get_user(sid, headers, db, refresh = true) +def get_user(sid, headers, db, logger, refresh = true) if email = db.query_one?("SELECT email FROM session_ids WHERE id = $1", sid, as: String) user = db.query_one("SELECT * FROM users WHERE email = $1", email, as: User) if refresh && Time.utc - user.updated > 1.minute - user, sid = fetch_user(sid, headers, db) + user, sid = fetch_user(sid, headers, db, logger) user_array = user.to_a user_array[4] = user_array[4].to_json # User preferences args = arg_array(user_array) @@ -290,7 +292,7 @@ def get_user(sid, headers, db, refresh = true) end end else - user, sid = fetch_user(sid, headers, db) + user, sid = fetch_user(sid, headers, db, logger) user_array = user.to_a user_array[4] = user_array[4].to_json # User preferences args = arg_array(user.to_a) @@ -311,7 +313,7 @@ def get_user(sid, headers, db, refresh = true) return user, sid end -def fetch_user(sid, headers, db) +def fetch_user(sid, headers, db, logger) feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) feed = XML.parse_html(feed.body) @@ -324,7 +326,7 @@ def fetch_user(sid, headers, db) end end - channels = get_batch_channels(channels, db, false, false) + channels = get_batch_channels(channels, db, logger, false, false) email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"])) if email @@ -425,7 +427,7 @@ def generate_captcha(key, db) end def generate_text_captcha(key, db) - response = make_client(TEXTCAPTCHA_URL).get("/omarroth@protonmail.com.json").body + response = make_client(TEXTCAPTCHA_URL, &.get("/omarroth@protonmail.com.json").body) response = JSON.parse(response) tokens = response["a"].as_a.map do |answer| diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 8e314fe0..4a831110 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -235,6 +235,7 @@ struct VideoPreferences property preferred_captions : Array(String) property player_style : String property quality : String + property quality_dash : String property raw : Bool property region : String? property related_videos : Bool @@ -816,7 +817,7 @@ end def extract_polymer_config(body) params = {} of String => JSON::Any - player_response = body.match(/window\["ytInitialPlayerResponse"\]\s*=\s*(?<info>.*?);\n/) + player_response = body.match(/(window\["ytInitialPlayerResponse"\]|var\sytInitialPlayerResponse)\s*=\s*(?<info>{.*?});/m) .try { |r| JSON.parse(r["info"]).as_h } if body.includes?("To continue with your YouTube experience, please fill out the form below.") || @@ -830,7 +831,8 @@ def extract_polymer_config(body) params["reason"] = JSON::Any.new(reason) end - params["sessionToken"] = JSON::Any.new(body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]?) + session_token_json_encoded = body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || "" + params["sessionToken"] = JSON.parse(%({"key": "#{session_token_json_encoded}"}))["key"] params["shortDescription"] = JSON::Any.new(body.match(/"og:description" content="(?<description>[^"]+)"/).try &.["description"]?) return params if !player_response @@ -839,8 +841,7 @@ def extract_polymer_config(body) params[f] = player_response[f] if player_response[f]? end - yt_initial_data = body.match(/(window\["ytInitialData"\]|var\s+ytInitialData)\s*=\s*(?<info>.*?);\s*\n/) - .try { |r| JSON.parse(r["info"]).as_h } + yt_initial_data = extract_initial_data(body) params["relatedVideos"] = yt_initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]? .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r| @@ -915,10 +916,14 @@ def extract_polymer_config(body) .try { |r| JSON.parse(r["info"]) }.try &.["args"]["player_response"]? .try &.as_s?.try &.try { |r| JSON.parse(r).as_h } - return params if !initial_data - - {"playabilityStatus", "streamingData"}.each do |f| - params[f] = initial_data[f] if initial_data[f]? + if initial_data + {"playabilityStatus", "streamingData"}.each do |f| + params[f] = initial_data[f] if initial_data[f]? + end + else + {"playabilityStatus", "streamingData"}.each do |f| + params[f] = player_response[f] if player_response[f]? + end end params @@ -999,7 +1004,7 @@ def fetch_video(id, region) }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) end - raise info["reason"]?.try &.as_s || "" if !info["videoDetails"]? + raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]? video = Video.new({ id: id, @@ -1039,6 +1044,7 @@ def process_video_params(query, preferences) player_style = query["player_style"]? preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase } quality = query["quality"]? + quality_dash = query["quality_dash"]? region = query["region"]? related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } speed = query["speed"]?.try &.rchop("x").to_f? @@ -1057,6 +1063,7 @@ def process_video_params(query, preferences) player_style ||= preferences.player_style preferred_captions ||= preferences.captions quality ||= preferences.quality + quality_dash ||= preferences.quality_dash related_videos ||= preferences.related_videos.to_unsafe speed ||= preferences.speed video_loop ||= preferences.video_loop.to_unsafe @@ -1073,6 +1080,7 @@ def process_video_params(query, preferences) player_style ||= CONFIG.default_user_preferences.player_style preferred_captions ||= CONFIG.default_user_preferences.captions quality ||= CONFIG.default_user_preferences.quality + quality_dash ||= CONFIG.default_user_preferences.quality_dash related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe speed ||= CONFIG.default_user_preferences.speed video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe @@ -1125,6 +1133,7 @@ def process_video_params(query, preferences) player_style: player_style, preferred_captions: preferred_captions, quality: quality, + quality_dash: quality_dash, raw: raw, region: region, related_videos: related_videos, diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 0e6664fa..625c6fee 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -4,7 +4,7 @@ <% if params.video_loop %>loop<% end %> <% if params.controls %>controls<% end %>> <% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %> - <source src="<%= URI.parse(hlsvp).full_path %>?local=true" type="application/x-mpegURL" label="livestream"> + <source src="<%= URI.parse(hlsvp).full_path %><% if params.local %>?local=true<% end %>" type="application/x-mpegURL" label="livestream"> <% else %> <% if params.listen %> <% audio_streams.each_with_index do |fmt, i| %> diff --git a/src/invidious/views/components/player_sources.ecr b/src/invidious/views/components/player_sources.ecr index d02f82d2..8162546e 100644 --- a/src/invidious/views/components/player_sources.ecr +++ b/src/invidious/views/components/player_sources.ecr @@ -3,7 +3,6 @@ <link rel="stylesheet" href="/css/videojs.markers.min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/videojs-share.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/videojs-vtt-thumbnails.css?v=<%= ASSET_COMMIT %>"> -<link rel="stylesheet" href="/css/videojs-vtt-thumbnails-fix.css?v=<%= ASSET_COMMIT %>"> <script src="/js/global.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/video.min.js?v=<%= ASSET_COMMIT %>"></script> <script src="/js/videojs-contrib-quality-levels.min.js?v=<%= ASSET_COMMIT %>"></script> diff --git a/src/invidious/views/data_control.ecr b/src/invidious/views/data_control.ecr index e3edb9ea..74ccc06c 100644 --- a/src/invidious/views/data_control.ecr +++ b/src/invidious/views/data_control.ecr @@ -14,7 +14,7 @@ <div class="pure-control-group"> <label for="import_youtube"> - <a rel="noopener" target="_blank" href="https://support.google.com/youtube/answer/6224202?hl=en"> + <a rel="noopener" target="_blank" href="https://github.com/iv-org/documentation/blob/master/Export-YouTube-subscriptions.md"> <%= translate(locale, "Import YouTube subscriptions") %> </a> </label> diff --git a/src/invidious/views/embed.ecr b/src/invidious/views/embed.ecr index 48dbc55f..dbb86009 100644 --- a/src/invidious/views/embed.ecr +++ b/src/invidious/views/embed.ecr @@ -9,12 +9,11 @@ <link rel="stylesheet" href="/css/videojs-overlay.css?v=<%= ASSET_COMMIT %>"> <script src="/js/videojs-overlay.min.js?v=<%= ASSET_COMMIT %>"></script> <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> - <link rel="stylesheet" href="/css/darktheme.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/embed.css?v=<%= ASSET_COMMIT %>"> <title><%= HTML.escape(video.title) %> - Invidious</title> </head> -<body> +<body class="dark-theme"> <script id="video_data" type="application/json"> <%= { diff --git a/src/invidious/views/message.ecr b/src/invidious/views/message.ecr new file mode 100644 index 00000000..8c7bf611 --- /dev/null +++ b/src/invidious/views/message.ecr @@ -0,0 +1,12 @@ +<% content_for "header" do %> +<meta name="description" content="<%= translate(locale, "An alternative front-end to YouTube") %>"> +<title> + Invidious +</title> +<% end %> + +<%= rendered "components/feed_menu" %> + +<p> + <%= message %> +</p> diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr index 0c48be96..a77d106d 100644 --- a/src/invidious/views/playlists.ecr +++ b/src/invidious/views/playlists.ecr @@ -27,7 +27,7 @@ </div> <div class="h-box"> - <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p> + <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p> </div> <div class="h-box"> diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index fb5bd44b..1ef080be 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -57,6 +57,17 @@ </select> </div> + <% if !CONFIG.disabled?("dash") %> + <div class="pure-control-group"> + <label for="quality_dash"><%= translate(locale, "Preferred dash video quality: ") %></label> + <select name="quality_dash" id="quality_dash"> + <% {"auto", "best", "4320p", "2160p", "1440p", "1080p", "720p", "480p", "360p", "240p", "144p", "worst"}.each do |option| %> + <option value="<%= option %>" <% if preferences.quality_dash == option %> selected <% end %>><%= translate(locale, option) %></option> + <% end %> + </select> + </div> + <% end %> + <div class="pure-control-group"> <label for="volume"><%= translate(locale, "Player volume: ") %></label> <input name="volume" id="volume" data-onrange="update_volume_value" type="range" min="0" max="100" step="5" value="<%= preferences.volume %>"> @@ -68,7 +79,7 @@ <% preferences.comments.each_with_index do |comments, index| %> <select name="comments[<%= index %>]" id="comments[<%= index %>]"> <% {"", "youtube", "reddit"}.each do |option| %> - <option value="<%= option %>" <% if preferences.comments[index] == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.comments[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> <% end %> @@ -79,7 +90,7 @@ <% preferences.captions.each_with_index do |caption, index| %> <select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]"> <% CAPTION_LANGUAGES.each do |option| %> - <option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> <% end %> @@ -119,7 +130,7 @@ <label for="dark_mode"><%= translate(locale, "Theme: ") %></label> <select name="dark_mode" id="dark_mode"> <% {"", "light", "dark"}.each do |option| %> - <option value="<%= option %>" <% if preferences.dark_mode == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.dark_mode == option %> selected <% end %>><%= translate(locale, option.blank? ? "auto" : option) %></option> <% end %> </select> </div> @@ -130,16 +141,16 @@ </div> <% if env.get?("user") %> - <% feed_options = {"", "Popular", "Top", "Trending", "Subscriptions", "Playlists"} %> + <% feed_options = {"", "Popular", "Trending", "Subscriptions", "Playlists"} %> <% else %> - <% feed_options = {"", "Popular", "Top", "Trending"} %> + <% feed_options = {"", "Popular", "Trending"} %> <% end %> <div class="pure-control-group"> <label for="default_home"><%= translate(locale, "Default homepage: ") %></label> <select name="default_home" id="default_home"> <% feed_options.each do |option| %> - <option value="<%= option %>" <% if preferences.default_home == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.default_home == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> </div> @@ -149,7 +160,7 @@ <% (feed_options.size - 1).times do |index| %> <select name="feed_menu[<%= index %>]" id="feed_menu[<%= index %>]"> <% feed_options.each do |option| %> - <option value="<%= option %>" <% if preferences.feed_menu[index]? == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if preferences.feed_menu[index]? == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> <% end %> @@ -211,7 +222,7 @@ <label for="admin_default_home"><%= translate(locale, "Default homepage: ") %></label> <select name="admin_default_home" id="admin_default_home"> <% feed_options.each do |option| %> - <option value="<%= option %>" <% if CONFIG.default_user_preferences.default_home == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if CONFIG.default_user_preferences.default_home == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> </div> @@ -221,13 +232,19 @@ <% (feed_options.size - 1).times do |index| %> <select name="admin_feed_menu[<%= index %>]" id="admin_feed_menu[<%= index %>]"> <% feed_options.each do |option| %> - <option value="<%= option %>" <% if CONFIG.default_user_preferences.feed_menu[index]? == option %> selected <% end %>><%= translate(locale, option) %></option> + <option value="<%= option %>" <% if CONFIG.default_user_preferences.feed_menu[index]? == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option> <% end %> </select> <% end %> </div> <div class="pure-control-group"> + <label for="popular_enabled"><%= translate(locale, "Popular enabled: ") %></label> + <input name="popular_enabled" id="popular_enabled" type="checkbox" <% if config.popular_enabled %>checked<% end %>> + </div> + + + <div class="pure-control-group"> <label for="captcha_enabled"><%= translate(locale, "CAPTCHA enabled: ") %></label> <input name="captcha_enabled" id="captcha_enabled" type="checkbox" <% if config.captcha_enabled %>checked<% end %>> </div> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 61cf5c3a..f6e5262d 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -4,7 +4,6 @@ <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> - <meta name="referrer" content="no-referrer"> <%= yield_content "header" %> <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png?v=<%= ASSET_COMMIT %>"> <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png?v=<%= ASSET_COMMIT %>"> @@ -18,13 +17,12 @@ <link rel="stylesheet" href="/css/grids-responsive-min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/ionicons.min.css?v=<%= ASSET_COMMIT %>"> <link rel="stylesheet" href="/css/default.css?v=<%= ASSET_COMMIT %>"> - <link rel="stylesheet" href="/css/darktheme.css?v=<%= ASSET_COMMIT %>" id="dark_theme" <% if env.get("preferences").as(Preferences).dark_mode != "dark" %>media="none"<% end %>> - <link rel="stylesheet" href="/css/lighttheme.css?v=<%= ASSET_COMMIT %>" id="light_theme" <% if env.get("preferences").as(Preferences).dark_mode == "dark" %>media="none"<% end %>> </head> <% locale = LOCALES[env.get("preferences").as(Preferences).locale]? %> +<% dark_mode = env.get("preferences").as(Preferences).dark_mode %> -<body> +<body class="<%= dark_mode.blank? ? "no" : dark_mode %>-theme"> <span style="display:none" id="dark_mode_pref"><%= env.get("preferences").as(Preferences).dark_mode %></span> <div class="pure-g"> <div class="pure-u-1 pure-u-md-2-24"></div> @@ -116,16 +114,15 @@ </a> </div> <div class="pure-u-1 pure-u-md-1-3"> - <i class="icon ion-logo-bitcoin"></i> - BTC: <a href="bitcoin:356DpZyMXu6rYd55Yqzjs29n79kGKWcYrY">356DpZyMXu6rYd55Yqzjs29n79kGKWcYrY</a> + <i class="icon ion-ios-wallet"></i> + BTC: <a href="bitcoin:bc1qfhe7rq3lqzuayzjxzyt9waz9ytrs09kla3tsgr">bc1qfhe7rq3lqzuayzjxzyt9waz9ytrs09kla3tsgr</a> </div> <div class="pure-u-1 pure-u-md-1-3"> - <i class="icon ion-logo-bitcoin"></i> - BCH: <a href="bitcoincash:qq4ptclkzej5eza6a50et5ggc58hxsq5aylqut2npk">qq4ptclkzej5eza6a50et5ggc58hxsq5aylqut2npk</a> + <i class="icon ion-ios-wallet"></i> + XMR: <a href="monero:41nMCtek197boJtiUvGnTFYMatrLEpnpkQDmUECqx5Es2uX3sTKKWVhSL76suXsG3LXqkEJBrCZBgPTwJrDp1FrZJfycGPR">Click here</a> </div> <div class="pure-u-1 pure-u-md-1-3"> - <i class="icon ion-logo-usd"></i> - <a href="https://liberapay.com/omarroth">Liberapay</a> + <a href="https://github.com/iv-org/documentation">Documentation</a> </div> <div class="pure-u-1 pure-u-md-1-3"> <i class="icon ion-logo-javascript"></i> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 9a1e6c32..786a88b6 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -88,7 +88,11 @@ <div class="h-box"> <span id="watch-on-youtube"> <a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "Watch on YouTube") %></a> + (<a href="https://www.youtube.com/embed/<%= video.id %>"><%= translate(locale, "Embed") %></a>) </span> + <p id="embed-link"> + <a href="<%= embed_link %>"><%= translate(locale, "Embed Link") %></a> + </p> <p id="annotations"> <% if params.annotations %> <a href="/watch?<%= env.params.query %>&iv_load_policy=3"> |