diff options
Diffstat (limited to 'src')
40 files changed, 1208 insertions, 858 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 04b18a65..abc459b7 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -30,6 +30,8 @@ require "./invidious/database/*" require "./invidious/database/migrations/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" +require "./invidious/frontend/*" + require "./invidious/*" require "./invidious/channels/*" require "./invidious/user/*" @@ -39,14 +41,13 @@ require "./invidious/jobs/**" CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) -PG_DB = DB.open CONFIG.database_url -ARCHIVE_URL = URI.parse("https://archive.org") -LOGIN_URL = URI.parse("https://accounts.google.com") -PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") -REDDIT_URL = URI.parse("https://www.reddit.com") -TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com") -YT_URL = URI.parse("https://www.youtube.com") -HOST_URL = make_host_url(Kemal.config) +PG_DB = DB.open CONFIG.database_url +ARCHIVE_URL = URI.parse("https://archive.org") +LOGIN_URL = URI.parse("https://accounts.google.com") +PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") +REDDIT_URL = URI.parse("https://www.reddit.com") +YT_URL = URI.parse("https://www.youtube.com") +HOST_URL = make_host_url(Kemal.config) CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} @@ -160,8 +161,8 @@ if CONFIG.popular_enabled Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB) end -connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32) -Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(connection_channel, CONFIG.database_url) +CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32) +Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url) Invidious::Jobs.start_all @@ -240,6 +241,7 @@ before_all do |env| "/api/manifest/", "/videoplayback", "/latest_version", + "/download", }.any? { |r| env.request.resource.starts_with? r } if env.request.cookies.has_key? "SID" @@ -330,6 +332,9 @@ end Invidious::Routing.get "/channel/:ucid/playlists", Invidious::Routes::Channels, :playlists Invidious::Routing.get "/channel/:ucid/community", Invidious::Routes::Channels, :community Invidious::Routing.get "/channel/:ucid/about", Invidious::Routes::Channels, :about + Invidious::Routing.get "/channel/:ucid/live", Invidious::Routes::Channels, :live + Invidious::Routing.get "/user/:user/live", Invidious::Routes::Channels, :live + Invidious::Routing.get "/c/:user/live", Invidious::Routes::Channels, :live ["", "/videos", "/playlists", "/community", "/about"].each do |path| # /c/LinusTechTips @@ -352,6 +357,8 @@ end Invidious::Routing.get "/e/:id", Invidious::Routes::Watch, :redirect Invidious::Routing.get "/redirect", Invidious::Routes::Misc, :cross_instance_redirect + Invidious::Routing.post "/download", Invidious::Routes::Watch, :download + Invidious::Routing.get "/embed/", Invidious::Routes::Embed, :redirect Invidious::Routing.get "/embed/:id", Invidious::Routes::Embed, :show @@ -366,20 +373,14 @@ end Invidious::Routing.post "/playlist_ajax", Invidious::Routes::Playlists, :playlist_ajax Invidious::Routing.get "/playlist", Invidious::Routes::Playlists, :show Invidious::Routing.get "/mix", Invidious::Routes::Playlists, :mix + Invidious::Routing.get "/watch_videos", Invidious::Routes::Playlists, :watch_videos Invidious::Routing.get "/opensearch.xml", Invidious::Routes::Search, :opensearch Invidious::Routing.get "/results", Invidious::Routes::Search, :results Invidious::Routing.get "/search", Invidious::Routes::Search, :search - Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page - Invidious::Routing.post "/login", Invidious::Routes::Login, :login - Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout - - Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show - Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update - Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme - Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control - Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control + # User routes + define_user_routes() # Feeds Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Feeds, :view_all_playlists_redirect @@ -419,404 +420,6 @@ define_v1_api_routes() define_api_manifest_routes() define_video_playback_routes() -get "/change_password" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY) - - templated "change_password" -end - -post "/change_password" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - # We don't store passwords for Google accounts - if !user.password - next error_template(400, "Cannot change password for Google accounts") - end - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - next error_template(400, ex) - end - - password = env.params.body["password"]? - if !password - next error_template(401, "Password is a required field") - end - - new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } - - if new_passwords.size <= 1 || new_passwords.uniq.size != 1 - next error_template(400, "New passwords must match") - end - - new_password = new_passwords.uniq[0] - if new_password.empty? - next error_template(401, "Password cannot be empty") - end - - if new_password.bytesize > 55 - next error_template(400, "Password cannot be longer than 55 characters") - end - - if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) - next error_template(401, "Incorrect password") - end - - new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) - Invidious::Database::Users.update_password(user, new_password.to_s) - - env.redirect referer -end - -get "/delete_account" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY) - - templated "delete_account" -end - -post "/delete_account" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - next error_template(400, ex) - end - - view_name = "subscriptions_#{sha256(user.email)}" - Invidious::Database::Users.delete(user) - Invidious::Database::SessionIDs.delete(email: user.email) - PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") - - env.request.cookies.each do |cookie| - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - - env.redirect referer -end - -get "/clear_watch_history" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY) - - templated "clear_watch_history" -end - -post "/clear_watch_history" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - next error_template(400, ex) - end - - Invidious::Database::Users.clear_watch_history(user) - env.redirect referer -end - -get "/authorize_token" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY) - - scopes = env.params.query["scopes"]?.try &.split(",") - scopes ||= [] of String - - callback_url = env.params.query["callback_url"]? - if callback_url - callback_url = URI.parse(callback_url) - end - - expire = env.params.query["expire"]?.try &.to_i? - - templated "authorize_token" -end - -post "/authorize_token" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = env.get("user").as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - next error_template(400, ex) - end - - scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } - callback_url = env.params.body["callbackUrl"]? - expire = env.params.body["expire"]?.try &.to_i? - - access_token = generate_token(user.email, scopes, expire, HMAC_KEY) - - if callback_url - access_token = URI.encode_www_form(access_token) - url = URI.parse(callback_url) - - if url.query - query = HTTP::Params.parse(url.query.not_nil!) - else - query = HTTP::Params.new - end - - query["token"] = access_token - url.query = query.to_s - - env.redirect url.to_s - else - csrf_token = "" - env.set "access_token", access_token - templated "authorize_token" - end -end - -get "/token_manager" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env, "/subscription_manager") - - if !user - next env.redirect referer - end - - user = user.as(User) - tokens = Invidious::Database::SessionIDs.select_all(user.email) - - templated "token_manager" -end - -post "/token_ajax" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - redirect = env.params.query["redirect"]? - redirect ||= "true" - redirect = redirect == "true" - - if !user - if redirect - next env.redirect referer - else - next error_json(403, "No such user") - end - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - if redirect - next error_template(400, ex) - else - next error_json(400, ex) - end - end - - if env.params.query["action_revoke_token"]? - action = "action_revoke_token" - else - next env.redirect referer - end - - session = env.params.query["session"]? - session ||= "" - - case action - when .starts_with? "action_revoke_token" - Invidious::Database::SessionIDs.delete(sid: session, email: user.email) - else - next error_json(400, "Unsupported action #{action}") - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - -# Channels - -{"/channel/:ucid/live", "/user/:user/live", "/c/:user/live"}.each do |route| - get route do |env| - locale = env.get("preferences").as(Preferences).locale - - # Appears to be a bug in routing, having several routes configured - # as `/a/:a`, `/b/:a`, `/c/:a` results in 404 - value = env.request.resource.split("/")[2] - body = "" - {"channel", "user", "c"}.each do |type| - response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1") - if response.status_code == 200 - body = response.body - end - end - - video_id = body.match(/'VIDEO_ID': "(?<id>[a-zA-Z0-9_-]{11})"/).try &.["id"]? - if video_id - params = [] of String - env.params.query.each do |k, v| - params << "#{k}=#{v}" - end - params = params.join("&") - - url = "/watch?v=#{video_id}" - if !params.empty? - url += "&#{params}" - end - - env.redirect url - else - env.redirect "/channel/#{value}" - end - end -end - -# Authenticated endpoints - -# The notification APIs can't be extracted yet -# due to the requirement of the `connection_channel` -# used by the `NotificationJob` - -get "/api/v1/auth/notifications" do |env| - env.response.content_type = "text/event-stream" - - topics = env.params.query["topics"]?.try &.split(",").uniq.first(1000) - topics ||= [] of String - - create_notification_stream(env, topics, connection_channel) -end - -post "/api/v1/auth/notifications" do |env| - env.response.content_type = "text/event-stream" - - topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) - topics ||= [] of String - - create_notification_stream(env, topics, connection_channel) -end - -get "/Captcha" do |env| - headers = HTTP::Headers{":authority" => "accounts.google.com"} - response = YT_POOL.client &.get(env.request.resource, headers) - env.response.headers["Content-Type"] = response.headers["Content-Type"] - response.body -end - -# Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos -get "/watch_videos" do |env| - response = YT_POOL.client &.get(env.request.resource) - if url = response.headers["Location"]? - url = URI.parse(url).request_target - next env.redirect url - end - - env.response.status_code = response.status_code -end - error 404 do |env| if md = env.request.path.match(/^\/(?<id>([a-zA-Z0-9_-]{11})|(\w+))$/) item = md["id"] @@ -881,7 +484,7 @@ add_handler AuthHandler.new add_handler DenyFrame.new add_context_storage_type(Array(String)) add_context_storage_type(Preferences) -add_context_storage_type(User) +add_context_storage_type(Invidious::User) Kemal.config.logger = LOGGER Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 65f4b135..ab9fcc8b 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -78,7 +78,8 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b when "RELOAD_CONTINUATION_SLOT_HEADER" header = item["reloadContinuationItemsCommand"]["continuationItems"][0] when "RELOAD_CONTINUATION_SLOT_BODY" - contents = item["reloadContinuationItemsCommand"]["continuationItems"] + # continuationItems is nil when video has no comments + contents = item["reloadContinuationItemsCommand"]["continuationItems"]? end elsif item["appendContinuationItemsAction"]? contents = item["appendContinuationItemsAction"]["continuationItems"] diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 72e145da..93c4c0f7 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -23,6 +23,7 @@ struct ConfigPreferences property listen : Bool = false property local : Bool = false property locale : String = "en-US" + property watch_history : Bool = true property max_results : Int32 = 40 property notifications_only : Bool = false property player_style : String = "invidious" @@ -56,20 +57,35 @@ end class Config include YAML::Serializable - property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) - property feed_threads : Int32 = 1 # Number of threads to use for updating feeds - property output : String = "STDOUT" # Log file path or STDOUT - property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr - property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) - + # Number of threads to use for crawling videos from channels (for updating subscriptions) + property channel_threads : Int32 = 1 + # Time interval between two executions of the job that crawls channel videos (subscriptions update). + @[YAML::Field(converter: Preferences::TimeSpanConverter)] + property channel_refresh_interval : Time::Span = 30.minutes + # Number of threads to use for updating feeds + property feed_threads : Int32 = 1 + # Log file path or STDOUT + property output : String = "STDOUT" + # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr + property log_level : LogLevel = LogLevel::Info + # Database configuration with separate parameters (username, hostname, etc) + property db : DBConfig? = nil + + # Database configuration using 12-Factor "Database URL" syntax @[YAML::Field(converter: Preferences::URIConverter)] - property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax - property decrypt_polling : Bool = true # Use polling to keep decryption function up to date - property full_refresh : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel - property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// - property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions - property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required - property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property database_url : URI = URI.parse("") + # Use polling to keep decryption function up to date + property decrypt_polling : Bool = true + # Used for crawling channels: threads should check all videos uploaded by a channel + property full_refresh : Bool = false + # Used to tell Invidious it is behind a proxy, so links to resources should be https:// + property https_only : Bool? + # HMAC signing key for CSRF tokens and verifying pubsub subscriptions + property hmac_key : String? + # Domain to be used for links to resources on the site where an absolute URL is required + property domain : String? + # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property use_pubsub_feeds : Bool | Int32 = false property popular_enabled : Bool = true property captcha_enabled : Bool = true property login_enabled : Bool = true @@ -78,28 +94,42 @@ class Config property admins : Array(String) = [] of String property external_port : Int32? = nil property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("") - property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs - property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc. - property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards - property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc. - property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely - property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' + # For compliance with DMCA, disables download widget using list of video IDs + property dmca_content : Array(String) = [] of String + # Check table integrity, automatically try to add any missing columns, create tables, etc. + property check_tables : Bool = false + # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards + property cache_annotations : Bool = false + # Optional banner to be displayed along top of page for announcements, etc. + property banner : String? = nil + # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely + property hsts : Bool? = true + # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' + property disable_proxy : Bool? | Array(String)? = false # URL to the modified source code to be easily AGPL compliant # Will display in the footer, next to the main source code link property modified_source_code_url : String? = nil + # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) @[YAML::Field(converter: Preferences::FamilyConverter)] - property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) - property port : Int32 = 3000 # Port to listen for connections (overridden by command line argument) - property host_binding : String = "0.0.0.0" # Host to bind (overridden by command line argument) - property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) - property use_quic : Bool = false # Use quic transport for youtube api - + property force_resolve : Socket::Family = Socket::Family::UNSPEC + # Port to listen for connections (overridden by command line argument) + property port : Int32 = 3000 + # Host to bind (overridden by command line argument) + property host_binding : String = "0.0.0.0" + # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) + property pool_size : Int32 = 100 + # Use quic transport for youtube api + property use_quic : Bool = false + + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] - property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format - property captcha_key : String? = nil # Key for Anti-Captcha - property captcha_api_url : String = "https://api.anti-captcha.com" # API URL for Anti-Captcha + property cookies : HTTP::Cookies = HTTP::Cookies.new + # Key for Anti-Captcha + property captcha_key : String? = nil + # API URL for Anti-Captcha + property captcha_api_url : String = "https://api.anti-captcha.com" def disabled?(option) case disabled = CONFIG.disable_proxy diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr index 26be4270..f62b43ea 100644 --- a/src/invidious/database/users.cr +++ b/src/invidious/database/users.cr @@ -171,7 +171,7 @@ module Invidious::Database::Users WHERE email = $2 SQL - PG_DB.exec(request, user.email, pass) + PG_DB.exec(request, pass, user.email) end # ------------------- diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr new file mode 100644 index 00000000..80b67641 --- /dev/null +++ b/src/invidious/frontend/watch_page.cr @@ -0,0 +1,108 @@ +module Invidious::Frontend::WatchPage + extend self + + # A handy structure to pass many elements at + # once to the download widget function + struct VideoAssets + getter full_videos : Array(Hash(String, JSON::Any)) + getter video_streams : Array(Hash(String, JSON::Any)) + getter audio_streams : Array(Hash(String, JSON::Any)) + getter captions : Array(Caption) + + def initialize( + @full_videos, + @video_streams, + @audio_streams, + @captions + ) + end + end + + def download_widget(locale : String, video : Video, video_assets : VideoAssets) : String + if CONFIG.disabled?("downloads") + return "<p id=\"download\">#{translate(locale, "Download is disabled.")}</p>" + end + + return String.build(4000) do |str| + str << "<form" + str << " class=\"pure-form pure-form-stacked\"" + str << " action='/download'" + str << " method='post'" + str << " rel='noopener'" + str << " target='_blank'>" + str << '\n' + + # Hidden inputs for video id and title + str << "<input type='hidden' name='id' value='" << video.id << "'/>\n" + str << "<input type='hidden' name='title' value='" << HTML.escape(video.title) << "'/>\n" + + str << "\t<div class=\"pure-control-group\">\n" + + str << "\t\t<label for='download_widget'>" + str << translate(locale, "Download as: ") + str << "</label>\n" + + # TODO: remove inline style + str << "\t\t<select style=\"width:100%\" name='download_widget' id='download_widget'>\n" + + # Non-DASH videos (audio+video) + + video_assets.full_videos.each do |option| + mimetype = option["mimeType"].as_s.split(";")[0] + + height = itag_to_metadata?(option["itag"]).try &.["height"]? + + value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json + + str << "\t\t\t<option value='" << value << "'>" + str << (height || "~240") << "p - " << mimetype + str << "</option>\n" + end + + # DASH video streams + + video_assets.video_streams.each do |option| + mimetype = option["mimeType"].as_s.split(";")[0] + + value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json + + str << "\t\t\t<option value='" << value << "'>" + str << option["qualityLabel"] << " - " << mimetype << " @ " << option["fps"] << "fps - video only" + str << "</option>\n" + end + + # DASH audio streams + + video_assets.audio_streams.each do |option| + mimetype = option["mimeType"].as_s.split(";")[0] + + value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json + + str << "\t\t\t<option value='" << value << "'>" + str << mimetype << " @ " << (option["bitrate"]?.try &.as_i./ 1000) << "k - audio only" + str << "</option>\n" + end + + # Subtitles (a.k.a "closed captions") + + video_assets.captions.each do |caption| + value = {"label": caption.name, "ext": "#{caption.language_code}.vtt"}.to_json + + str << "\t\t\t<option value='" << value << "'>" + str << translate(locale, "download_subtitles", translate(locale, caption.name)) + str << "</option>\n" + end + + # End of form + + str << "\t\t</select>\n" + str << "\t</div>\n" + + str << "\t<button type=\"submit\" class=\"pure-button pure-button-primary\">\n" + str << "\t\t<b>" << translate(locale, "Download") << "</b>\n" + str << "\t</button>\n" + + str << "</form>\n" + end + end +end diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 6571dbe6..39e183f2 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -30,6 +30,7 @@ LOCALES_LIST = { "pt-PT" => "Português de Portugal", # Portuguese (Portugal) "ro" => "Română", # Romanian "ru" => "русский", # Russian + "sq" => "Shqip", # Albanian "sr" => "srpski (latinica)", # Serbian (Latin) "sr_Cyrl" => "српски (ћирилица)", # Serbian (Cyrillic) "sv-SE" => "Svenska", # Swedish diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index a58a21b1..c1dc17db 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -51,6 +51,24 @@ def recode_length_seconds(time) end end +def decode_interval(string : String) : Time::Span + rawMinutes = string.try &.to_i32? + + if !rawMinutes + hours = /(?<hours>\d+)h/.match(string).try &.["hours"].try &.to_i32 + hours ||= 0 + + minutes = /(?<minutes>\d+)m(?!s)/.match(string).try &.["minutes"].try &.to_i32 + minutes ||= 0 + + time = Time::Span.new(hours: hours, minutes: minutes) + else + time = Time::Span.new(minutes: rawMinutes) + end + + return time +end + def decode_time(string) time = string.try &.to_f? diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 55fb8154..92681408 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -58,9 +58,8 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob end end - # TODO: make this configurable - LOGGER.debug("RefreshChannelsJob: Done, sleeping for thirty minutes") - sleep 30.minutes + LOGGER.debug("RefreshChannelsJob: Done, sleeping for #{CONFIG.channel_refresh_interval}") + sleep CONFIG.channel_refresh_interval Fiber.yield end end diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr new file mode 100644 index 00000000..9bb73136 --- /dev/null +++ b/src/invidious/routes/account.cr @@ -0,0 +1,358 @@ +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::Account + extend self + + # ------------------- + # Password update + # ------------------- + + # Show the password change interface (GET request) + def get_change_password(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY) + + templated "user/change_password" + end + + # Handle the password change (POST request) + def post_change_password(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + # We don't store passwords for Google accounts + if !user.password + return error_template(400, "Cannot change password for Google accounts") + end + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + return error_template(400, ex) + end + + password = env.params.body["password"]? + if !password + return error_template(401, "Password is a required field") + end + + new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } + + if new_passwords.size <= 1 || new_passwords.uniq.size != 1 + return error_template(400, "New passwords must match") + end + + new_password = new_passwords.uniq[0] + if new_password.empty? + return error_template(401, "Password cannot be empty") + end + + if new_password.bytesize > 55 + return error_template(400, "Password cannot be longer than 55 characters") + end + + if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) + return error_template(401, "Incorrect password") + end + + new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) + Invidious::Database::Users.update_password(user, new_password.to_s) + + env.redirect referer + end + + # ------------------- + # Account deletion + # ------------------- + + # Show the account deletion confirmation prompt (GET request) + def get_delete(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY) + + templated "user/delete_account" + end + + # Handle the account deletion (POST request) + def post_delete(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + return error_template(400, ex) + end + + view_name = "subscriptions_#{sha256(user.email)}" + Invidious::Database::Users.delete(user) + Invidious::Database::SessionIDs.delete(email: user.email) + PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") + + env.request.cookies.each do |cookie| + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + + env.redirect referer + end + + # ------------------- + # Clear history + # ------------------- + + # Show the watch history deletion confirmation prompt (GET request) + def get_clear_history(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY) + + templated "user/clear_watch_history" + end + + # Handle the watch history clearing (POST request) + def post_clear_history(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + return error_template(400, ex) + end + + Invidious::Database::Users.clear_watch_history(user) + env.redirect referer + end + + # ------------------- + # Authorize tokens + # ------------------- + + # Show the "authorize token?" confirmation prompt (GET request) + def get_authorize_token(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY) + + scopes = env.params.query["scopes"]?.try &.split(",") + scopes ||= [] of String + + callback_url = env.params.query["callback_url"]? + if callback_url + callback_url = URI.parse(callback_url) + end + + expire = env.params.query["expire"]?.try &.to_i? + + templated "user/authorize_token" + end + + # Handle token authorization (POST request) + def post_authorize_token(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = env.get("user").as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + return error_template(400, ex) + end + + scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } + callback_url = env.params.body["callbackUrl"]? + expire = env.params.body["expire"]?.try &.to_i? + + access_token = generate_token(user.email, scopes, expire, HMAC_KEY) + + if callback_url + access_token = URI.encode_www_form(access_token) + url = URI.parse(callback_url) + + if url.query + query = HTTP::Params.parse(url.query.not_nil!) + else + query = HTTP::Params.new + end + + query["token"] = access_token + url.query = query.to_s + + env.redirect url.to_s + else + csrf_token = "" + env.set "access_token", access_token + templated "user/authorize_token" + end + end + + # ------------------- + # Manage tokens + # ------------------- + + # Show the token manager page (GET request) + def token_manager(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env, "/subscription_manager") + + if !user + return env.redirect referer + end + + user = user.as(User) + tokens = Invidious::Database::SessionIDs.select_all(user.email) + + templated "user/token_manager" + end + + # ------------------- + # AJAX for tokens + # ------------------- + + # Handle internal (non-API) token actions (POST request) + def token_ajax(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + redirect = env.params.query["redirect"]? + redirect ||= "true" + redirect = redirect == "true" + + if !user + if redirect + return env.redirect referer + else + return error_json(403, "No such user") + end + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + if redirect + return error_template(400, ex) + else + return error_json(400, ex) + end + end + + if env.params.query["action_revoke_token"]? + action = "action_revoke_token" + else + return env.redirect referer + end + + session = env.params.query["session"]? + session ||= "" + + case action + when .starts_with? "action_revoke_token" + Invidious::Database::SessionIDs.delete(sid: session, email: user.email) + else + return error_json(400, "Unsupported action #{action}") + end + + if redirect + return env.redirect referer + else + env.response.content_type = "application/json" + return "{}" + end + end +end diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 4e9fc801..b559a01a 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -343,7 +343,7 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "text/html" csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true) - return templated "authorize_token" + return templated "user/authorize_token" else env.response.content_type = "application/json" @@ -397,4 +397,14 @@ module Invidious::Routes::API::V1::Authenticated env.response.status_code = 204 end + + def self.notifications(env) + env.response.content_type = "text/event-stream" + + raw_topics = env.params.body["topics"]? || env.params.query["topics"]? + topics = raw_topics.try &.split(",").uniq.first(1000) + topics ||= [] of String + + create_notification_stream(env, topics, CONNECTION_CHANNEL) + end end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index a1ce0cbc..844fedb8 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -4,10 +4,10 @@ module Invidious::Routes::API::V1::Misc env.response.content_type = "application/json" if !CONFIG.statistics_enabled - return error_json(400, "Statistics are not enabled.") + return {"software" => SOFTWARE}.to_json + else + return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json end - - Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json end # APIv1 currently uses the same logic for both diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 0b0853b1..5666460d 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -43,20 +43,20 @@ module Invidious::Routes::API::V1::Search end def self.search_suggestions(env) - locale = env.get("preferences").as(Preferences).locale - region = env.params.query["region"]? + preferences = env.get("preferences").as(Preferences) + region = env.params.query["region"]? || preferences.region env.response.content_type = "application/json" - query = env.params.query["q"]? - query ||= "" + query = env.params.query["q"]? || "" begin - headers = HTTP::Headers{":authority" => "suggestqueries.google.com"} - response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body + client = HTTP::Client.new("suggestqueries-clients6.youtube.com") + url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt" + + response = client.get(url).body - body = response[35..-2] - body = JSON.parse(body).as_a + body = JSON.parse(response[5..-1]).as_a suggestions = body[1].as_a[0..-2] JSON.build do |json| diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 86eb26ee..a9f891f5 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -23,7 +23,11 @@ module Invidious::Routes::API::V1::Videos env.response.content_type = "application/json" id = env.params.url["id"] - region = env.params.query["region"]? + region = env.params.query["region"]? || env.params.body["region"]? + + if id.nil? || id.size != 11 || !id.matches?(/^[\w-]+$/) + return error_json(400, "Invalid video ID") + end # See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354 # It is possible to use `/api/timedtext?type=list&v=#{id}` and @@ -130,7 +134,13 @@ module Invidious::Routes::API::V1::Videos end end else + # Some captions have "align:[start/end]" and "position:[num]%" + # attributes. Those are causing issues with VideoJS, which is unable + # to properly align the captions on the video, so we remove them. + # + # See: https://github.com/iv-org/invidious/issues/2391 webvtt = YT_POOL.client &.get("#{url}&format=vtt").body + .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1") end if title = env.params.query["title"]? diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 6cb1e1f7..cd2e3323 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -147,6 +147,39 @@ module Invidious::Routes::Channels end end + def self.live(env) + locale = env.get("preferences").as(Preferences).locale + + # Appears to be a bug in routing, having several routes configured + # as `/a/:a`, `/b/:a`, `/c/:a` results in 404 + value = env.request.resource.split("/")[2] + body = "" + {"channel", "user", "c"}.each do |type| + response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1") + if response.status_code == 200 + body = response.body + end + end + + video_id = body.match(/'VIDEO_ID': "(?<id>[a-zA-Z0-9_-]{11})"/).try &.["id"]? + if video_id + params = [] of String + env.params.query.each do |k, v| + params << "#{k}=#{v}" + end + params = params.join("&") + + url = "/watch?v=#{video_id}" + if !params.empty? + url += "&#{params}" + end + + env.redirect url + else + env.redirect "/channel/#{value}" + end + end + private def self.fetch_basic_information(env) locale = env.get("preferences").as(Preferences).locale diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index f4859e6f..99fc13a2 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -27,7 +27,7 @@ module Invidious::Routes::Login tfa = env.params.query["tfa"]? prompt = nil - templated "login" + templated "user/login" end def self.login(env) @@ -133,7 +133,7 @@ module Invidious::Routes::Login tfa = tfa_code captcha = {tokens: [token], question: ""} - return templated "login" + return templated "user/login" end if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED" @@ -190,7 +190,7 @@ module Invidious::Routes::Login tfa = nil captcha = nil - return templated "login" + return templated "user/login" end tl = challenge_results[1][2] @@ -282,18 +282,8 @@ module Invidious::Routes::Login host = URI.parse(env.request.headers["Host"]).host - if Kemal.config.ssl || CONFIG.https_only - secure = true - else - secure = false - end - cookies.each do |cookie| - if Kemal.config.ssl || CONFIG.https_only - cookie.secure = secure - else - cookie.secure = secure - end + cookie.secure = Invidious::User::Cookies::SECURE if cookie.extension cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host) @@ -338,19 +328,7 @@ module Invidious::Routes::Login sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) Invidious::Database::SessionIDs.insert(sid, email) - if Kemal.config.ssl || CONFIG.https_only - secure = true - else - secure = false - end - - if CONFIG.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{CONFIG.domain}", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end + env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) else return error_template(401, "Wrong username or password") end @@ -393,12 +371,12 @@ module Invidious::Routes::Login prompt = "" if captcha_type == "image" - captcha = generate_captcha(HMAC_KEY) + captcha = Invidious::User::Captcha.generate_image(HMAC_KEY) else - captcha = generate_text_captcha(HMAC_KEY) + captcha = Invidious::User::Captcha.generate_text(HMAC_KEY) end - return templated "login" + return templated "user/login" end tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v } @@ -455,19 +433,7 @@ module Invidious::Routes::Login view_name = "subscriptions_#{sha256(user.email)}" PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") - if Kemal.config.ssl || CONFIG.https_only - secure = true - else - secure = false - end - - if CONFIG.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{CONFIG.domain}", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end + env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) if env.request.cookies["PREFS"]? user.preferences = env.get("preferences").as(Preferences) @@ -515,4 +481,11 @@ module Invidious::Routes::Login env.redirect referer end + + def self.captcha(env) + headers = HTTP::Headers{":authority" => "accounts.google.com"} + response = YT_POOL.client &.get(env.request.resource, headers) + env.response.headers["Content-Type"] = response.headers["Content-Type"] + response.body + end end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 1ed29e79..dbeb4f97 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -443,4 +443,15 @@ module Invidious::Routes::Playlists templated "mix" end + + # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos + def self.watch_videos(env) + response = YT_POOL.client &.get(env.request.resource) + if url = response.headers["Location"]? + url = URI.parse(url).request_target + return env.redirect url + end + + env.response.status_code = response.status_code + end end diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 930c588b..570cba69 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -8,7 +8,7 @@ module Invidious::Routes::PreferencesRoute preferences = env.get("preferences").as(Preferences) - templated "preferences" + templated "user/preferences" end def self.update(env) @@ -47,6 +47,10 @@ module Invidious::Routes::PreferencesRoute local ||= "off" local = local == "on" + watch_history = env.params.body["watch_history"]?.try &.as(String) + watch_history ||= "off" + watch_history = watch_history == "on" + speed = env.params.body["speed"]?.try &.as(String).to_f32? speed ||= CONFIG.default_user_preferences.speed @@ -149,6 +153,7 @@ module Invidious::Routes::PreferencesRoute latest_only: latest_only, listen: listen, local: local, + watch_history: watch_history, locale: locale, max_results: max_results, notifications_only: notifications_only, @@ -214,19 +219,7 @@ module Invidious::Routes::PreferencesRoute File.write("config/config.yml", CONFIG.to_yaml) end else - if Kemal.config.ssl || CONFIG.https_only - secure = true - else - secure = false - end - - if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, - secure: secure, http_only: true) - end + env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences) end env.redirect referer @@ -261,21 +254,7 @@ module Invidious::Routes::PreferencesRoute preferences.dark_mode = "dark" end - preferences = preferences.to_json - - if Kemal.config.ssl || CONFIG.https_only - secure = true - else - secure = false - end - - if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, - secure: secure, http_only: true) - end + env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences) end if redirect @@ -298,7 +277,7 @@ module Invidious::Routes::PreferencesRoute user = user.as(User) - templated "data_control" + templated "user/data_control" end def self.update_data_control(env) @@ -321,149 +300,27 @@ module Invidious::Routes::PreferencesRoute # TODO: Unify into single import based on content-type case part.name when "import_invidious" - body = JSON.parse(body) - - if body["subscriptions"]? - user.subscriptions += body["subscriptions"].as_a.map(&.as_s) - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) - end - - if body["watch_history"]? - user.watched += body["watch_history"].as_a.map(&.as_s) - user.watched.uniq! - Invidious::Database::Users.update_watch_history(user) - end - - if body["preferences"]? - user.preferences = Preferences.from_json(body["preferences"].to_json) - Invidious::Database::Users.update_preferences(user) - end - - if playlists = body["playlists"]?.try &.as_a? - playlists.each do |item| - title = item["title"]?.try &.as_s?.try &.delete("<>") - description = item["description"]?.try &.as_s?.try &.delete("\r") - privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } - - next if !title - next if !description - next if !privacy - - playlist = create_playlist(title, privacy, user) - Invidious::Database::Playlists.update_description(playlist.id, description) - - videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| - raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 - - video_id = video_id.try &.as_s? - next if !video_id - - begin - video = get_video(video_id) - rescue ex - next - end - - playlist_video = PlaylistVideo.new({ - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, - length_seconds: video.length_seconds, - published: video.published, - plid: playlist.id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX), - }) - - Invidious::Database::PlaylistVideos.insert(playlist_video) - Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) - end - end - end + Invidious::User::Import.from_invidious(user, body) when "import_youtube" filename = part.filename || "" - extension = filename.split(".").last - - if extension == "xml" || type == "application/xml" || type == "text/xml" - subscriptions = XML.parse(body) - user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] - end - elsif extension == "json" || type == "application/json" - subscriptions = JSON.parse(body) - user.subscriptions += subscriptions.as_a.compact_map do |entry| - entry["snippet"]["resourceId"]["channelId"].as_s - end - elsif extension == "csv" || type == "text/csv" - subscriptions = parse_subscription_export_csv(body) - user.subscriptions += subscriptions - else + success = Invidious::User::Import.from_youtube(user, body, filename, type) + + if !success haltf(env, status_code: 415, response: error_template(415, "Invalid subscription file uploaded") ) end - - user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) when "import_freetube" - user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md| - md["channel_id"] - end - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) + Invidious::User::Import.from_freetube(user, body) when "import_newpipe_subscriptions" - body = JSON.parse(body) - user.subscriptions += body["subscriptions"].as_a.compact_map do |channel| - if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/) - next match["channel"] - elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/) - response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US") - html = XML.parse_html(response.body) - ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] - next ucid if ucid - end - - nil - end - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) + Invidious::User::Import.from_newpipe_subs(user, body) when "import_newpipe" - Compress::Zip::Reader.open(IO::Memory.new(body)) do |file| - file.each_entry do |entry| - if entry.filename == "newpipe.db" - tempfile = File.tempfile(".db") - File.write(tempfile.path, entry.io.gets_to_end) - db = DB.open("sqlite3://" + tempfile.path) - - user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v=")) - user.watched.uniq! - - Invidious::Database::Users.update_watch_history(user) - - user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/")) - user.subscriptions.uniq! + success = Invidious::User::Import.from_newpipe(user, body) - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) - - db.close - tempfile.delete - end - end + if !success + haltf(env, status_code: 415, + response: error_template(415, "Uploaded file is too large") + ) end else nil # Ignore end diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr index ec8fe67b..7b1fa876 100644 --- a/src/invidious/routes/subscriptions.cr +++ b/src/invidious/routes/subscriptions.cr @@ -163,6 +163,6 @@ module Invidious::Routes::Subscriptions end end - templated "subscription_manager" + templated "user/subscription_manager" end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index f6340c57..3a92ef96 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -14,12 +14,18 @@ module Invidious::Routes::VideoPlayback end if query_params["host"]? && !query_params["host"].empty? - host = "https://#{query_params["host"]}" + host = query_params["host"] query_params.delete("host") else - host = "https://r#{fvip}---#{mns.pop}.googlevideo.com" + host = "r#{fvip}---#{mns.pop}.googlevideo.com" end + # Sanity check, to avoid being used as an open proxy + if !host.matches?(/[\w-]+.googlevideo.com/) + return error_template(400, "Invalid \"host\" parameter.") + end + + host = "https://#{host}" url = "/videoplayback?#{query_params}" headers = HTTP::Headers.new @@ -158,7 +164,9 @@ module Invidious::Routes::VideoPlayback if title = query_params["title"]? # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" + filename = URI.encode_www_form(title, space_to_plus: false) + header = "attachment; filename=\"#{filename}\"; filename*=UTF-8''#{filename}" + env.response.headers["Content-Disposition"] = header end if !resp.headers.includes_word?("Transfer-Encoding", "chunked") @@ -236,31 +244,25 @@ module Invidious::Routes::VideoPlayback # YouTube /videoplayback links expire after 6 hours, # so we have a mechanism here to redirect to the latest version def self.latest_version(env) - if env.params.query["download_widget"]? - download_widget = JSON.parse(env.params.query["download_widget"]) + id = env.params.query["id"]? + itag = env.params.query["itag"]?.try &.to_i? - id = download_widget["id"].as_s - title = URI.decode_www_form(download_widget["title"].as_s) - - if label = download_widget["label"]? - return env.redirect "/api/v1/captions/#{id}?label=#{label}&title=#{title}" - else - itag = download_widget["itag"].as_s.to_i - local = "true" - end + # Sanity checks + if id.nil? || id.size != 11 || !id.matches?(/^[\w-]+$/) + return error_template(400, "Invalid video ID") end - id ||= env.params.query["id"]? - itag ||= env.params.query["itag"]?.try &.to_i + if itag.nil? || itag <= 0 || itag >= 1000 + return error_template(400, "Invalid itag") + end region = env.params.query["region"]? + local = (env.params.query["local"]? == "true") - local ||= env.params.query["local"]? - local ||= "false" - local = local == "true" + title = env.params.query["title"]? - if !id || !itag - haltf env, status_code: 400, response: "TESTING" + if title && CONFIG.disabled?("downloads") + return error_template(403, "Administrator has disabled this endpoint.") end video = get_video(id, region: region) @@ -272,8 +274,10 @@ module Invidious::Routes::VideoPlayback haltf env, status_code: 404 end - url = URI.parse(url).request_target.not_nil! if local - url = "#{url}&title=#{title}" if title + if local + url = URI.parse(url).request_target.not_nil! + url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title + end return env.redirect url end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 42bc4219..867ffa6a 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -75,7 +75,7 @@ module Invidious::Routes::Watch end env.params.query.delete_all("iv_load_policy") - if watched && !watched.includes? id + if watched && preferences.watch_history && !watched.includes? id Invidious::Database::Users.mark_watched(user.as(User), id) end @@ -189,6 +189,14 @@ module Invidious::Routes::Watch return env.redirect url end + # Structure used for the download widget + video_assets = Invidious::Frontend::WatchPage::VideoAssets.new( + full_videos: fmt_stream, + video_streams: video_streams, + audio_streams: audio_streams, + captions: video.captions + ) + templated "watch" end @@ -281,4 +289,49 @@ module Invidious::Routes::Watch return error_template(404, "The requested clip doesn't exist") end end + + def self.download(env) + if CONFIG.disabled?("downloads") + return error_template(403, "Administrator has disabled this endpoint.") + end + + title = env.params.body["title"]? || "" + video_id = env.params.body["id"]? || "" + selection = env.params.body["download_widget"]? + + if title.empty? || video_id.empty? || selection.nil? + return error_template(400, "Missing form data") + end + + download_widget = JSON.parse(selection) + + extension = download_widget["ext"].as_s + filename = "#{video_id}-#{title}.#{extension}" + + # Pass form parameters as URL parameters for the handlers of both + # /latest_version and /api/v1/captions. This avoids an un-necessary + # redirect and duplicated (and hazardous) sanity checks. + env.params.query["id"] = video_id + env.params.query["title"] = filename + + # Delete the useless ones + env.params.body.delete("id") + env.params.body.delete("title") + env.params.body.delete("download_widget") + + if label = download_widget["label"]? + # URL params specific to /api/v1/captions/:id + env.params.query["label"] = URI.encode_www_form(label.as_s, space_to_plus: false) + + return Invidious::Routes::API::V1::Videos.captions(env) + elsif itag = download_widget["itag"]?.try &.as_i + # URL params specific to /latest_version + env.params.query["itag"] = itag.to_s + env.params.query["local"] = "true" + + return Invidious::Routes::VideoPlayback.latest_version(env) + else + return error_template(400, "Invalid label or itag") + end + end end diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 7551f22d..bd72c577 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -10,6 +10,33 @@ module Invidious::Routing {% end %} end +macro define_user_routes + # User login/out + Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page + Invidious::Routing.post "/login", Invidious::Routes::Login, :login + Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout + Invidious::Routing.get "/Captcha", Invidious::Routes::Login, :captcha + + # User preferences + Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show + Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update + Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme + Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control + Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control + + # User account management + Invidious::Routing.get "/change_password", Invidious::Routes::Account, :get_change_password + Invidious::Routing.post "/change_password", Invidious::Routes::Account, :post_change_password + Invidious::Routing.get "/delete_account", Invidious::Routes::Account, :get_delete + Invidious::Routing.post "/delete_account", Invidious::Routes::Account, :post_delete + Invidious::Routing.get "/clear_watch_history", Invidious::Routes::Account, :get_clear_history + Invidious::Routing.post "/clear_watch_history", Invidious::Routes::Account, :post_clear_history + Invidious::Routing.get "/authorize_token", Invidious::Routes::Account, :get_authorize_token + Invidious::Routing.post "/authorize_token", Invidious::Routes::Account, :post_authorize_token + Invidious::Routing.get "/token_manager", Invidious::Routes::Account, :token_manager + Invidious::Routing.post "/token_ajax", Invidious::Routes::Account, :token_ajax +end + macro define_v1_api_routes {{namespace = Invidious::Routes::API::V1}} # Videos @@ -69,6 +96,9 @@ macro define_v1_api_routes Invidious::Routing.post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token Invidious::Routing.post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token + Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + # Misc Invidious::Routing.get "/api/v1/stats", {{namespace}}::Misc, :stats Invidious::Routing.get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist diff --git a/src/invidious/search.cr b/src/invidious/search.cr index d8971e79..ae106bf6 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -176,7 +176,7 @@ end def process_search_query(query, page, user, region) if user - user = user.as(User) + user = user.as(Invidious::User) view_name = "subscriptions_#{sha256(user.email)}" end diff --git a/src/invidious/user/captcha.cr b/src/invidious/user/captcha.cr new file mode 100644 index 00000000..8a0f67e5 --- /dev/null +++ b/src/invidious/user/captcha.cr @@ -0,0 +1,78 @@ +require "openssl/hmac" + +struct Invidious::User + module Captcha + extend self + + private TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com") + + def generate_image(key) + second = Random::Secure.rand(12) + second_angle = second * 30 + second = second * 5 + + minute = Random::Secure.rand(12) + minute_angle = minute * 30 + minute = minute * 5 + + hour = Random::Secure.rand(12) + hour_angle = hour * 30 + minute_angle.to_f / 12 + if hour == 0 + hour = 12 + end + + clock_svg = <<-END_SVG + <svg viewBox="0 0 100 100" width="200px" height="200px"> + <circle cx="50" cy="50" r="45" fill="#eee" stroke="black" stroke-width="2"></circle> + + <text x="69" y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 1</text> + <text x="82.909" y="34" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 2</text> + <text x="88" y="53" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 3</text> + <text x="82.909" y="72" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 4</text> + <text x="69" y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 5</text> + <text x="50" y="91" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 6</text> + <text x="31" y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 7</text> + <text x="17.091" y="72" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 8</text> + <text x="12" y="53" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 9</text> + <text x="17.091" y="34" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">10</text> + <text x="31" y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">11</text> + <text x="50" y="15" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">12</text> + + <circle cx="50" cy="50" r="3" fill="black"></circle> + <line id="second" transform="rotate(#{second_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="12" fill="black" stroke="black" stroke-width="1"></line> + <line id="minute" transform="rotate(#{minute_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="16" fill="black" stroke="black" stroke-width="2"></line> + <line id="hour" transform="rotate(#{hour_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="24" fill="black" stroke="black" stroke-width="2"></line> + </svg> + END_SVG + + image = "data:image/png;base64," + image += Process.run(%(rsvg-convert -w 400 -h 400 -b none -f png), shell: true, + input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe + ) do |proc| + Base64.strict_encode(proc.output.gets_to_end) + end + + answer = "#{hour}:#{minute.to_s.rjust(2, '0')}:#{second.to_s.rjust(2, '0')}" + answer = OpenSSL::HMAC.hexdigest(:sha256, key, answer) + + return { + question: image, + tokens: {generate_response(answer, {":login"}, key, use_nonce: true)}, + } + end + + def generate_text(key) + response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body) + response = JSON.parse(response) + + tokens = response["a"].as_a.map do |answer| + generate_response(answer.as_s, {":login"}, key, use_nonce: true) + end + + return { + question: response["q"].as_s, + tokens: tokens, + } + end + end +end diff --git a/src/invidious/user/cookies.cr b/src/invidious/user/cookies.cr new file mode 100644 index 00000000..99df1b07 --- /dev/null +++ b/src/invidious/user/cookies.cr @@ -0,0 +1,37 @@ +require "http/cookie" + +struct Invidious::User + module Cookies + extend self + + # Note: we use ternary operator because the two variables + # used in here are not booleans. + SECURE = (Kemal.config.ssl || CONFIG.https_only) ? true : false + + # Session ID (SID) cookie + # Parameter "domain" comes from the global config + def sid(domain : String?, sid) : HTTP::Cookie + return HTTP::Cookie.new( + name: "SID", + domain: domain, + value: sid, + expires: Time.utc + 2.years, + secure: SECURE, + http_only: true + ) + end + + # Preferences (PREFS) cookie + # Parameter "domain" comes from the global config + def prefs(domain : String?, preferences : Preferences) : HTTP::Cookie + return HTTP::Cookie.new( + name: "PREFS", + domain: domain, + value: URI.encode_www_form(preferences.to_json), + expires: Time.utc + 2.years, + secure: SECURE, + http_only: true + ) + end + end +end diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 2ae1dcb1..f8b9e4e4 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -1,27 +1,242 @@ require "csv" -def parse_subscription_export_csv(csv_content : String) - rows = CSV.new(csv_content, headers: true) - subscriptions = Array(String).new +struct Invidious::User + module Import + extend self - # Counter to limit the amount of imports. - # This is intended to prevent DoS. - row_counter = 0 + # Parse a youtube CSV subscription file + def parse_subscription_export_csv(csv_content : String) + rows = CSV.new(csv_content, headers: true) + subscriptions = Array(String).new - rows.each do |row| - # Limit to 1200 - row_counter += 1 - break if row_counter > 1_200 + # Counter to limit the amount of imports. + # This is intended to prevent DoS. + row_counter = 0 - # Channel ID is the first column in the csv export we can't use the header - # name, because the header name is localized depending on the - # language the user has set on their account - channel_id = row[0].strip + rows.each do |row| + # Limit to 1200 + row_counter += 1 + break if row_counter > 1_200 - next if channel_id.empty? + # Channel ID is the first column in the csv export we can't use the header + # name, because the header name is localized depending on the + # language the user has set on their account + channel_id = row[0].strip - subscriptions << channel_id - end + next if channel_id.empty? + subscriptions << channel_id + end - return subscriptions + return subscriptions + end + + # ------------------- + # Invidious + # ------------------- + + # Import from another invidious account + def from_invidious(user : User, body : String) + data = JSON.parse(body) + + if data["subscriptions"]? + user.subscriptions += data["subscriptions"].as_a.map(&.as_s) + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + end + + if data["watch_history"]? + user.watched += data["watch_history"].as_a.map(&.as_s) + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + end + + if data["preferences"]? + user.preferences = Preferences.from_json(data["preferences"].to_json) + Invidious::Database::Users.update_preferences(user) + end + + if playlists = data["playlists"]?.try &.as_a? + playlists.each do |item| + title = item["title"]?.try &.as_s?.try &.delete("<>") + description = item["description"]?.try &.as_s?.try &.delete("\r") + privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } + + next if !title + next if !description + next if !privacy + + playlist = create_playlist(title, privacy, user) + Invidious::Database::Playlists.update_description(playlist.id, description) + + videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| + raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 + + video_id = video_id.try &.as_s? + next if !video_id + + begin + video = get_video(video_id) + rescue ex + next + end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: playlist.id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) + end + end + end + end + + # ------------------- + # Youtube + # ------------------- + + private def is_opml?(mimetype : String, extension : String) + opml_mimetypes = [ + "application/xml", + "text/xml", + "text/x-opml", + "text/x-opml+xml", + ] + + opml_extensions = ["xml", "opml"] + + return opml_mimetypes.any?(&.== mimetype) || opml_extensions.any?(&.== extension) + end + + # Import subscribed channels from Youtube + # Returns success status + def from_youtube(user : User, body : String, filename : String, type : String) : Bool + extension = filename.split(".").last + + if is_opml?(type, extension) + subscriptions = XML.parse(body) + user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| + channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + end + elsif extension == "json" || type == "application/json" + subscriptions = JSON.parse(body) + user.subscriptions += subscriptions.as_a.compact_map do |entry| + entry["snippet"]["resourceId"]["channelId"].as_s + end + elsif extension == "csv" || type == "text/csv" + subscriptions = parse_subscription_export_csv(body) + user.subscriptions += subscriptions + else + return false + end + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + return true + end + + # ------------------- + # Freetube + # ------------------- + + def from_freetube(user : User, body : String) + # Legacy import? + matches = body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/) + subs = matches.map(&.["channel_id"]) + + if subs.empty? + data = JSON.parse(body)["subscriptions"] + subs = data.as_a.map(&.["id"].as_s) + end + + user.subscriptions += subs + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + end + + # ------------------- + # Newpipe + # ------------------- + + def from_newpipe_subs(user : User, body : String) + data = JSON.parse(body) + + user.subscriptions += data["subscriptions"].as_a.compact_map do |channel| + if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/) + next match["channel"] + elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/) + # Resolve URL using the API + resolved_url = YoutubeAPI.resolve_url("https://www.youtube.com/user/#{match["user"]}") + ucid = resolved_url.dig?("endpoint", "browseEndpoint", "browseId") + next ucid.as_s if ucid + end + + nil + end + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + end + + def from_newpipe(user : User, body : String) : Bool + io = IO::Memory.new(body) + + Compress::Zip::File.open(io) do |file| + file.entries.each do |entry| + entry.open do |file_io| + # Ensure max size of 4MB + io_sized = IO::Sized.new(file_io, 0x400000) + + next if entry.filename != "newpipe.db" + + tempfile = File.tempfile(".db") + + begin + File.write(tempfile.path, io_sized.gets_to_end) + rescue + return false + end + + db = DB.open("sqlite3://" + tempfile.path) + + user.watched += db.query_all("SELECT url FROM streams", as: String) + .map(&.lchop("https://www.youtube.com/watch?v=")) + + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + + user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) + .map(&.lchop("https://www.youtube.com/channel/")) + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + + db.close + tempfile.delete + end + end + end + + # Success! + return true + end + end # module end diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index bf7ea401..b3059403 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -23,6 +23,7 @@ struct Preferences property latest_only : Bool = CONFIG.default_user_preferences.latest_only property listen : Bool = CONFIG.default_user_preferences.listen property local : Bool = CONFIG.default_user_preferences.local + property watch_history : Bool = CONFIG.default_user_preferences.watch_history property vr_mode : Bool = CONFIG.default_user_preferences.vr_mode property show_nick : Bool = CONFIG.default_user_preferences.show_nick @@ -256,4 +257,18 @@ struct Preferences cookies end end + + module TimeSpanConverter + def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder) + return yaml.scalar value.total_minutes.to_i32 + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span + if node.is_a?(YAML::Nodes::Scalar) + return decode_interval(node.value) + else + node.raise "Expected scalar, not #{node.class}" + end + end + end end diff --git a/src/invidious/user/user.cr b/src/invidious/user/user.cr new file mode 100644 index 00000000..a6d05fd1 --- /dev/null +++ b/src/invidious/user/user.cr @@ -0,0 +1,27 @@ +require "db" + +struct Invidious::User + include DB::Serializable + + property updated : Time + property notifications : Array(String) + property subscriptions : Array(String) + property email : String + + @[DB::Field(converter: Invidious::User::PreferencesConverter)] + property preferences : Preferences + property password : String? + property token : String + property watched : Array(String) + property feed_needs_update : Bool? + + module PreferencesConverter + def self.from_rs(rs) + begin + Preferences.from_json(rs.read(String)) + rescue ex + Preferences.from_json("{}") + end + end + end +end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 9810f8a2..b763596b 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -3,32 +3,6 @@ require "crypto/bcrypt/password" # Materialized views may not be defined using bound parameters (`$1` as used elsewhere) MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" } -struct User - include DB::Serializable - - property updated : Time - property notifications : Array(String) - property subscriptions : Array(String) - property email : String - - @[DB::Field(converter: User::PreferencesConverter)] - property preferences : Preferences - property password : String? - property token : String - property watched : Array(String) - property feed_needs_update : Bool? - - module PreferencesConverter - def self.from_rs(rs) - begin - Preferences.from_json(rs.read(String)) - rescue ex - Preferences.from_json("{}") - end - end - end -end - def get_user(sid, headers, refresh = true) if email = Invidious::Database::SessionIDs.select_email(sid) user = Invidious::Database::Users.select!(email: email) @@ -84,7 +58,7 @@ def fetch_user(sid, headers) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new({ + user = Invidious::User.new({ updated: Time.utc, notifications: [] of String, subscriptions: channels, @@ -102,7 +76,7 @@ def create_user(sid, email, password) password = Crypto::Bcrypt::Password.create(password, cost: 10) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new({ + user = Invidious::User.new({ updated: Time.utc, notifications: [] of String, subscriptions: [] of String, @@ -117,75 +91,6 @@ def create_user(sid, email, password) return user, sid end -def generate_captcha(key) - second = Random::Secure.rand(12) - second_angle = second * 30 - second = second * 5 - - minute = Random::Secure.rand(12) - minute_angle = minute * 30 - minute = minute * 5 - - hour = Random::Secure.rand(12) - hour_angle = hour * 30 + minute_angle.to_f / 12 - if hour == 0 - hour = 12 - end - - clock_svg = <<-END_SVG - <svg viewBox="0 0 100 100" width="200px" height="200px"> - <circle cx="50" cy="50" r="45" fill="#eee" stroke="black" stroke-width="2"></circle> - - <text x="69" y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 1</text> - <text x="82.909" y="34" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 2</text> - <text x="88" y="53" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 3</text> - <text x="82.909" y="72" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 4</text> - <text x="69" y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 5</text> - <text x="50" y="91" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 6</text> - <text x="31" y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 7</text> - <text x="17.091" y="72" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 8</text> - <text x="12" y="53" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 9</text> - <text x="17.091" y="34" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">10</text> - <text x="31" y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">11</text> - <text x="50" y="15" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">12</text> - - <circle cx="50" cy="50" r="3" fill="black"></circle> - <line id="second" transform="rotate(#{second_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="12" fill="black" stroke="black" stroke-width="1"></line> - <line id="minute" transform="rotate(#{minute_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="16" fill="black" stroke="black" stroke-width="2"></line> - <line id="hour" transform="rotate(#{hour_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="24" fill="black" stroke="black" stroke-width="2"></line> - </svg> - END_SVG - - image = "data:image/png;base64," - image += Process.run(%(rsvg-convert -w 400 -h 400 -b none -f png), shell: true, - input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe - ) do |proc| - Base64.strict_encode(proc.output.gets_to_end) - end - - answer = "#{hour}:#{minute.to_s.rjust(2, '0')}:#{second.to_s.rjust(2, '0')}" - answer = OpenSSL::HMAC.hexdigest(:sha256, key, answer) - - return { - question: image, - tokens: {generate_response(answer, {":login"}, key, use_nonce: true)}, - } -end - -def generate_text_captcha(key) - response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body) - response = JSON.parse(response) - - tokens = response["a"].as_a.map do |answer| - generate_response(answer.as_s, {":login"}, key, use_nonce: true) - end - - return { - question: response["q"].as_s, - tokens: tokens, - } -end - def subscribe_ajax(channel_id, action, env_headers) headers = HTTP::Headers.new headers["Cookie"] = env_headers["Cookie"] diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 335f6b60..81fce5b8 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -881,11 +881,13 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) - if player_response["playabilityStatus"]?.try &.["status"]?.try &.as_s != "OK" - reason = player_response["playabilityStatus"]["errorScreen"]?.try &.["playerErrorMessageRenderer"]?.try &.["subreason"]?.try { |s| - s["simpleText"]?.try &.as_s || s["runs"].as_a.map { |r| r["text"] }.join("") - } || player_response["playabilityStatus"]["reason"].as_s + if player_response.dig?("playabilityStatus", "status").try &.as_s != "OK" + subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") + reason = subreason.try &.[]?("simpleText").try &.as_s + reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") + reason ||= player_response.dig("playabilityStatus", "reason").as_s params["reason"] = JSON::Any.new(reason) + return params end params["shortDescription"] = player_response.dig?("videoDetails", "shortDescription") || JSON::Any.new(nil) @@ -928,11 +930,8 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results primary_results = main_results.dig?("results", "results", "contents") - secondary_results = main_results - .dig?("secondaryResults", "secondaryResults", "results") raise BrokenTubeException.new("results") if !primary_results - raise BrokenTubeException.new("secondaryResults") if !secondary_results video_primary_renderer = primary_results .as_a.find(&.["videoPrimaryInfoRenderer"]?) @@ -952,7 +951,9 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ related = [] of JSON::Any # Parse "compactVideoRenderer" items (under secondary results) - secondary_results.as_a.each do |element| + secondary_results = main_results + .dig?("secondaryResults", "secondaryResults", "results") + secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? related_video = parse_related_video(item) related << JSON::Any.new(related_video) if related_video @@ -1119,7 +1120,9 @@ def fetch_video(id, region) info = embed_info if !embed_info["reason"]? end - raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]? + if reason = info["reason"]? + raise InfoException.new(reason.as_s || "") + end video = Video.new({ id: id, diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 92df1272..bd908dd6 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -52,7 +52,7 @@ </div> <div class="pure-u-1-4"> <a id="notification_ticker" title="<%= translate(locale, "Subscriptions") %>" href="/feed/subscriptions" class="pure-menu-heading"> - <% notification_count = env.get("user").as(User).notifications.size %> + <% notification_count = env.get("user").as(Invidious::User).notifications.size %> <% if notification_count > 0 %> <span id="notification_count"><%= notification_count %></span> <i class="icon ion-ios-notifications"></i> <% else %> @@ -67,7 +67,7 @@ </div> <% if env.get("preferences").as(Preferences).show_nick %> <div class="pure-u-1-4"> - <span id="user_name"><%= env.get("user").as(User).email %></span> + <span id="user_name"><%= env.get("user").as(Invidious::User).email %></span> </div> <% end %> <div class="pure-u-1-4"> diff --git a/src/invidious/views/authorize_token.ecr b/src/invidious/views/user/authorize_token.ecr index 725f392e..725f392e 100644 --- a/src/invidious/views/authorize_token.ecr +++ b/src/invidious/views/user/authorize_token.ecr diff --git a/src/invidious/views/change_password.ecr b/src/invidious/views/user/change_password.ecr index 1b9eb82e..1b9eb82e 100644 --- a/src/invidious/views/change_password.ecr +++ b/src/invidious/views/user/change_password.ecr diff --git a/src/invidious/views/clear_watch_history.ecr b/src/invidious/views/user/clear_watch_history.ecr index c9acbe44..c9acbe44 100644 --- a/src/invidious/views/clear_watch_history.ecr +++ b/src/invidious/views/user/clear_watch_history.ecr diff --git a/src/invidious/views/data_control.ecr b/src/invidious/views/user/data_control.ecr index 74ccc06c..74ccc06c 100644 --- a/src/invidious/views/data_control.ecr +++ b/src/invidious/views/user/data_control.ecr diff --git a/src/invidious/views/delete_account.ecr b/src/invidious/views/user/delete_account.ecr index 67351bbf..67351bbf 100644 --- a/src/invidious/views/delete_account.ecr +++ b/src/invidious/views/user/delete_account.ecr diff --git a/src/invidious/views/login.ecr b/src/invidious/views/user/login.ecr index 01d7a210..01d7a210 100644 --- a/src/invidious/views/login.ecr +++ b/src/invidious/views/user/login.ecr diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/user/preferences.ecr index 96904259..dbb5e9db 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -207,6 +207,11 @@ <legend><%= translate(locale, "preferences_category_subscription") %></legend> <div class="pure-control-group"> + <label for="watch_history"><%= translate(locale, "preferences_watch_history_label") %></label> + <input name="watch_history" id="watch_history" type="checkbox" <% if preferences.watch_history %>checked<% end %>> + </div> + + <div class="pure-control-group"> <label for="annotations_subscribed"><%= translate(locale, "preferences_annotations_subscribed_label") %></label> <input name="annotations_subscribed" id="annotations_subscribed" type="checkbox" <% if preferences.annotations_subscribed %>checked<% end %>> </div> @@ -252,7 +257,7 @@ <% end %> <% end %> - <% if env.get?("user") && CONFIG.admins.includes? env.get?("user").as(User).email %> + <% if env.get?("user") && CONFIG.admins.includes? env.get?("user").as(Invidious::User).email %> <legend><%= translate(locale, "preferences_category_admin") %></legend> <div class="pure-control-group"> diff --git a/src/invidious/views/subscription_manager.ecr b/src/invidious/views/user/subscription_manager.ecr index c2a89ca2..c2a89ca2 100644 --- a/src/invidious/views/subscription_manager.ecr +++ b/src/invidious/views/user/subscription_manager.ecr diff --git a/src/invidious/views/token_manager.ecr b/src/invidious/views/user/token_manager.ecr index 79f905a1..79f905a1 100644 --- a/src/invidious/views/token_manager.ecr +++ b/src/invidious/views/user/token_manager.ecr diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 2e0aee99..0e4af3ab 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -168,41 +168,7 @@ we're going to need to do it here in order to allow for translations. <% end %> <% end %> - <% if CONFIG.dmca_content.includes?(video.id) || CONFIG.disabled?("downloads") %> - <p id="download"><%= translate(locale, "Download is disabled.") %></p> - <% else %> - <form class="pure-form pure-form-stacked" action="/latest_version" method="get" rel="noopener" target="_blank"> - <div class="pure-control-group"> - <label for="download_widget"><%= translate(locale, "Download as: ") %></label> - <select style="width:100%" name="download_widget" id="download_widget"> - <% fmt_stream.each do |option| %> - <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'> - <%= itag_to_metadata?(option["itag"]).try &.["height"]? || "~240" %>p - <%= option["mimeType"].as_s.split(";")[0] %> - </option> - <% end %> - <% video_streams.each do |option| %> - <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'> - <%= option["qualityLabel"] %> - <%= option["mimeType"].as_s.split(";")[0] %> @ <%= option["fps"] %>fps - video only - </option> - <% end %> - <% audio_streams.each do |option| %> - <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["mimeType"].as_s.split(";")[0].split("/")[1] %>"}'> - <%= option["mimeType"].as_s.split(";")[0] %> @ <%= option["bitrate"]?.try &.as_i./ 1000 %>k - audio only - </option> - <% end %> - <% captions.each do |caption| %> - <option value='{"id":"<%= video.id %>","label":"<%= caption.name %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= caption.language_code %>.vtt"}'> - <%= translate(locale, "download_subtitles", translate(locale, caption.name)) %> - </option> - <% end %> - </select> - </div> - - <button type="submit" class="pure-button pure-button-primary"> - <b><%= translate(locale, "Download") %></b> - </button> - </form> - <% end %> + <%= Invidious::Frontend::WatchPage.download_widget(locale, video, video_assets) %> <p id="views"><i class="icon ion-ios-eye"></i> <%= number_with_separator(video.views) %></p> <p id="likes"><i class="icon ion-ios-thumbs-up"></i> <%= number_with_separator(video.likes) %></p> |