diff options
Diffstat (limited to 'src')
33 files changed, 270 insertions, 266 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 42a2b23a..c5101e4f 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -17,7 +17,6 @@ require "digest/md5" require "file_utils" require "kemal" -require "markdown" require "openssl/hmac" require "option_parser" require "pg" @@ -86,6 +85,7 @@ LOCALES = { "nl" => load_locale("nl"), "pl" => load_locale("pl"), "ru" => load_locale("ru"), + "tr" => load_locale("tr"), "uk" => load_locale("uk"), "zh-CN" => load_locale("zh-CN"), } @@ -296,7 +296,7 @@ before_all do |env| current_page += "?#{query}" end - env.set "current_page", URI.escape(current_page) + env.set "current_page", URI.encode_www_form(current_page) end get "/" do |env| @@ -391,7 +391,7 @@ get "/watch" do |env| begin video = get_video(id, PG_DB, region: params.region) rescue ex : VideoRedirect - next env.redirect "/watch?v=#{ex.message}" + next env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -668,7 +668,7 @@ get "/embed/:id" do |env| begin video = get_video(id, PG_DB, region: params.region) rescue ex : VideoRedirect - next env.redirect "/embed/#{ex.message}" + next env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -841,7 +841,7 @@ get "/results" do |env| page ||= 1 if query - env.redirect "/search?q=#{URI.escape(query)}&page=#{page}" + env.redirect "/search?q=#{URI.encode_www_form(query)}&page=#{page}" else env.redirect "/" end @@ -1018,6 +1018,7 @@ post "/login" do |env| headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8" headers["Google-Accounts-XSRF"] = "1" + headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36" response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req)) lookup_results = JSON.parse(response.body[5..-1]) @@ -1049,7 +1050,7 @@ post "/login" do |env| traceback << "done, returned #{response.status_code}.<br/>" - headers["Cookie"] = URI.unescape(headers["Cookie"]) + headers["Cookie"] = URI.decode_www_form(headers["Cookie"]) if challenge_results[0][3]?.try &.== 7 error_message = translate(locale, "Account has temporarily been disabled") @@ -1057,6 +1058,13 @@ post "/login" do |env| next templated "error" end + # TODO: Handle Google's CAPTCHA + if captcha = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a? + error_message = "Unhandled CAPTCHA. Please try again later." + env.response.status_code = 401 + next templated "error" + end + if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED" error_message = translate(locale, "Incorrect password") env.response.status_code = 401 @@ -1074,7 +1082,7 @@ post "/login" do |env| end # Prefer Authenticator app and SMS over unsupported protocols - if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8]) && prompt_type == 4 + if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2 tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0] traceback << "Selecting challenge #{tfa[8]}..." @@ -1182,8 +1190,12 @@ post "/login" do |env| break end - # TODO: Occasionally there will be a second page after login confirming - # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently choke on. + # Occasionally there will be a second page after login confirming + # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle. + + if location.includes? "/b/0/SmsAuthInterstitial" + traceback << "Unhandled dialog /b/0/SmsAuthInterstitial." + end login = client.get(location, headers) headers = login.cookies.add_request_headers(headers) @@ -1384,7 +1396,7 @@ post "/login" do |env| user_array[4] = user_array[4].to_json args = arg_array(user_array) - PG_DB.exec("INSERT INTO users VALUES (#{args})", user_array) + PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) PG_DB.exec("INSERT INTO session_ids VALUES ($1, $2, $3)", sid, email, Time.utc) view_name = "subscriptions_#{sha256(user.email)}" @@ -2411,7 +2423,7 @@ post "/authorize_token" do |env| access_token = generate_token(user.email, scopes, expire, HMAC_KEY, PG_DB) if callback_url - access_token = URI.escape(access_token) + access_token = URI.encode_www_form(access_token) url = URI.parse(callback_url) if url.query @@ -2634,6 +2646,8 @@ get "/feed/channel/:ucid" do |env| begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -2900,7 +2914,7 @@ post "/feed/webhook/:token" do |env| PG_DB.exec("INSERT INTO channel_videos VALUES (#{args}) \ ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, premiere_timestamp = $9, views = $10", video_array) + live_now = $8, premiere_timestamp = $9, views = $10", args: video_array) # Update all users affected by insert if emails.empty? @@ -3041,6 +3055,8 @@ get "/channel/:ucid" do |env| begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -3108,6 +3124,8 @@ get "/channel/:ucid/playlists" do |env| begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -3146,6 +3164,8 @@ get "/channel/:ucid/community" do |env| begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + next env.redirect env.request.resource.gsub(ucid, ex.channel_id) rescue ex error_message = ex.message env.response.status_code = 500 @@ -3201,7 +3221,10 @@ get "/api/v1/storyboards/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - next env.redirect "/api/v1/storyboards/#{ex.message}" + error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + next error_message rescue ex env.response.status_code = 500 next @@ -3286,7 +3309,10 @@ get "/api/v1/captions/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - next env.redirect "/api/v1/captions/#{ex.message}" + error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + next error_message rescue ex env.response.status_code = 500 next @@ -3307,7 +3333,7 @@ get "/api/v1/captions/:id" do |env| json.object do json.field "label", caption.name.simpleText json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.escape(caption.name.simpleText)}" + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}" end end end @@ -3386,7 +3412,7 @@ get "/api/v1/captions/:id" do |env| if title = env.params.query["title"]? # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.escape(title)}\"; filename*=UTF-8''#{URI.escape(title)}" + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" end webvtt @@ -3574,7 +3600,7 @@ get "/api/v1/annotations/:id" do |env| id = id.sub(/^-/, 'A') end - file = URI.escape("#{id[0, 3]}/#{id}.xml") + file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") client = make_client(ARCHIVE_URL) location = client.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}") @@ -3626,7 +3652,10 @@ get "/api/v1/videos/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - next env.redirect "/api/v1/videos/#{ex.message}" + error_message = {"error" => "Video is unavailable", "videoId" => ex.video_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id) + next error_message rescue ex error_message = {"error" => ex.message}.to_json env.response.status_code = 500 @@ -3729,6 +3758,11 @@ get "/api/v1/channels/:ucid" do |env| begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + next error_message rescue ex error_message = {"error" => ex.message}.to_json env.response.status_code = 500 @@ -3859,6 +3893,11 @@ end begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + next error_message rescue ex error_message = {"error" => ex.message}.to_json env.response.status_code = 500 @@ -3923,6 +3962,11 @@ end begin channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + error_message = {"error" => "Channel is unavailable", "authorId" => ex.channel_id}.to_json + env.response.status_code = 302 + env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) + next error_message rescue ex error_message = {"error" => ex.message}.to_json env.response.status_code = 500 @@ -4055,7 +4099,7 @@ get "/api/v1/search/suggestions" do |env| begin client = make_client(URI.parse("https://suggestqueries.google.com")) - response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.escape(query)}&callback=suggestCallback").body + response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback").body body = response[35..-2] body = JSON.parse(body).as_a @@ -4439,7 +4483,7 @@ post "/api/v1/auth/tokens/register" do |env| access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY, PG_DB) if callback_url - access_token = URI.escape(access_token) + access_token = URI.encode_www_form(access_token) if query = callback_url.query query = HTTP::Params.parse(query.not_nil!) @@ -4507,11 +4551,7 @@ get "/api/manifest/dash/id/:id" do |env| begin video = get_video(id, PG_DB, region: region) rescue ex : VideoRedirect - url = "/api/manifest/dash/id/#{ex.message}" - if env.params.query - url += "?#{env.params.query}" - end - next env.redirect url + next env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex env.response.status_code = 403 next @@ -4678,7 +4718,7 @@ get "/api/manifest/hls_playlist/*" do |env| raw_params = {} of String => Array(String) path.each_slice(2) do |pair| key, value = pair - value = URI.unescape(value) + value = URI.decode_www_form(value) if raw_params[key]? raw_params[key] << value @@ -4803,7 +4843,7 @@ get "/videoplayback/*" do |env| raw_params = {} of String => Array(String) path.each_slice(2) do |pair| key, value = pair - value = URI.unescape(value) + value = URI.decode_www_form(value) if raw_params[key]? raw_params[key] << value @@ -4977,7 +5017,7 @@ get "/videoplayback" do |env| if title = query_params["title"]? # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/ - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.escape(title)}\"; filename*=UTF-8''#{URI.escape(title)}" + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" end if !response.headers.includes_word?("Transfer-Encoding", "chunked") @@ -5290,4 +5330,6 @@ add_context_storage_type(Preferences) add_context_storage_type(User) Kemal.config.logger = logger +Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding +Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port Kemal.run diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 85502505..cce98310 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -120,7 +120,7 @@ struct AboutChannel description_html: String, paid: Bool, total_views: Int64, - sub_count: Int64, + sub_count: Int32, joined: Time, is_family_friendly: Bool, allowed_regions: Array(String), @@ -129,6 +129,13 @@ struct AboutChannel }) end +class ChannelRedirect < Exception + property channel_id : String + + def initialize(@channel_id) + end +end + def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, max_threads = 10) finished_channel = Channel(String | Nil).new @@ -174,14 +181,14 @@ def get_channel(id, db, refresh = true, pull_all_videos = true) args = arg_array(channel_array) db.exec("INSERT INTO channels VALUES (#{args}) \ - ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", channel_array) + ON CONFLICT (id) DO UPDATE SET author = $2, updated = $3", args: channel_array) end else channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) channel_array = channel.to_a args = arg_array(channel_array) - db.exec("INSERT INTO channels VALUES (#{args})", channel_array) + db.exec("INSERT INTO channels VALUES (#{args})", args: channel_array) end return channel @@ -270,7 +277,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) db.exec("INSERT INTO channel_videos VALUES (#{args}) \ ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10", video_array) + live_now = $8, views = $10", args: video_array) # Update all users affected by insert if emails.empty? @@ -338,7 +345,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) db.exec("INSERT INTO channel_videos VALUES (#{args}) \ ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ updated = $4, ucid = $5, author = $6, length_seconds = $7, \ - live_now = $8, views = $10", video_array) + live_now = $8, views = $10", args: video_array) # Update all users affected by insert if emails.empty? @@ -471,7 +478,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " end data = Base64.urlsafe_encode(data) - cursor = URI.escape(data) + cursor = URI.encode_www_form(data) data = IO::Memory.new @@ -492,7 +499,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" @@ -542,7 +549,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated data.rewind data = Base64.urlsafe_encode(data) - continuation = URI.escape(data) + continuation = URI.encode_www_form(data) data = IO::Memory.new @@ -563,7 +570,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" @@ -573,7 +580,7 @@ end def extract_channel_playlists_cursor(url, auto_generated) continuation = HTTP::Params.parse(URI.parse(url).query.not_nil!)["continuation"] - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0xe2 0xa9 0x85 0xb2 0x02 @@ -592,7 +599,7 @@ def extract_channel_playlists_cursor(url, auto_generated) data.read inner_continuation continuation = String.new(inner_continuation) - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0x12 0x09 playlists @@ -609,7 +616,7 @@ def extract_channel_playlists_cursor(url, auto_generated) cursor = String.new(cursor) if !auto_generated - cursor = URI.unescape(cursor) + cursor = URI.decode_www_form(cursor) cursor = Base64.decode_string(cursor) end @@ -872,7 +879,7 @@ def fetch_channel_community(ucid, continuation, locale, config, kemal_config, fo end def produce_channel_community_continuation(ucid, cursor) - cursor = URI.escape(cursor) + cursor = URI.encode_www_form(cursor) data = IO::Memory.new @@ -893,13 +900,13 @@ def produce_channel_community_continuation(ucid, cursor) IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) return continuation end def extract_channel_community_cursor(continuation) - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0xe2 0xa9 0x85 0xb2 0x02 @@ -918,7 +925,7 @@ def extract_channel_community_cursor(continuation) data.read_byte end - return URI.unescape(data.gets_to_end) + return URI.decode_www_form(data.gets_to_end) end def get_about_info(ucid, locale) @@ -929,6 +936,10 @@ def get_about_info(ucid, locale) about = client.get("/user/#{ucid}/about?disable_polymer=1&gl=US&hl=en") end + if md = about.headers["location"]?.try &.match(/\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/) + raise ChannelRedirect.new(channel_id: md["ucid"]) + end + about = XML.parse_html(about.body) if about.xpath_node(%q(//div[contains(@class, "channel-empty-message")])) @@ -942,12 +953,6 @@ def get_about_info(ucid, locale) raise error_message end - sub_count = about.xpath_node(%q(//span[contains(text(), "subscribers")])) - if sub_count - sub_count = sub_count.content.delete(", subscribers").to_i? - end - sub_count ||= 0 - author = about.xpath_node(%q(//span[contains(@class,"qualified-channel-title-text")]/a)).not_nil!.content author_url = about.xpath_node(%q(//span[contains(@class,"qualified-channel-title-text")]/a)).not_nil!["href"] author_thumbnail = about.xpath_node(%q(//img[@class="channel-header-profile-image"])).not_nil!["src"] @@ -991,21 +996,14 @@ def get_about_info(ucid, locale) ) end - total_views = 0_i64 - sub_count = 0_i64 - - joined = Time.unix(0) - metadata = about.xpath_nodes(%q(//span[@class="about-stat"])) - metadata.each do |item| - case item.content - when .includes? "views" - total_views = item.content.gsub(/\D/, "").to_i64 - when .includes? "subscribers" - sub_count = item.content.delete("subscribers").gsub(/\D/, "").to_i64 - when .includes? "Joined" - joined = Time.parse(item.content.lchop("Joined "), "%b %-d, %Y", Time::Location.local) - end - end + joined = about.xpath_node(%q(//span[contains(., "Joined")])) + .try &.content.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) + + total_views = about.xpath_node(%q(//span[contains(., "views")]/b)) + .try &.content.try &.gsub(/\D/, "").to_i64? || 0_i64 + + sub_count = about.xpath_node(%q(.//span[contains(@class, "subscriber-count")])) + .try &.["title"].try { |text| short_text_to_number(text) } || 0 # Auto-generated channels # https://support.google.com/youtube/answer/2579942 @@ -1017,7 +1015,7 @@ def get_about_info(ucid, locale) tabs = about.xpath_nodes(%q(//ul[@id="channel-navigation-menu"]/li/a/span)).map { |node| node.content.downcase } - return AboutChannel.new( + AboutChannel.new( ucid: ucid, author: author, auto_generated: auto_generated, diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 04ba6f5d..740449d3 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -573,7 +573,7 @@ def content_to_comment_html(content) end def extract_comment_cursor(continuation) - continuation = URI.unescape(continuation) + continuation = URI.decode_www_form(continuation) data = IO::Memory.new(Base64.decode(continuation)) # 0x12 0x26 @@ -653,7 +653,7 @@ def produce_comment_continuation(video_id, cursor = "", sort_by = "top") end continuation = Base64.urlsafe_encode(data) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) return continuation end @@ -695,7 +695,7 @@ def produce_comment_reply_continuation(video_id, ucid, comment_id) data.write(Bytes[0x48, 0x0a]) continuation = Base64.urlsafe_encode(data.to_slice) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) return continuation end diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 56c1c488..949eb335 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -95,8 +95,8 @@ class AuthHandler < Kemal::Handler begin if token = env.request.headers["Authorization"]? - token = JSON.parse(URI.unescape(token.lchop("Bearer "))) - session = URI.unescape(token["session"].as_s) + token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) + session = URI.decode_www_form(token["session"].as_s) scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, PG_DB, nil) if email = PG_DB.query_one?("SELECT email FROM session_ids WHERE id = $1", session, as: String) @@ -238,40 +238,15 @@ class HTTP::Client end end -# https://github.com/will/crystal-pg/pull/171 -class PG::Statement < ::DB::Statement - protected def perform_query(args : Enumerable) : ResultSet - params = args.map { |arg| PQ::Param.encode(arg) } - conn = self.conn - conn.send_parse_message(@sql) - conn.send_bind_message params - conn.send_describe_portal_message - conn.send_execute_message - conn.send_sync_message - conn.expect_frame PQ::Frame::ParseComplete - conn.expect_frame PQ::Frame::BindComplete - frame = conn.read - case frame - when PQ::Frame::RowDescription - fields = frame.fields - when PQ::Frame::NoData - fields = nil +struct Crystal::ThreadLocalValue(T) + @values = Hash(Thread, T).new + + def get(&block : -> T) + th = Thread.current + if !@values[th]? + @values[th] = yield else - raise "expected RowDescription or NoData, got #{frame}" + @values[th] end - ResultSet.new(self, fields) - rescue IO::Error - raise DB::ConnectionLost.new(connection) - end - - protected def perform_exec(args : Enumerable) : ::DB::ExecResult - result = perform_query(args) - result.each { } - ::DB::ExecResult.new( - rows_affected: result.rows_affected, - last_insert_id: 0_i64 # postgres doesn't support this - ) - rescue IO::Error - raise DB::ConnectionLost.new(connection) end end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 331f6360..615e62df 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -235,6 +235,8 @@ struct Config hsts: {type: Bool?, default: true}, # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely disable_proxy: {type: Bool? | Array(String)?, default: false}, # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' force_resolve: {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) + port: {type: Int32, default: 3000}, # Port to listen for connections (overrided by command line argument) + host_binding: {type: String, default: "0.0.0.0"}, # Host to bind (overrided by command line argument) }) end @@ -415,19 +417,19 @@ def extract_items(nodeset, ucid = nil, author_name = nil) author_thumbnail ||= "" - subscriber_count = node.xpath_node(%q(.//span[contains(@class, "yt-subscriber-count")])).try &.["title"].gsub(/\D/, "").to_i? - subscriber_count ||= 0 + subscriber_count = node.xpath_node(%q(.//span[contains(@class, "subscriber-count")])) + .try &.["title"].try { |text| short_text_to_number(text) } || 0 video_count = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li)).try &.content.split(" ")[0].gsub(/\D/, "").to_i? - video_count ||= 0 items << SearchChannel.new( author: author, ucid: ucid, author_thumbnail: author_thumbnail, subscriber_count: subscriber_count, - video_count: video_count, - description_html: description_html + video_count: video_count || 0, + description_html: description_html, + auto_generated: video_count ? false : true, ) else id = id.lchop("/watch?v=") diff --git a/src/invidious/helpers/patch_mapping.cr b/src/invidious/helpers/patch_mapping.cr index e138aa1c..19bd8ca1 100644 --- a/src/invidious/helpers/patch_mapping.cr +++ b/src/invidious/helpers/patch_mapping.cr @@ -50,7 +50,7 @@ macro patched_json_mapping(_properties_, strict = false) rescue exc : ::JSON::ParseException raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc) end - while %pull.kind != :end_object + until %pull.kind.end_object? %key_location = %pull.location key = %pull.read_object_key case key diff --git a/src/invidious/helpers/static_file_handler.cr b/src/invidious/helpers/static_file_handler.cr index 87edbcd3..20d92b9c 100644 --- a/src/invidious/helpers/static_file_handler.cr +++ b/src/invidious/helpers/static_file_handler.cr @@ -119,7 +119,7 @@ module Kemal config = Kemal.config.serve_static original_path = context.request.path.not_nil! - request_path = URI.unescape(original_path) + request_path = URI.decode_www_form(original_path) # File path cannot contains '\0' (NUL) because all filesystem I know # don't accept '\0' character as file name. diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index f946fc2c..30f7d4f4 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -69,7 +69,7 @@ end def validate_request(token, session, request, key, db, locale = nil) case token when String - token = JSON.parse(URI.unescape(token)).as_h + token = JSON.parse(URI.decode_www_form(token)).as_h when JSON::Any token = token.as_h when Nil diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 5a813486..ed55dc9c 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -157,7 +157,7 @@ def number_with_separator(number) number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse end -def short_text_to_number(short_text) +def short_text_to_number(short_text : String) : Int32 case short_text when .ends_with? "M" number = short_text.rstrip(" mM").to_f @@ -246,7 +246,7 @@ def get_referer(env, fallback = "/", unroll = true) if referer.query params = HTTP::Params.parse(referer.query.not_nil!) if params["referer"]? - referer = URI.parse(URI.unescape(params["referer"])) + referer = URI.parse(URI.decode_www_form(params["referer"])) else break end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 7965d990..a5383daf 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -172,7 +172,7 @@ def produce_playlist_url(id, index) continuation.print data data = Base64.urlsafe_encode(continuation) - cursor = URI.escape(data) + cursor = URI.encode_www_form(data) data = IO::Memory.new @@ -193,7 +193,7 @@ def produce_playlist_url(id, index) IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 56035160..08171ffe 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -187,8 +187,10 @@ struct SearchChannel end end + json.field "autoGenerated", self.auto_generated json.field "subCount", self.subscriber_count json.field "videoCount", self.video_count + json.field "description", html_to_content(self.description_html) json.field "descriptionHtml", self.description_html end @@ -211,6 +213,7 @@ struct SearchChannel subscriber_count: Int32, video_count: Int32, description_html: String, + auto_generated: Bool, }) end @@ -265,7 +268,7 @@ def search(query, page = 1, search_params = produce_search_params(content_type: return {0, [] of SearchItem} end - html = client.get("/results?q=#{URI.escape(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body + html = client.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en&disable_polymer=1").body if html.empty? return {0, [] of SearchItem} end @@ -370,7 +373,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte end token = Base64.urlsafe_encode(token.to_slice) - token = URI.escape(token) + token = URI.encode_www_form(token) return token end @@ -395,7 +398,7 @@ def produce_channel_search_url(ucid, query, page) data.rewind data = Base64.urlsafe_encode(data) - continuation = URI.escape(data) + continuation = URI.encode_www_form(data) data = IO::Memory.new @@ -420,7 +423,7 @@ def produce_channel_search_url(ucid, query, page) IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) - continuation = URI.escape(continuation) + continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 32908157..26db51ea 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -42,7 +42,7 @@ end def extract_plid(url) wrapper = HTTP::Params.parse(URI.parse(url).query.not_nil!)["bp"] - wrapper = URI.unescape(wrapper) + wrapper = URI.decode_www_form(wrapper) wrapper = Base64.decode(wrapper) # 0xe2 0x02 0x2e diff --git a/src/invidious/users.cr b/src/invidious/users.cr index d3da28d7..6149ae7a 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -107,7 +107,7 @@ def get_user(sid, headers, db, refresh = true) args = arg_array(user_array) db.exec("INSERT INTO users VALUES (#{args}) \ - ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", user_array) + ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array) db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) @@ -126,7 +126,7 @@ def get_user(sid, headers, db, refresh = true) args = arg_array(user.to_a) db.exec("INSERT INTO users VALUES (#{args}) \ - ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", user_array) + ON CONFLICT (email) DO UPDATE SET updated = $1, subscriptions = $3", args: user_array) db.exec("INSERT INTO session_ids VALUES ($1,$2,$3) \ ON CONFLICT (id) DO NOTHING", sid, user.email, Time.utc) @@ -295,7 +295,7 @@ def get_subscription_feed(db, user, max_results = 40, page = 1) args = arg_array(notifications) - notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", notifications, as: ChannelVideo) + notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: ChannelVideo) videos = [] of ChannelVideo notifications.sort_by! { |video| video.published }.reverse! diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index ef3f4d4b..e175ae39 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -316,10 +316,10 @@ struct Video json.field "premiereTimestamp", self.premiere_timestamp.not_nil!.to_unix end - if self.player_response["streamingData"]?.try &.["hlsManifestUrl"]? + if player_response["streamingData"]?.try &.["hlsManifestUrl"]? host_url = make_host_url(config, kemal_config) - hlsvp = self.player_response["streamingData"]["hlsManifestUrl"].as_s + hlsvp = player_response["streamingData"]["hlsManifestUrl"].as_s hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", host_url) json.field "hlsUrl", hlsvp @@ -408,7 +408,7 @@ struct Video json.object do json.field "label", caption.name.simpleText json.field "languageCode", caption.languageCode - json.field "url", "/api/v1/captions/#{id}?label=#{URI.escape(caption.name.simpleText)}" + json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name.simpleText)}" end end end @@ -489,7 +489,7 @@ struct Video end def live_now - live_now = self.player_response["videoDetails"]?.try &.["isLive"]?.try &.as_bool + live_now = player_response["videoDetails"]?.try &.["isLive"]?.try &.as_bool if live_now.nil? return false @@ -536,7 +536,7 @@ struct Video end def keywords - keywords = self.player_response["videoDetails"]?.try &.["keywords"]?.try &.as_a + keywords = player_response["videoDetails"]?.try &.["keywords"]?.try &.as_a keywords ||= [] of String return keywords @@ -545,7 +545,7 @@ struct Video def fmt_stream(decrypt_function) streams = [] of HTTP::Params - if fmt_streams = self.player_response["streamingData"]?.try &.["formats"]? + if fmt_streams = player_response["streamingData"]?.try &.["formats"]? fmt_streams.as_a.each do |fmt_stream| if !fmt_stream.as_h? next @@ -619,7 +619,7 @@ struct Video def adaptive_fmts(decrypt_function) adaptive_fmts = [] of HTTP::Params - if fmts = self.player_response["streamingData"]?.try &.["adaptiveFormats"]? + if fmts = player_response["streamingData"]?.try &.["adaptiveFormats"]? fmts.as_a.each do |adaptive_fmt| if !adaptive_fmt.as_h? next @@ -706,23 +706,18 @@ struct Video return audio_streams end - def recommended_videos - @recommended_json = JSON.parse(@info["recommended_videos"]) if !@recommended_json - @recommended_json.not_nil! - end - def player_response @player_json = JSON.parse(@info["player_response"]) if !@player_json @player_json.not_nil! end def storyboards - storyboards = self.player_response["storyboards"]? + storyboards = player_response["storyboards"]? .try &.as_h .try &.["playerStoryboardSpecRenderer"]? if !storyboards - storyboards = self.player_response["storyboards"]? + storyboards = player_response["storyboards"]? .try &.as_h .try &.["playerLiveStoryboardSpecRenderer"]? @@ -789,13 +784,8 @@ struct Video end def paid - reason = self.player_response["playabilityStatus"]?.try &.["reason"]? - - if reason == "This video requires payment to watch." - paid = true - else - paid = false - end + reason = player_response["playabilityStatus"]?.try &.["reason"]? + paid = reason == "This video requires payment to watch." ? true : false return paid end @@ -841,7 +831,7 @@ struct Video end def length_seconds - self.player_response["videoDetails"]["lengthSeconds"].as_s.to_i + player_response["videoDetails"]["lengthSeconds"].as_s.to_i end db_mapping({ @@ -887,6 +877,10 @@ struct CaptionName end class VideoRedirect < Exception + property video_id : String + + def initialize(@video_id) + end end def get_video(id, db, refresh = true, region = nil, force_refresh = false) @@ -906,7 +900,7 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false) db.exec("UPDATE videos SET (info,updated,title,views,likes,dislikes,wilson_score,\ published,description,language,author,ucid,allowed_regions,is_family_friendly,\ genre,genre_url,license,sub_count_text,author_thumbnail)\ - = (#{args}) WHERE id = $1", video_array) + = (#{args}) WHERE id = $1", args: video_array) rescue ex db.exec("DELETE FROM videos * WHERE id = $1", id) raise ex @@ -919,7 +913,7 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false) args = arg_array(video_array) if !region - db.exec("INSERT INTO videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", video_array) + db.exec("INSERT INTO videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", args: video_array) end end @@ -936,6 +930,9 @@ def extract_recommended(recommended_videos) recommended_video = HTTP::Params.new recommended_video["id"] = video_renderer["videoId"].as_s recommended_video["title"] = video_renderer["title"]["simpleText"].as_s + + next if !video_renderer["shortBylineText"]? + recommended_video["author"] = video_renderer["shortBylineText"]["runs"].as_a[0]["text"].as_s recommended_video["ucid"] = video_renderer["shortBylineText"]["runs"].as_a[0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s recommended_video["author_thumbnail"] = video_renderer["channelThumbnail"]["thumbnails"][0]["url"].as_s @@ -1113,15 +1110,18 @@ def extract_player_config(body, html) .try &.["results"]? .try &.as_a - rvs = extract_recommended(rvs) - rvs.each_with_index do |rv, i| - if !rv["view_count"]? + rvs = extract_recommended(rvs).compact_map do |rv| + if !rv["short_view_count_text"]? rv_params = rvs_params.select { |rv_params| rv_params["id"]? == (rv["id"]? || "") }[0]? - if rv_params - rvs[i]["short_view_count_text"] = rv_params["short_view_count_text"] + + if rv_params.try &.["short_view_count_text"]? + rv["short_view_count_text"] = rv_params.not_nil!["short_view_count_text"] + rv else - rvs.delete_at(i) + nil end + else + rv end end params["rvs"] = (rvs.map &.to_s).join(",") @@ -1151,7 +1151,7 @@ def fetch_video(id, region) response = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") if md = response.headers["location"]?.try &.match(/v=(?<id>[a-zA-Z0-9_-]{11})/) - raise VideoRedirect.new(md["id"]) + raise VideoRedirect.new(video_id: md["id"]) end html = XML.parse_html(response.body) @@ -1205,6 +1205,11 @@ def fetch_video(id, region) player_json = JSON.parse(info["player_response"]) + reason = player_json["playabilityStatus"]?.try &.["reason"]?.try &.as_s + if reason == "This video is not available." + raise "This video is not available." + end + title = player_json["videoDetails"]["title"].as_s author = player_json["videoDetails"]["author"]?.try &.as_s || "" ucid = player_json["videoDetails"]["channelId"]?.try &.as_s || "" @@ -1257,7 +1262,7 @@ def fetch_video(id, region) end license = html.xpath_node(%q(//h4[contains(text(),"License")]/parent::*/ul/li)).try &.content || "" - sub_count_text = html.xpath_node(%q(//span[contains(@class, "yt-subscriber-count")])).try &.["title"]? || "0" + sub_count_text = html.xpath_node(%q(//span[contains(@class, "subscriber-count")])).try &.["title"]? || "0" author_thumbnail = html.xpath_node(%(//span[@class="yt-thumb-clip"]/img)).try &.["data-thumb"]?.try &.gsub(/^\/\//, "https://") || "" video = Video.new(id, info, Time.utc, title, views, likes, dislikes, wilson_score, published, description_html, diff --git a/src/invidious/views/authorize_token.ecr b/src/invidious/views/authorize_token.ecr index 53b8f001..8ea99010 100644 --- a/src/invidious/views/authorize_token.ecr +++ b/src/invidious/views/authorize_token.ecr @@ -72,7 +72,7 @@ <input type="hidden" name="expire" value="<%= expire %>"> <% end %> - <input type="hidden" name="csrf_token" value="<%= URI.escape(csrf_token) %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(csrf_token) %>"> </form> </div> <% end %> diff --git a/src/invidious/views/change_password.ecr b/src/invidious/views/change_password.ecr index 2e68556b..fb558f1d 100644 --- a/src/invidious/views/change_password.ecr +++ b/src/invidious/views/change_password.ecr @@ -6,7 +6,7 @@ <div class="pure-u-1 pure-u-lg-1-5"></div> <div class="pure-u-1 pure-u-lg-3-5"> <div class="h-box"> - <form class="pure-form pure-form-aligned" action="/change_password?referer=<%= URI.escape(referer) %>" method="post"> + <form class="pure-form pure-form-aligned" action="/change_password?referer=<%= URI.encode_www_form(referer) %>" method="post"> <legend><%= translate(locale, "Change password") %></legend> <fieldset> @@ -23,7 +23,7 @@ <%= translate(locale, "Change password") %> </button> - <input type="hidden" name="csrf_token" value="<%= URI.escape(csrf_token) %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(csrf_token) %>"> </fieldset> </form> </div> diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 1074598d..b5eb46ea 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -34,7 +34,7 @@ <div class="h-box"> <% ucid = channel.ucid %> <% author = channel.author %> - <% sub_count_text = channel.sub_count.format %> + <% sub_count_text = number_to_short_text(channel.sub_count) %> <%= rendered "components/subscribe_widget" %> </div> diff --git a/src/invidious/views/clear_watch_history.ecr b/src/invidious/views/clear_watch_history.ecr index 2bb9884c..5f9d1032 100644 --- a/src/invidious/views/clear_watch_history.ecr +++ b/src/invidious/views/clear_watch_history.ecr @@ -3,7 +3,7 @@ <% end %> <div class="h-box"> - <form class="pure-form pure-form-aligned" action="/clear_watch_history?referer=<%= URI.escape(referer) %>" method="post"> + <form class="pure-form pure-form-aligned" action="/clear_watch_history?referer=<%= URI.encode_www_form(referer) %>" method="post"> <legend><%= translate(locale, "Clear watch history?") %></legend> <div class="pure-g"> @@ -13,12 +13,12 @@ </button> </div> <div class="pure-u-1-2"> - <a class="pure-button" href="<%= URI.escape(referer) %>"> + <a class="pure-button" href="<%= URI.encode_www_form(referer) %>"> <%= translate(locale, "No") %> </a> </div> </div> - <input type="hidden" name="csrf_token" value="<%= URI.escape(csrf_token) %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(csrf_token) %>"> </form> </div> diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr index 9d086b5d..218cc2d4 100644 --- a/src/invidious/views/community.ecr +++ b/src/invidious/views/community.ecr @@ -33,7 +33,7 @@ <div class="h-box"> <% ucid = channel.ucid %> <% author = channel.author %> - <% sub_count_text = channel.sub_count.format %> + <% sub_count_text = number_to_short_text(channel.sub_count) %> <%= rendered "components/subscribe_widget" %> </div> diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 71ae70df..d78d8c4b 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -11,7 +11,7 @@ <p><%= item.author %></p> </a> <p><%= translate(locale, "`x` subscribers", number_with_separator(item.subscriber_count)) %></p> - <p><%= translate(locale, "`x` videos", number_with_separator(item.video_count)) %></p> + <% if !item.auto_generated %><p><%= translate(locale, "`x` videos", number_with_separator(item.video_count)) %></p><% end %> <h5><%= item.description_html %></h5> <% when SearchPlaylist %> <% if item.id.starts_with? "RD" %> @@ -91,7 +91,7 @@ <img class="thumbnail" src="/vi/<%= item.id %>/mqdefault.jpg"/> <% if env.get? "show_watched" %> <form onsubmit="return false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post"> - <input type="hidden" name="csrf_token" value="<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <p class="watched"> <a onclick="mark_watched(this)" data-id="<%= item.id %>" href="javascript:void(0)"> <button type="submit" style="all:unset"> diff --git a/src/invidious/views/components/subscribe_widget.ecr b/src/invidious/views/components/subscribe_widget.ecr index 1d9966b3..471e6c1c 100644 --- a/src/invidious/views/components/subscribe_widget.ecr +++ b/src/invidious/views/components/subscribe_widget.ecr @@ -2,7 +2,7 @@ <% if subscriptions.includes? ucid %> <p> <form action="/subscription_ajax?action_remove_subscriptions=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post"> - <input type="hidden" name="csrf_token" value="<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <button data-type="unsubscribe" id="subscribe" class="pure-button pure-button-primary"> <b><input style="all:unset" type="submit" value="<%= translate(locale, "Unsubscribe") %> | <%= sub_count_text %>"></b> </button> @@ -11,7 +11,7 @@ <% else %> <p> <form action="/subscription_ajax?action_create_subscription_to_channel=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post"> - <input type="hidden" name="csrf_token" value="<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <button data-type="subscribe" id="subscribe" class="pure-button pure-button-primary"> <b><input style="all:unset" type="submit" value="<%= translate(locale, "Subscribe") %> | <%= sub_count_text %>"></b> </button> @@ -24,7 +24,7 @@ ucid: '<%= ucid %>', author: '<%= HTML.escape(author) %>', sub_count_text: '<%= HTML.escape(sub_count_text) %>', - csrf_token: '<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>', + csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>', subscribe_text: '<%= HTML.escape(translate(locale, "Subscribe")) %>', unsubscribe_text: '<%= HTML.escape(translate(locale, "Unsubscribe")) %>' } diff --git a/src/invidious/views/data_control.ecr b/src/invidious/views/data_control.ecr index 463d5fd4..e3edb9ea 100644 --- a/src/invidious/views/data_control.ecr +++ b/src/invidious/views/data_control.ecr @@ -3,7 +3,7 @@ <% end %> <div class="h-box"> - <form class="pure-form pure-form-aligned" enctype="multipart/form-data" action="/data_control?referer=<%= URI.escape(referer) %>" method="post"> + <form class="pure-form pure-form-aligned" enctype="multipart/form-data" action="/data_control?referer=<%= URI.encode_www_form(referer) %>" method="post"> <fieldset> <legend><%= translate(locale, "Import") %></legend> diff --git a/src/invidious/views/delete_account.ecr b/src/invidious/views/delete_account.ecr index 0fa1e77c..9103d5b7 100644 --- a/src/invidious/views/delete_account.ecr +++ b/src/invidious/views/delete_account.ecr @@ -3,7 +3,7 @@ <% end %> <div class="h-box"> - <form class="pure-form pure-form-aligned" action="/delete_account?referer=<%= URI.escape(referer) %>" method="post"> + <form class="pure-form pure-form-aligned" action="/delete_account?referer=<%= URI.encode_www_form(referer) %>" method="post"> <legend><%= translate(locale, "Delete account?") %></legend> <div class="pure-g"> @@ -13,12 +13,12 @@ </button> </div> <div class="pure-u-1-2"> - <a class="pure-button" href="<%= URI.escape(referer) %>"> + <a class="pure-button" href="<%= URI.encode_www_form(referer) %>"> <%= translate(locale, "No") %> </a> </div> </div> - <input type="hidden" name="csrf_token" value="<%= URI.escape(csrf_token) %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(csrf_token) %>"> </form> </div> diff --git a/src/invidious/views/history.ecr b/src/invidious/views/history.ecr index e5154560..7d7ded2c 100644 --- a/src/invidious/views/history.ecr +++ b/src/invidious/views/history.ecr @@ -20,7 +20,7 @@ <script> var watched_data = { - csrf_token: '<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>', + csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>', } </script> <script src="/js/watched_widget.js"></script> @@ -35,7 +35,7 @@ var watched_data = { <div class="thumbnail"> <img class="thumbnail" src="/vi/<%= item %>/mqdefault.jpg"/> <form onsubmit="return false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post"> - <input type="hidden" name="csrf_token" value="<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <p class="watched"> <a onclick="mark_unwatched(this)" data-id="<%= item %>" href="javascript:void(0)"> <button type="submit" style="all:unset"> diff --git a/src/invidious/views/login.ecr b/src/invidious/views/login.ecr index 614978bc..8518ca81 100644 --- a/src/invidious/views/login.ecr +++ b/src/invidious/views/login.ecr @@ -22,7 +22,7 @@ <hr> <% if account_type == "invidious" %> - <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.escape(referer) %>&type=invidious" method="post"> + <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=invidious" method="post"> <fieldset> <% if email %> <input name="email" type="hidden" value="<%= email %>"> @@ -44,7 +44,7 @@ <% captcha = captcha.not_nil! %> <img style="width:100%" src='<%= captcha[:question] %>'/> <% captcha[:tokens].each_with_index do |token, i| %> - <input type="hidden" name="token[<%= i %>]" value="<%= URI.escape(token) %>"> + <input type="hidden" name="token[<%= i %>]" value="<%= URI.encode_www_form(token) %>"> <% end %> <input type="hidden" name="captcha_type" value="image"> <label for="answer"><%= translate(locale, "Time (h:mm:ss):") %></label> @@ -52,7 +52,7 @@ <% when "text" %> <% captcha = captcha.not_nil! %> <% captcha[:tokens].each_with_index do |token, i| %> - <input type="hidden" name="token[<%= i %>]" value="<%= URI.escape(token) %>"> + <input type="hidden" name="token[<%= i %>]" value="<%= URI.encode_www_form(token) %>"> <% end %> <input type="hidden" name="captcha_type" value="text"> <label for="answer"><%= captcha[:question] %></label> @@ -85,7 +85,7 @@ </fieldset> </form> <% elsif account_type == "google" %> - <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.escape(referer) %>&type=google" method="post"> + <form class="pure-form pure-form-stacked" action="/login?referer=<%= URI.encode_www_form(referer) %>&type=google" method="post"> <fieldset> <% if email %> <input name="email" type="hidden" value="<%= email %>"> diff --git a/src/invidious/views/playlists.ecr b/src/invidious/views/playlists.ecr index 400922ff..a32192b5 100644 --- a/src/invidious/views/playlists.ecr +++ b/src/invidious/views/playlists.ecr @@ -33,7 +33,7 @@ <div class="h-box"> <% ucid = channel.ucid %> <% author = channel.author %> - <% sub_count_text = channel.sub_count.format %> + <% sub_count_text = number_to_short_text(channel.sub_count) %> <%= rendered "components/subscribe_widget" %> </div> diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index 6ea01fba..56334dd9 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -9,7 +9,7 @@ function update_value(element) { </script> <div class="h-box"> - <form class="pure-form pure-form-aligned" action="/preferences?referer=<%= URI.escape(referer) %>" method="post"> + <form class="pure-form pure-form-aligned" action="/preferences?referer=<%= URI.encode_www_form(referer) %>" method="post"> <fieldset> <legend><%= translate(locale, "Player preferences") %></legend> @@ -242,15 +242,15 @@ function update_value(element) { <legend><%= translate(locale, "Data preferences") %></legend> <div class="pure-control-group"> - <a href="/clear_watch_history?referer=<%= URI.escape(referer) %>"><%= translate(locale, "Clear watch history") %></a> + <a href="/clear_watch_history?referer=<%= URI.encode_www_form(referer) %>"><%= translate(locale, "Clear watch history") %></a> </div> <div class="pure-control-group"> - <a href="/change_password?referer=<%= URI.escape(referer) %>"><%= translate(locale, "Change password") %></a> + <a href="/change_password?referer=<%= URI.encode_www_form(referer) %>"><%= translate(locale, "Change password") %></a> </div> <div class="pure-control-group"> - <a href="/data_control?referer=<%= URI.escape(referer) %>"><%= translate(locale, "Import/export data") %></a> + <a href="/data_control?referer=<%= URI.encode_www_form(referer) %>"><%= translate(locale, "Import/export data") %></a> </div> <div class="pure-control-group"> @@ -266,7 +266,7 @@ function update_value(element) { </div> <div class="pure-control-group"> - <a href="/delete_account?referer=<%= URI.escape(referer) %>"><%= translate(locale, "Delete account") %></a> + <a href="/delete_account?referer=<%= URI.encode_www_form(referer) %>"><%= translate(locale, "Delete account") %></a> </div> <% end %> diff --git a/src/invidious/views/privacy.ecr b/src/invidious/views/privacy.ecr index f273b250..643f880b 100644 --- a/src/invidious/views/privacy.ecr +++ b/src/invidious/views/privacy.ecr @@ -3,73 +3,52 @@ <% end %> <div class="h-box"> - <%= Markdown.to_html(<<-END_PRIVACY_POLICY - ## Privacy - - This document concerns what data you provide to this website, the purpose of the data, how the data is stored, and how the data can be removed. - - ### Data you directly provide - - Data that you provide to the website for the purpose of the site's operation (for example: an account name, account password, or channel subscription) will be stored in the website's database until the user decides to remove it. This data will not be intentionally shared with anyone or anything. - - Information stored about a registered user is limited to: - - - a list of session tokens for remaining logged in across devices - - the last time an account was updated (to provide accurate notifications) - - a list of video IDs identifying notifications from a user's subscriptions - - a list of channel UCIDs the user is subscribed to - - a user ID (for persistent storage of subscriptions and preferences) - - a json object containing user preferences - - a hashed password if applicable (not present on google accounts) - - a randomly generated token for providing an RSS feed of a user's subscriptions - - a list of video IDs identifying watched videos - - The above list reflects [this code](https://github.com/omarroth/invidious/blob/master/src/invidious/users.cr#L14-L51). - - Users can clear their watch history using the [clear watch history](/clear_watch_history) page. - - If a user is logged in with a Google account, no password will ever be stored. This website uses the session token provided by Google to identify a user, but does not store the information required to make requests on a user's behalf without their knowledge or consent. - - ### Data you passively provide - - When you request any resource from this website (for example: a page, a font, an image, or an API endpoint) information about the request may be logged. - - Information about a request is limited to: - - - the time the request was made - - the status code of the response - - the method of the request - - the requested URL - - how long it took to complete the request. - - No identifying information is logged, such as the visitor's cookie, user-agent, or IP address. Here are a couple lines to serve as an example: - - ``` - 2019-01-19 16:37:47 +00:00 200 GET /api/v1/comments/xrlETJYzH-c?format=html&hl=en-US 1345.88ms - 2019-01-19 16:37:53 +00:00 200 GET /vi/r5P-f5arPXE/maxres.jpg 1085.41ms - 2019-01-19 16:37:54 +00:00 200 GET /watch 7.04ms - ``` - - This website does not store the visitor's user-agent or IP address and does not use fingerprinting, advertisements, or tracking of any form. - - This website provides links to googlevideo.com to provide audio and video playback. googlevideo.com is owned by Google and is subject to their [privacy policy](https://policies.google.com/privacy). - - ### Data stored in your browser - - This website uses browser cookies to authenticate registered users. This data consists of: - - - An account token to keep you logged into the website between visits, which is sent when any page is loaded while you are logged in - - This website also provides an option to store site preferences, such as the theme or locale, without an account. Using this feature will store a cookie in the visitor's browser containing their preferences. This cookie is sent on every request and does not contain any identifying information. - - You can remove this data from your browser by logging out of this website, or by using your browser's cookie-related controls to delete the data. - - ### Removal of data - - To remove data stored in your browser, you can log out of the website, or you can use your browser's cookie-related controls to delete the data. - - To remove data that has been stored in the website's database, you can use the [delete my account](/delete_account) page. - END_PRIVACY_POLICY - ) - %> + <h2>Privacy</h2> + <p>This document concerns what data you provide to this website, the purpose of the data, how the data is stored, and how the data can be removed.</p> + + <h3>Data you directly provide</h3> + <p>Data that you provide to the website for the purpose of the site's operation (for example: an account name, account password, or channel subscription) will be stored in the website's database until the user decides to remove it. This data will not be intentionally shared with anyone or anything.</p> + <p>Information stored about a registered user is limited to:</p> + <ul> + <li>a list of session tokens for remaining logged in across devices</li> + <li>the last time an account was updated (to provide accurate notifications)</li> + <li>a list of video IDs identifying notifications from a user's subscriptions</li> + <li>a list of channel UCIDs the user is subscribed to</li> + <li>a user ID (for persistent storage of subscriptions and preferences)</li> + <li>a json object containing user preferences</li> + <li>a hashed password if applicable (not present on google accounts)</li> + <li>a randomly generated token for providing an RSS feed of a user's subscriptions</li> + <li>a list of video IDs identifying watched videos</li> + </ul> + <p>Users can clear their watch history using the <a href="/clear_watch_history">clear watch history</a> page.</p> + <p>If a user is logged in with a Google account, no password will ever be stored. This website uses the session token provided by Google to identify a user, but does not store the information required to make requests on a user's behalf without their knowledge or consent.</p> + + <h3>Data you passively provide</h3> + <p>When you request any resource from this website (for example: a page, a font, an image, or an API endpoint) information about the request may be logged.</p> + <p>Information about a request is limited to:</p> + <ul> + <li>the time the request was made</li> + <li>the status code of the response</li> + <li>the method of the request</li> + <li>the requested URL</li> + <li>how long it took to complete the request.</li> + </ul> + <p>No identifying information is logged, such as the visitor's cookie, user-agent, or IP address. Here are a couple lines to serve as an example:</p> + <pre><code>2019-01-19 16:37:47 +00:00 200 GET /api/v1/comments/xrlETJYzH-c?format=html&hl=en-US 1345.88ms +2019-01-19 16:37:53 +00:00 200 GET /vi/r5P-f5arPXE/maxres.jpg 1085.41ms +2019-01-19 16:37:54 +00:00 200 GET /watch 7.04ms</code></pre> + <p>This website does not store the visitor's user-agent or IP address and does not use fingerprinting, advertisements, or tracking of any form.</p> + <p>This website provides links to googlevideo.com to provide audio and video playback. googlevideo.com is owned by Google and is subject to their <a href="https://policies.google.com/privacy">privacy policy</a>.</p> + + <h3>Data stored in your browser</h3> + <p>This website uses browser cookies to authenticate registered users. This data consists of:</p> + <ul> + <li>An account token to keep you logged into the website between visits, which is sent when any page is loaded while you are logged in</li> + </ul> + <p>This website also provides an option to store site preferences, such as the theme or locale, without an account. Using this feature will store a cookie in the visitor's browser containing their preferences. This cookie is sent on every request and does not contain any identifying information.</p> + <p>You can remove this data from your browser by logging out of this website, or by using your browser's cookie-related controls to delete the data.</p> + + <h3>Removal of data</h3> + <p>To remove data stored in your browser, you can log out of the website, or you can use your browser's cookie-related controls to delete the data.</p> + <p>To remove data that has been stored in the website's database, you can use the <a href="/delete_account">delete my account</a> page.</p> </div> diff --git a/src/invidious/views/subscription_manager.ecr b/src/invidious/views/subscription_manager.ecr index aa6c5d81..43d14b37 100644 --- a/src/invidious/views/subscription_manager.ecr +++ b/src/invidious/views/subscription_manager.ecr @@ -19,7 +19,7 @@ </div> <div class="pure-u-1-3" style="text-align:right"> <h3> - <a href="/data_control?referer=<%= URI.escape(referer) %>"> + <a href="/data_control?referer=<%= URI.encode_www_form(referer) %>"> <%= translate(locale, "Import/export") %> </a> </h3> @@ -38,7 +38,7 @@ <div class="pure-u-1-5" style="text-align:right"> <h3 style="padding-right:0.5em"> <form onsubmit="return false" action="/subscription_ajax?action_remove_subscriptions=1&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post"> - <input type="hidden" name="csrf_token" value="<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <a onclick="remove_subscription(this)" data-ucid="<%= channel.id %>" href="#"> <input style="all:unset" type="submit" value="<%= translate(locale, "unsubscribe") %>"> </a> @@ -78,6 +78,6 @@ function remove_subscription(target) { } } - xhr.send('csrf_token=<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>'); + xhr.send('csrf_token=<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>'); } </script> diff --git a/src/invidious/views/subscriptions.ecr b/src/invidious/views/subscriptions.ecr index f98eec6f..ee31d241 100644 --- a/src/invidious/views/subscriptions.ecr +++ b/src/invidious/views/subscriptions.ecr @@ -47,7 +47,7 @@ <script> var watched_data = { - csrf_token: '<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>', + csrf_token: '<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>', } </script> <script src="/js/watched_widget.js"></script> diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 8d8cec88..cfefbc2e 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -68,7 +68,7 @@ </div> <div class="pure-u-1-4"> <form action="/signout?referer=<%= env.get?("current_page") %>" method="post"> - <input type="hidden" name="csrf_token" value="<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <a class="pure-menu-heading" href="#"> <input style="all:unset" type="submit" value="<%= translate(locale, "Log out") %>"> </a> diff --git a/src/invidious/views/token_manager.ecr b/src/invidious/views/token_manager.ecr index e1c0228f..b626d99c 100644 --- a/src/invidious/views/token_manager.ecr +++ b/src/invidious/views/token_manager.ecr @@ -11,7 +11,7 @@ <div class="pure-u-1-3"></div> <div class="pure-u-1-3" style="text-align:right"> <h3> - <a href="/preferences?referer=<%= URI.escape(referer) %>"><%= translate(locale, "Preferences") %></a> + <a href="/preferences?referer=<%= URI.encode_www_form(referer) %>"><%= translate(locale, "Preferences") %></a> </h3> </div> </div> @@ -30,7 +30,7 @@ <div class="pure-u-1-5" style="text-align:right"> <h3 style="padding-right:0.5em"> <form onsubmit="return false" action="/token_ajax?action_revoke_token=1&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post"> - <input type="hidden" name="csrf_token" value="<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>"> + <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>"> <a onclick="revoke_token(this)" data-session="<%= token[:session] %>" href="#"> <input style="all:unset" type="submit" value="<%= translate(locale, "revoke") %>"> </a> @@ -70,6 +70,6 @@ function revoke_token(target) { } } - xhr.send('csrf_token=<%= URI.escape(env.get?("csrf_token").try &.as(String) || "") %>'); + xhr.send('csrf_token=<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>'); } </script> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 58c4b1a3..6e37f7a6 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -106,22 +106,22 @@ var video_data = { <label for="download_widget"><%= translate(locale, "Download as: ") %></label> <select style="width:100%" name="download_widget" id="download_widget"> <% fmt_stream.each do |option| %> - <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.escape(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'> + <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'> <%= itag_to_metadata?(option["itag"]).try &.["height"]? || "~240" %>p - <%= option["type"].split(";")[0] %> </option> <% end %> <% video_streams.each do |option| %> - <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.escape(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'> + <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'> <%= option["quality_label"] %> - <%= option["type"].split(";")[0] %> @ <%= option["fps"] %>fps - video only </option> <% end %> <% audio_streams.each do |option| %> - <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.escape(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'> + <option value='{"id":"<%= video.id %>","itag":"<%= option["itag"] %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= option["type"].split(";")[0].split("/")[1] %>"}'> <%= option["type"].split(";")[0] %> @ <%= option["bitrate"] %>k - audio only </option> <% end %> <% captions.each do |caption| %> - <option value='{"id":"<%= video.id %>","label":"<%= caption.name.simpleText %>","title":"<%= URI.escape(video.title) %>-<%= video.id %>.<%= caption.languageCode %>.vtt"}'> + <option value='{"id":"<%= video.id %>","label":"<%= caption.name.simpleText %>","title":"<%= URI.encode_www_form(video.title) %>-<%= video.id %>.<%= caption.languageCode %>.vtt"}'> <%= translate(locale, "Subtitles - `x` (.vtt)", caption.name.simpleText) %> </option> <% end %> |