diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/invidious.cr | 40 | ||||
| -rw-r--r-- | src/invidious/channels.cr | 43 | ||||
| -rw-r--r-- | src/invidious/comments.cr | 16 | ||||
| -rw-r--r-- | src/invidious/helpers/helpers.cr | 37 | ||||
| -rw-r--r-- | src/invidious/playlists.cr | 15 | ||||
| -rw-r--r-- | src/invidious/search.cr | 11 | ||||
| -rw-r--r-- | src/invidious/users.cr | 14 | ||||
| -rw-r--r-- | src/invidious/videos.cr | 2 |
8 files changed, 118 insertions, 60 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 563a3768..89d99ecc 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -33,16 +33,7 @@ require "./invidious/jobs/**" CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) -PG_URL = URI.new( - scheme: "postgres", - user: CONFIG.db.user, - password: CONFIG.db.password, - host: CONFIG.db.host, - port: CONFIG.db.port, - path: CONFIG.db.dbname, -) - -PG_DB = DB.open PG_URL +PG_DB = DB.open CONFIG.database_url ARCHIVE_URL = URI.parse("https://archive.org") LOGIN_URL = URI.parse("https://accounts.google.com") PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") @@ -195,7 +186,7 @@ if CONFIG.captcha_key end connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32) -Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(connection_channel, PG_URL) +Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(connection_channel, CONFIG.database_url) Invidious::Jobs.start_all @@ -298,6 +289,7 @@ before_all do |env| preferences.dark_mode = dark_mode preferences.thin_mode = thin_mode preferences.locale = locale + env.set "preferences", preferences current_page = env.request.path if env.request.query @@ -760,10 +752,16 @@ post "/data_control" do |env| end end when "import_youtube" - subscriptions = JSON.parse(body) - - user.subscriptions += subscriptions.as_a.compact_map do |entry| - entry["snippet"]["resourceId"]["channelId"].as_s + if body[0..4] == "<opml" + subscriptions = XML.parse(body) + user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| + channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + end + else + subscriptions = JSON.parse(body) + user.subscriptions += subscriptions.as_a.compact_map do |entry| + entry["snippet"]["resourceId"]["channelId"].as_s + end end user.subscriptions.uniq! @@ -1557,12 +1555,12 @@ post "/feed/webhook/:token" do |env| views: video.views, }) - was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, \ + was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, + updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8, premiere_timestamp = $9, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), \ + PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert end end @@ -2560,12 +2558,12 @@ get "/api/v1/search" do |env| content_type ||= "video" begin - search_params = produce_search_params(sort_by, date, content_type, duration, features) + search_params = produce_search_params(page, sort_by, date, content_type, duration, features) rescue ex next error_json(400, ex) end - count, search_results = search(query, page, search_params, region).as(Tuple) + count, search_results = search(query, search_params, region).as(Tuple) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 9986fe1b..b9808d98 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -233,7 +233,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) videos = [] of SearchVideo begin - initial_data = JSON.parse(response.body).as_a.find &.["response"]? + initial_data = JSON.parse(response.body) raise InfoException.new("Could not extract channel JSON") if !initial_data LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data") @@ -305,7 +305,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) loop do response = get_channel_videos_response(ucid, page, auto_generated: auto_generated) - initial_data = JSON.parse(response.body).as_a.find &.["response"]? + initial_data = JSON.parse(response.body) raise InfoException.new("Could not extract channel JSON") if !initial_data videos = extract_videos(initial_data.as_h, author, ucid) @@ -388,7 +388,7 @@ def fetch_channel_playlists(ucid, author, auto_generated, continuation, sort_by) return items, continuation end -def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) +def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) object = { "80226972:embedded" => { "2:string" => ucid, @@ -444,6 +444,11 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } + return continuation +end + +def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) + continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" end @@ -932,9 +937,27 @@ def get_about_info(ucid, locale) }) end -def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest") - url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) - return YT_POOL.client &.get(url) +def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest", youtubei_browse = true) + if youtubei_browse + continuation = produce_channel_videos_continuation(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) + data = { + "context": { + "client": { + "clientName": "WEB", + "clientVersion": "2.20201021.03.00", + }, + }, + "continuation": continuation, + }.to_json + return YT_POOL.client &.post( + "/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + headers: HTTP::Headers{"content-type" => "application/json"}, + body: data + ) + else + url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated, sort_by: sort_by, v2: true) + return YT_POOL.client &.get(url) + end end def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") @@ -942,7 +965,7 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") 2.times do |i| response = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) - initial_data = JSON.parse(response.body).as_a.find &.["response"]? + initial_data = JSON.parse(response.body) break if !initial_data videos.concat extract_videos(initial_data.as_h, author, ucid) end @@ -951,10 +974,10 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") end def get_latest_videos(ucid) - response = get_channel_videos_response(ucid, 1) - initial_data = JSON.parse(response.body).as_a.find &.["response"]? + response = get_channel_videos_response(ucid) + initial_data = JSON.parse(response.body) return [] of SearchVideo if !initial_data - author = initial_data["response"]?.try &.["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s + author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s items = extract_videos(initial_data.as_h, author, ucid) return items diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 13ebbd73..20e64a08 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -488,8 +488,12 @@ def replace_links(html) length_seconds = decode_time(anchor.content) end - anchor["href"] = "javascript:void(0)" - anchor["onclick"] = "player.currentTime(#{length_seconds})" + if length_seconds > 0 + anchor["href"] = "javascript:void(0)" + anchor["onclick"] = "player.currentTime(#{length_seconds})" + else + anchor["href"] = url.request_target + end end end @@ -528,11 +532,7 @@ end def content_to_comment_html(content) comment_html = content.map do |run| - text = HTML.escape(run["text"].as_s) - - if run["text"] == "\n" - text = "<br>" - end + text = HTML.escape(run["text"].as_s).gsub("\n", "<br>") if run["bold"]? text = "<b>#{text}</b>" @@ -559,7 +559,7 @@ def content_to_comment_html(content) length_seconds = watch_endpoint["startTimeSeconds"]? video_id = watch_endpoint["videoId"].as_s - if length_seconds + if length_seconds && length_seconds.as_i > 0 text = %(<a href="javascript:void(0)" data-onclick="jump_to_time" data-jump-time="#{length_seconds}">#{text}</a>) else text = %(<a href="/watch?v=#{video_id}">#{text}</a>) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 944d869b..5d127e1a 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -64,11 +64,14 @@ end class Config include YAML::Serializable - property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) - property feed_threads : Int32 = 1 # Number of threads to use for updating feeds - property output : String = "STDOUT" # Log file path or STDOUT - property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr - property db : DBConfig # Database configuration + property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) + property feed_threads : Int32 = 1 # Number of threads to use for updating feeds + property output : String = "STDOUT" # Log file path or STDOUT + property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr + property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) + + @[YAML::Field(converter: Preferences::URIConverter)] + property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax property decrypt_polling : Bool = true # Use polling to keep decryption function up to date property full_refresh : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// @@ -170,6 +173,23 @@ class Config end {% end %} + # Build database_url from db.* if it's not set directly + if config.database_url.to_s.empty? + if db = config.db + config.database_url = URI.new( + scheme: "postgres", + user: db.user, + password: db.password, + host: db.host, + port: db.port, + path: db.dbname, + ) + else + puts "Config : Either database_url or db.* is required" + exit(1) + end + end + return config end end @@ -363,10 +383,9 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri items = [] of SearchItem channel_v2_response = initial_data - .try &.["response"]? - .try &.["continuationContents"]? - .try &.["gridContinuation"]? - .try &.["items"]? + .try &.["continuationContents"]? + .try &.["gridContinuation"]? + .try &.["items"]? if channel_v2_response channel_v2_response.try &.as_a.each { |item| diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 25797a36..0251a69c 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -101,6 +101,7 @@ struct Playlist property author_thumbnail : String property ucid : String property description : String + property description_html : String property video_count : Int32 property views : Int64 property updated : Time @@ -163,10 +164,6 @@ struct Playlist def privacy PlaylistPrivacy::Public end - - def description_html - HTML.escape(self.description).gsub("\n", "<br>") - end end enum PlaylistPrivacy @@ -375,7 +372,12 @@ def fetch_playlist(plid, locale) title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || "" desc_item = playlist_info["description"]? - description = desc_item.try &.["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || "" + + description_txt = desc_item.try &.["runs"]?.try &.as_a + .map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || "" + + description_html = desc_item.try &.["runs"]?.try &.as_a + .try { |run| content_to_comment_html(run).try &.to_s } || "<p></p>" thumbnail = playlist_info["thumbnailRenderer"]?.try &.["playlistVideoThumbnailRenderer"]? .try &.["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s @@ -415,7 +417,8 @@ def fetch_playlist(plid, locale) author: author, author_thumbnail: author_thumbnail, ucid: ucid, - description: description, + description: description_txt, + description_html: description_html, video_count: video_count, views: views, updated: updated, diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 1c4bc74e..cf8fd790 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -249,10 +249,10 @@ def channel_search(query, page, channel) return items.size, items end -def search(query, page = 1, search_params = produce_search_params(content_type: "all"), region = nil) +def search(query, search_params = produce_search_params(content_type: "all"), region = nil) return 0, [] of SearchItem if query.empty? - body = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en").body) + body = YT_POOL.client(region, &.get("/results?search_query=#{URI.encode_www_form(query)}&sp=#{search_params}&hl=en").body) return 0, [] of SearchItem if body.empty? initial_data = extract_initial_data(body) @@ -263,11 +263,12 @@ def search(query, page = 1, search_params = produce_search_params(content_type: return items.size, items end -def produce_search_params(sort : String = "relevance", date : String = "", content_type : String = "", +def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "", duration : String = "", features : Array(String) = [] of String) object = { "1:varint" => 0_i64, "2:embedded" => {} of String => Int64, + "9:varint" => ((page - 1) * 20).to_i64, } case sort @@ -439,10 +440,10 @@ def process_search_query(query, page, user, region) count = 0 end else - search_params = produce_search_params(sort: sort, date: date, content_type: content_type, + search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, duration: duration, features: features) - count, items = search(search_query, page, search_params, region).as(Tuple) + count, items = search(search_query, search_params, region).as(Tuple) end {search_query, count, items, operators} diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 153e3b6a..7a948b76 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -173,6 +173,20 @@ struct Preferences end end + module URIConverter + def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder) + yaml.scalar value.normalize! + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI + if node.is_a?(YAML::Nodes::Scalar) + URI.parse node.value + else + node.raise "Expected scalar, not #{node.class}" + end + end + end + module ProcessString def self.to_json(value : String, json : JSON::Builder) json.string value diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 95d9a80c..e6d4c764 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -764,7 +764,7 @@ struct Video end def engagement : Float64 - ((likes + dislikes) / views).round(4) + (((likes + dislikes) / views) * 100).round(4) end def reason : String? |