diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/invidious.cr | 118 | ||||
| -rw-r--r-- | src/invidious/channels.cr | 204 | ||||
| -rw-r--r-- | src/invidious/comments.cr | 69 | ||||
| -rw-r--r-- | src/invidious/helpers/helpers.cr | 414 | ||||
| -rw-r--r-- | src/invidious/helpers/jobs.cr | 2 | ||||
| -rw-r--r-- | src/invidious/helpers/macros.cr | 80 | ||||
| -rw-r--r-- | src/invidious/helpers/patch_mapping.cr | 166 | ||||
| -rw-r--r-- | src/invidious/mixes.cr | 57 | ||||
| -rw-r--r-- | src/invidious/playlists.cr | 171 | ||||
| -rw-r--r-- | src/invidious/search.cr | 80 | ||||
| -rw-r--r-- | src/invidious/users.cr | 333 | ||||
| -rw-r--r-- | src/invidious/videos.cr | 136 |
12 files changed, 847 insertions, 983 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 18c56cfc..d9db2fda 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1203,17 +1203,17 @@ post "/playlist_ajax" do |env| end end - playlist_video = PlaylistVideo.new( - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, length_seconds: video.length_seconds, - published: video.published, - plid: playlist_id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX) - ) + published: video.published, + plid: playlist_id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) video_array = playlist_video.to_a args = arg_array(video_array) @@ -1839,8 +1839,8 @@ post "/login" do |env| sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) user, sid = create_user(sid, email, password) user_array = user.to_a + user_array[4] = user_array[4].to_json # User preferences - user_array[4] = user_array[4].to_json args = arg_array(user_array) PG_DB.exec("INSERT INTO users VALUES (#{args})", args: user_array) @@ -2519,7 +2519,7 @@ post "/data_control" do |env| if user user = user.as(User) - # TODO: Find better way to prevent timeout + # TODO: Find a way to prevent browser timeout HTTP::FormData.parse(env.request) do |part| body = part.body.gets_to_end @@ -2546,7 +2546,7 @@ post "/data_control" do |env| end if body["preferences"]? - user.preferences = Preferences.from_json(body["preferences"].to_json, user.preferences) + user.preferences = Preferences.from_json(body["preferences"].to_json) PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email) end @@ -2573,17 +2573,17 @@ post "/data_control" do |env| next end - playlist_video = PlaylistVideo.new( - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, length_seconds: video.length_seconds, - published: video.published, - plid: playlist.id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX) - ) + published: video.published, + plid: playlist.id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) video_array = playlist_video.to_a args = arg_array(video_array) @@ -3154,20 +3154,20 @@ get "/feed/channel/:ucid" do |env| description_html = entry.xpath_node("group/description").not_nil!.to_s views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64 - SearchVideo.new( - title: title, - id: video_id, - author: author, - ucid: ucid, - published: published, - views: views, - description_html: description_html, - length_seconds: 0, - live_now: false, - paid: false, - premium: false, - premiere_timestamp: nil - ) + SearchVideo.new({ + title: title, + id: video_id, + author: author, + ucid: ucid, + published: published, + views: views, + description_html: description_html, + length_seconds: 0, + live_now: false, + paid: false, + premium: false, + premiere_timestamp: nil, + }) end XML.build(indent: " ", encoding: "UTF-8") do |xml| @@ -3397,18 +3397,18 @@ post "/feed/webhook/:token" do |env| }.to_json PG_DB.exec("NOTIFY notifications, E'#{payload}'") - video = ChannelVideo.new( - id: id, - title: video.title, - published: published, - updated: updated, - ucid: video.ucid, - author: author, - length_seconds: video.length_seconds, - live_now: video.live_now, + video = ChannelVideo.new({ + id: id, + title: video.title, + published: published, + updated: updated, + ucid: video.ucid, + author: author, + length_seconds: video.length_seconds, + live_now: video.live_now, premiere_timestamp: video.premiere_timestamp, - views: video.views, - ) + views: video.views, + }) PG_DB.query_all("UPDATE users SET feed_needs_update = true, notifications = array_append(notifications, $1) \ WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", @@ -4666,7 +4666,7 @@ post "/api/v1/auth/preferences" do |env| user = env.get("user").as(User) begin - preferences = Preferences.from_json(env.request.body || "{}", user.preferences) + preferences = Preferences.from_json(env.request.body || "{}") rescue preferences = user.preferences end @@ -4920,17 +4920,17 @@ post "/api/v1/auth/playlists/:plid/videos" do |env| next error_message end - playlist_video = PlaylistVideo.new( - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, length_seconds: video.length_seconds, - published: video.published, - plid: plid, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX) - ) + published: video.published, + plid: plid, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) video_array = playlist_video.to_a args = arg_array(video_array) diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index e7bcf00e..da062755 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -1,14 +1,27 @@ struct InvidiousChannel - db_mapping({ - id: String, - author: String, - updated: Time, - deleted: Bool, - subscribed: Time?, - }) + include DB::Serializable + + property id : String + property author : String + property updated : Time + property deleted : Bool + property subscribed : Time? end struct ChannelVideo + include DB::Serializable + + property id : String + property title : String + property published : Time + property updated : Time + property ucid : String + property author : String + property length_seconds : Int32 = 0 + property live_now : Bool = false + property premiere_timestamp : Time? = nil + property views : Int64? = nil + def to_json(locale, json : JSON::Builder) json.object do json.field "type", "shortVideo" @@ -84,49 +97,36 @@ struct ChannelVideo end end end - - db_mapping({ - id: String, - title: String, - published: Time, - updated: Time, - ucid: String, - author: String, - length_seconds: {type: Int32, default: 0}, - live_now: {type: Bool, default: false}, - premiere_timestamp: {type: Time?, default: nil}, - views: {type: Int64?, default: nil}, - }) end struct AboutRelatedChannel - db_mapping({ - ucid: String, - author: String, - author_url: String, - author_thumbnail: String, - }) + include DB::Serializable + + property ucid : String + property author : String + property author_url : String + property author_thumbnail : String end # TODO: Refactor into either SearchChannel or InvidiousChannel struct AboutChannel - db_mapping({ - ucid: String, - author: String, - auto_generated: Bool, - author_url: String, - author_thumbnail: String, - banner: String?, - description_html: String, - paid: Bool, - total_views: Int64, - sub_count: Int32, - joined: Time, - is_family_friendly: Bool, - allowed_regions: Array(String), - related_channels: Array(AboutRelatedChannel), - tabs: Array(String), - }) + include DB::Serializable + + property ucid : String + property author : String + property auto_generated : Bool + property author_url : String + property author_thumbnail : String + property banner : String? + property description_html : String + property paid : Bool + property total_views : Int64 + property sub_count : Int32 + property joined : Time + property is_family_friendly : Bool + property allowed_regions : Array(String) + property related_channels : Array(AboutRelatedChannel) + property tabs : Array(String) end class ChannelRedirect < Exception @@ -248,18 +248,18 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) premiere_timestamp = channel_video.try &.premiere_timestamp - video = ChannelVideo.new( - id: video_id, - title: title, - published: published, - updated: Time.utc, - ucid: ucid, - author: author, - length_seconds: length_seconds, - live_now: live_now, + video = ChannelVideo.new({ + id: video_id, + title: title, + published: published, + updated: Time.utc, + ucid: ucid, + author: author, + length_seconds: length_seconds, + live_now: live_now, premiere_timestamp: premiere_timestamp, - views: views, - ) + views: views, + }) emails = db.query_all("UPDATE users SET notifications = array_append(notifications, $1) \ WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications) RETURNING email", @@ -298,18 +298,18 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) videos = extract_videos(initial_data.as_h, author, ucid) count = videos.size - videos = videos.map { |video| ChannelVideo.new( - id: video.id, - title: video.title, - published: video.published, - updated: Time.utc, - ucid: video.ucid, - author: video.author, - length_seconds: video.length_seconds, - live_now: video.live_now, + videos = videos.map { |video| ChannelVideo.new({ + id: video.id, + title: video.title, + published: video.published, + updated: Time.utc, + ucid: video.ucid, + author: video.author, + length_seconds: video.length_seconds, + live_now: video.live_now, premiere_timestamp: video.premiere_timestamp, - views: video.views - ) } + views: video.views, + }) } videos.each do |video| ids << video.id @@ -352,7 +352,13 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid) end - channel = InvidiousChannel.new(ucid, author, Time.utc, false, nil) + channel = InvidiousChannel.new({ + id: ucid, + author: author, + updated: Time.utc, + deleted: false, + subscribed: nil, + }) return channel end @@ -395,12 +401,12 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " "80226972:embedded" => { "2:string" => ucid, "3:base64" => { - "2:string" => "videos", - "6:varint": 2_i64, - "7:varint": 1_i64, - "12:varint": 1_i64, - "13:string": "", - "23:varint": 0_i64, + "2:string" => "videos", + "6:varint" => 2_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "13:string" => "", + "23:varint" => 0_i64, }, }, } @@ -444,12 +450,12 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated "80226972:embedded" => { "2:string" => ucid, "3:base64" => { - "2:string" => "playlists", - "6:varint": 2_i64, - "7:varint": 1_i64, - "12:varint": 1_i64, - "13:string": "", - "23:varint": 0_i64, + "2:string" => "playlists", + "6:varint" => 2_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "13:string" => "", + "23:varint" => 0_i64, }, }, } @@ -849,12 +855,12 @@ def get_about_info(ucid, locale) related_author_thumbnail = node.xpath_node(%q(.//img)).try &.["data-thumb"] related_author_thumbnail ||= "" - AboutRelatedChannel.new( - ucid: related_id, - author: related_title, - author_url: related_author_url, + AboutRelatedChannel.new({ + ucid: related_id, + author: related_title, + author_url: related_author_url, author_thumbnail: related_author_thumbnail, - ) + }) end joined = about.xpath_node(%q(//span[contains(., "Joined")])) @@ -876,23 +882,23 @@ def get_about_info(ucid, locale) tabs = about.xpath_nodes(%q(//ul[@id="channel-navigation-menu"]/li/a/span)).map { |node| node.content.downcase } - AboutChannel.new( - ucid: ucid, - author: author, - auto_generated: auto_generated, - author_url: author_url, - author_thumbnail: author_thumbnail, - banner: banner, - description_html: description_html, - paid: paid, - total_views: total_views, - sub_count: sub_count, - joined: joined, + AboutChannel.new({ + ucid: ucid, + author: author, + auto_generated: auto_generated, + author_url: author_url, + author_thumbnail: author_thumbnail, + banner: banner, + description_html: description_html, + paid: paid, + total_views: total_views, + sub_count: sub_count, + joined: joined, is_family_friendly: is_family_friendly, - allowed_regions: allowed_regions, - related_channels: related_channels, - tabs: tabs - ) + allowed_regions: allowed_regions, + related_channels: related_channels, + tabs: tabs, + }) end def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index 5490d2ea..407cef78 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -1,11 +1,23 @@ class RedditThing - JSON.mapping({ - kind: String, - data: RedditComment | RedditLink | RedditMore | RedditListing, - }) + include JSON::Serializable + + property kind : String + property data : RedditComment | RedditLink | RedditMore | RedditListing end class RedditComment + include JSON::Serializable + + property author : String + property body_html : String + property replies : RedditThing | String + property score : Int32 + property depth : Int32 + property permalink : String + + @[JSON::Field(converter: RedditComment::TimeConverter)] + property created_utc : Time + module TimeConverter def self.from_json(value : JSON::PullParser) : Time Time.unix(value.read_float.to_i) @@ -15,46 +27,33 @@ class RedditComment json.number(value.to_unix) end end - - JSON.mapping({ - author: String, - body_html: String, - replies: RedditThing | String, - score: Int32, - depth: Int32, - permalink: String, - created_utc: { - type: Time, - converter: RedditComment::TimeConverter, - }, - }) end struct RedditLink - JSON.mapping({ - author: String, - score: Int32, - subreddit: String, - num_comments: Int32, - id: String, - permalink: String, - title: String, - }) + include JSON::Serializable + + property author : String + property score : Int32 + property subreddit : String + property num_comments : Int32 + property id : String + property permalink : String + property title : String end struct RedditMore - JSON.mapping({ - children: Array(String), - count: Int32, - depth: Int32, - }) + include JSON::Serializable + + property children : Array(String) + property count : Int32 + property depth : Int32 end class RedditListing - JSON.mapping({ - children: Array(RedditThing), - modhash: String, - }) + include JSON::Serializable + + property children : Array(RedditThing) + property modhash : String end def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, sort_by = "top") diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index cb4aec9b..4d697745 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -1,219 +1,100 @@ require "./macros" struct Nonce - db_mapping({ - nonce: String, - expire: Time, - }) + include DB::Serializable + + property nonce : String + property expire : Time end struct SessionId - db_mapping({ - id: String, - email: String, - issued: String, - }) + include DB::Serializable + + property id : String + property email : String + property issued : String end struct Annotation - db_mapping({ - id: String, - annotations: String, - }) + include DB::Serializable + + property id : String + property annotations : String end struct ConfigPreferences - module StringToArray - def self.to_json(value : Array(String), json : JSON::Builder) - json.array do - value.each do |element| - json.string element - end - end - end - - def self.from_json(value : JSON::PullParser) : Array(String) - begin - result = [] of String - value.read_array do - result << HTML.escape(value.read_string[0, 100]) - end - rescue ex - result = [HTML.escape(value.read_string[0, 100]), ""] - end - - result - end - - def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder) - yaml.sequence do - value.each do |element| - yaml.scalar element - end - end - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String) - begin - unless node.is_a?(YAML::Nodes::Sequence) - node.raise "Expected sequence, not #{node.class}" - end - - result = [] of String - node.nodes.each do |item| - unless item.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{item.class}" - end - - result << HTML.escape(item.value[0, 100]) - end - rescue ex - if node.is_a?(YAML::Nodes::Scalar) - result = [HTML.escape(node.value[0, 100]), ""] - else - result = ["", ""] - end - end - - result - end - end - - module BoolToString - def self.to_json(value : String, json : JSON::Builder) - json.string value - end - - def self.from_json(value : JSON::PullParser) : String - begin - result = value.read_string - - if result.empty? - CONFIG.default_user_preferences.dark_mode - else - result - end - rescue ex - if value.read_bool - "dark" - else - "light" - end - end - end - - def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) - yaml.scalar value - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String - unless node.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{node.class}" - end - - case node.value - when "true" - "dark" - when "false" - "light" - when "" - CONFIG.default_user_preferences.dark_mode - else - node.value - end - end + include YAML::Serializable + + property annotations : Bool = false + property annotations_subscribed : Bool = false + property autoplay : Bool = false + property captions : Array(String) = ["", "", ""] + property comments : Array(String) = ["youtube", ""] + property continue : Bool = false + property continue_autoplay : Bool = true + property dark_mode : String = "" + property latest_only : Bool = false + property listen : Bool = false + property local : Bool = false + property locale : String = "en-US" + property max_results : Int32 = 40 + property notifications_only : Bool = false + property player_style : String = "invidious" + property quality : String = "hd720" + property default_home : String = "Popular" + property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"] + property related_videos : Bool = true + property sort : String = "published" + property speed : Float32 = 1.0_f32 + property thin_mode : Bool = false + property unseen_only : Bool = false + property video_loop : Bool = false + property volume : Int32 = 100 + + def to_tuple + {% begin %} + { + {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}} + } + {% end %} end - - yaml_mapping({ - annotations: {type: Bool, default: false}, - annotations_subscribed: {type: Bool, default: false}, - autoplay: {type: Bool, default: false}, - captions: {type: Array(String), default: ["", "", ""], converter: StringToArray}, - comments: {type: Array(String), default: ["youtube", ""], converter: StringToArray}, - continue: {type: Bool, default: false}, - continue_autoplay: {type: Bool, default: true}, - dark_mode: {type: String, default: "", converter: BoolToString}, - latest_only: {type: Bool, default: false}, - listen: {type: Bool, default: false}, - local: {type: Bool, default: false}, - locale: {type: String, default: "en-US"}, - max_results: {type: Int32, default: 40}, - notifications_only: {type: Bool, default: false}, - player_style: {type: String, default: "invidious"}, - quality: {type: String, default: "hd720"}, - default_home: {type: String, default: "Popular"}, - feed_menu: {type: Array(String), default: ["Popular", "Trending", "Subscriptions", "Playlists"]}, - related_videos: {type: Bool, default: true}, - sort: {type: String, default: "published"}, - speed: {type: Float32, default: 1.0_f32}, - thin_mode: {type: Bool, default: false}, - unseen_only: {type: Bool, default: false}, - video_loop: {type: Bool, default: false}, - volume: {type: Int32, default: 100}, - }) end struct Config - module ConfigPreferencesConverter - def self.to_yaml(value : Preferences, yaml : YAML::Nodes::Builder) - value.to_yaml(yaml) - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Preferences - Preferences.new(*ConfigPreferences.new(ctx, node).to_tuple) - end - end - - module FamilyConverter - def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder) - case value - when Socket::Family::UNSPEC - yaml.scalar nil - when Socket::Family::INET - yaml.scalar "ipv4" - when Socket::Family::INET6 - yaml.scalar "ipv6" - when Socket::Family::UNIX - raise "Invalid socket family #{value}" - end - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family - if node.is_a?(YAML::Nodes::Scalar) - case node.value.downcase - when "ipv4" - Socket::Family::INET - when "ipv6" - Socket::Family::INET6 - else - Socket::Family::UNSPEC - end - else - node.raise "Expected scalar, not #{node.class}" - end - end - end - - module StringToCookies - def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder) - (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml) - end - - def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies - unless node.is_a?(YAML::Nodes::Scalar) - node.raise "Expected scalar, not #{node.class}" - end - - cookies = HTTP::Cookies.new - node.value.split(";").each do |cookie| - next if cookie.strip.empty? - name, value = cookie.split("=", 2) - cookies << HTTP::Cookie.new(name.strip, value.strip) - end - - cookies - end - end + include YAML::Serializable + + property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions) + property feed_threads : Int32 # Number of threads to use for updating feeds + property db : DBConfig # Database configuration + property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel + property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// + property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions + property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required + property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property captcha_enabled : Bool = true + property login_enabled : Bool = true + property registration_enabled : Bool = true + property statistics_enabled : Bool = false + property admins : Array(String) = [] of String + property external_port : Int32? = nil + property default_user_preferences : ConfigPreferences + property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs + property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc. + property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards + property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc. + property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely + property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' + + @[YAML::Field(converter: Preferences::FamilyConverter)] + property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) + property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument) + property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument) + property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) + property admin_email : String = "omarroth@protonmail.com" # Email for bug reports + + @[YAML::Field(converter: Preferences::StringToCookies)] + property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format + property captcha_key : String? = nil # Key for Anti-Captcha def disabled?(option) case disabled = CONFIG.disable_proxy @@ -229,50 +110,16 @@ struct Config return false end end - - YAML.mapping({ - channel_threads: Int32, # Number of threads to use for crawling videos from channels (for updating subscriptions) - feed_threads: Int32, # Number of threads to use for updating feeds - db: DBConfig, # Database configuration - full_refresh: Bool, # Used for crawling channels: threads should check all videos uploaded by a channel - https_only: Bool?, # Used to tell Invidious it is behind a proxy, so links to resources should be https:// - hmac_key: String?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions - domain: String?, # Domain to be used for links to resources on the site where an absolute URL is required - use_pubsub_feeds: {type: Bool | Int32, default: false}, # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) - captcha_enabled: {type: Bool, default: true}, - login_enabled: {type: Bool, default: true}, - registration_enabled: {type: Bool, default: true}, - statistics_enabled: {type: Bool, default: false}, - admins: {type: Array(String), default: [] of String}, - external_port: {type: Int32?, default: nil}, - default_user_preferences: {type: Preferences, - default: Preferences.new(*ConfigPreferences.from_yaml("").to_tuple), - converter: ConfigPreferencesConverter, - }, - dmca_content: {type: Array(String), default: [] of String}, # For compliance with DMCA, disables download widget using list of video IDs - check_tables: {type: Bool, default: false}, # Check table integrity, automatically try to add any missing columns, create tables, etc. - cache_annotations: {type: Bool, default: false}, # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards - banner: {type: String?, default: nil}, # Optional banner to be displayed along top of page for announcements, etc. - hsts: {type: Bool?, default: true}, # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely - disable_proxy: {type: Bool? | Array(String)?, default: false}, # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' - force_resolve: {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) - port: {type: Int32, default: 3000}, # Port to listen for connections (overrided by command line argument) - host_binding: {type: String, default: "0.0.0.0"}, # Host to bind (overrided by command line argument) - pool_size: {type: Int32, default: 100}, # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) - admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports - cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format - captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha - }) end struct DBConfig - yaml_mapping({ - user: String, - password: String, - host: String, - port: Int32, - dbname: String, - }) + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 + property dbname : String end def login_req(f_req) @@ -365,20 +212,20 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri end end - items << SearchVideo.new( - title: title, - id: video_id, - author: author, - ucid: author_id, - published: published, - views: view_count, - description_html: description_html, - length_seconds: length_seconds, - live_now: live_now, - paid: paid, - premium: premium, - premiere_timestamp: premiere_timestamp - ) + items << SearchVideo.new({ + title: title, + id: video_id, + author: author, + ucid: author_id, + published: published, + views: view_count, + description_html: description_html, + length_seconds: length_seconds, + live_now: live_now, + paid: paid, + premium: premium, + premiere_timestamp: premiere_timestamp, + }) elsif i = item["channelRenderer"]? author = i["title"]["simpleText"]?.try &.as_s || author_fallback || "" author_id = i["channelId"]?.try &.as_s || author_id_fallback || "" @@ -391,15 +238,15 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri video_count = i["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0 description_html = i["descriptionSnippet"]?.try { |t| parse_content(t) } || "" - items << SearchChannel.new( - author: author, - ucid: author_id, + items << SearchChannel.new({ + author: author, + ucid: author_id, author_thumbnail: author_thumbnail, subscriber_count: subscriber_count, - video_count: video_count, + video_count: video_count, description_html: description_html, - auto_generated: auto_generated, - ) + auto_generated: auto_generated, + }) elsif i = item["gridPlaylistRenderer"]? title = i["title"]["runs"].as_a[0]?.try &.["text"].as_s || "" plid = i["playlistId"]?.try &.as_s || "" @@ -407,15 +254,15 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri video_count = i["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0 playlist_thumbnail = i["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || "" - items << SearchPlaylist.new( - title: title, - id: plid, - author: author_fallback || "", - ucid: author_id_fallback || "", + items << SearchPlaylist.new({ + title: title, + id: plid, + author: author_fallback || "", + ucid: author_id_fallback || "", video_count: video_count, - videos: [] of SearchPlaylistVideo, - thumbnail: playlist_thumbnail - ) + videos: [] of SearchPlaylistVideo, + thumbnail: playlist_thumbnail, + }) elsif i = item["playlistRenderer"]? title = i["title"]["simpleText"]?.try &.as_s || "" plid = i["playlistId"]?.try &.as_s || "" @@ -432,24 +279,24 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri v_title = v["title"]["simpleText"]?.try &.as_s || "" v_id = v["videoId"]?.try &.as_s || "" v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0 - SearchPlaylistVideo.new( - title: v_title, - id: v_id, - length_seconds: v_length_seconds - ) + SearchPlaylistVideo.new({ + title: v_title, + id: v_id, + length_seconds: v_length_seconds, + }) end || [] of SearchPlaylistVideo # TODO: i["publishedTimeText"]? - items << SearchPlaylist.new( - title: title, - id: plid, - author: author, - ucid: author_id, + items << SearchPlaylist.new({ + title: title, + id: plid, + author: author, + ucid: author_id, video_count: video_count, - videos: videos, - thumbnail: playlist_thumbnail - ) + videos: videos, + thumbnail: playlist_thumbnail, + }) elsif i = item["radioRenderer"]? # Mix # TODO elsif i = item["showRenderer"]? # Show @@ -465,6 +312,7 @@ end def check_enum(db, logger, enum_name, struct_type = nil) return # TODO + if !db.query_one?("SELECT true FROM pg_type WHERE typname = $1", enum_name, as: Bool) logger.puts("CREATE TYPE #{enum_name}") @@ -488,7 +336,7 @@ def check_table(db, logger, table_name, struct_type = nil) return if !struct_type - struct_array = struct_type.to_type_tuple + struct_array = struct_type.type_array column_array = get_column_array(db, table_name) column_types = File.read("config/sql/#{table_name}.sql").match(/CREATE TABLE public\.#{table_name}\n\((?<types>[\d\D]*?)\);/) .try &.["types"].split(",").map { |line| line.strip }.reject &.starts_with?("CONSTRAINT") diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index befed471..4594c1e0 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr @@ -67,7 +67,7 @@ def refresh_feeds(db, logger, config) begin # Drop outdated views column_array = get_column_array(db, view_name) - ChannelVideo.to_type_tuple.each_with_index do |name, i| + ChannelVideo.type_array.each_with_index do |name, i| if name != column_array[i]? logger.puts("DROP MATERIALIZED VIEW #{view_name}") db.exec("DROP MATERIALIZED VIEW #{view_name}") diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr index ddfb9f8e..8b74bc86 100644 --- a/src/invidious/helpers/macros.cr +++ b/src/invidious/helpers/macros.cr @@ -1,43 +1,51 @@ -macro db_mapping(mapping) - def initialize({{*mapping.keys.map { |id| "@#{id}".id }}}) +module DB::Serializable + macro included + {% verbatim do %} + macro finished + def self.type_array + \{{ @type.instance_vars + .reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] } + .map { |name| name.stringify } + }} + end + + def initialize(tuple) + \{% for var in @type.instance_vars %} + \{% ann = var.annotation(::DB::Field) %} + \{% if ann && ann[:ignore] %} + \{% else %} + @\{{var.name}} = tuple[:\{{var.name.id}}] + \{% end %} + \{% end %} + end + + def to_a + \{{ @type.instance_vars + .reject { |var| var.annotation(::DB::Field) && var.annotation(::DB::Field)[:ignore] } + .map { |name| name } + }} + end + end + {% end %} end - - def to_a - return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ] - end - - def self.to_type_tuple - return { {{*mapping.keys.map { |id| "#{id}" }}} } - end - - DB.mapping( {{mapping}} ) -end - -macro json_mapping(mapping) - def initialize({{*mapping.keys.map { |id| "@#{id}".id }}}) - end - - def to_a - return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ] - end - - patched_json_mapping( {{mapping}} ) - YAML.mapping( {{mapping}} ) end -macro yaml_mapping(mapping) - def initialize({{*mapping.keys.map { |id| "@#{id}".id }}}) - end - - def to_a - return [ {{*mapping.keys.map { |id| "@#{id}".id }}} ] - end - - def to_tuple - return { {{*mapping.keys.map { |id| "@#{id}".id }}} } +module JSON::Serializable + macro included + {% verbatim do %} + macro finished + def initialize(tuple) + \{% for var in @type.instance_vars %} + \{% ann = var.annotation(::JSON::Field) %} + \{% if ann && ann[:ignore] %} + \{% else %} + @\{{var.name}} = tuple[:\{{var.name.id}}] + \{% end %} + \{% end %} + end + end + {% end %} end - - YAML.mapping({{mapping}}) end macro templated(filename, template = "template") diff --git a/src/invidious/helpers/patch_mapping.cr b/src/invidious/helpers/patch_mapping.cr deleted file mode 100644 index 19bd8ca1..00000000 --- a/src/invidious/helpers/patch_mapping.cr +++ /dev/null @@ -1,166 +0,0 @@ -# Overloads https://github.com/crystal-lang/crystal/blob/0.28.0/src/json/from_json.cr#L24 -def Object.from_json(string_or_io, default) : self - parser = JSON::PullParser.new(string_or_io) - new parser, default -end - -# Adds configurable 'default' -macro patched_json_mapping(_properties_, strict = false) - {% for key, value in _properties_ %} - {% _properties_[key] = {type: value} unless value.is_a?(HashLiteral) || value.is_a?(NamedTupleLiteral) %} - {% end %} - - {% for key, value in _properties_ %} - {% _properties_[key][:key_id] = key.id.gsub(/\?$/, "") %} - {% end %} - - {% for key, value in _properties_ %} - @{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }} - - {% if value[:setter] == nil ? true : value[:setter] %} - def {{value[:key_id]}}=(_{{value[:key_id]}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }}) - @{{value[:key_id]}} = _{{value[:key_id]}} - end - {% end %} - - {% if value[:getter] == nil ? true : value[:getter] %} - def {{key.id}} : {{value[:type]}}{{ (value[:nilable] ? "?" : "").id }} - @{{value[:key_id]}} - end - {% end %} - - {% if value[:presence] %} - @{{value[:key_id]}}_present : Bool = false - - def {{value[:key_id]}}_present? - @{{value[:key_id]}}_present - end - {% end %} - {% end %} - - def initialize(%pull : ::JSON::PullParser, default = nil) - {% for key, value in _properties_ %} - %var{key.id} = nil - %found{key.id} = false - {% end %} - - %location = %pull.location - begin - %pull.read_begin_object - rescue exc : ::JSON::ParseException - raise ::JSON::MappingError.new(exc.message, self.class.to_s, nil, *%location, exc) - end - until %pull.kind.end_object? - %key_location = %pull.location - key = %pull.read_object_key - case key - {% for key, value in _properties_ %} - when {{value[:key] || value[:key_id].stringify}} - %found{key.id} = true - begin - %var{key.id} = - {% if value[:nilable] || value[:default] != nil %} %pull.read_null_or { {% end %} - - {% if value[:root] %} - %pull.on_key!({{value[:root]}}) do - {% end %} - - {% if value[:converter] %} - {{value[:converter]}}.from_json(%pull) - {% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %} - {{value[:type]}}.new(%pull) - {% else %} - ::Union({{value[:type]}}).new(%pull) - {% end %} - - {% if value[:root] %} - end - {% end %} - - {% if value[:nilable] || value[:default] != nil %} } {% end %} - rescue exc : ::JSON::ParseException - raise ::JSON::MappingError.new(exc.message, self.class.to_s, {{value[:key] || value[:key_id].stringify}}, *%key_location, exc) - end - {% end %} - else - {% if strict %} - raise ::JSON::MappingError.new("Unknown JSON attribute: #{key}", self.class.to_s, nil, *%key_location, nil) - {% else %} - %pull.skip - {% end %} - end - end - %pull.read_next - - {% for key, value in _properties_ %} - {% unless value[:nilable] || value[:default] != nil %} - if %var{key.id}.nil? && !%found{key.id} && !::Union({{value[:type]}}).nilable? - raise ::JSON::MappingError.new("Missing JSON attribute: {{(value[:key] || value[:key_id]).id}}", self.class.to_s, nil, *%location, nil) - end - {% end %} - - {% if value[:nilable] %} - {% if value[:default] != nil %} - @{{value[:key_id]}} = %found{key.id} ? %var{key.id} : (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}}) - {% else %} - @{{value[:key_id]}} = %var{key.id} - {% end %} - {% elsif value[:default] != nil %} - @{{value[:key_id]}} = %var{key.id}.nil? ? (default.responds_to?(:{{value[:key_id]}}) ? default.{{value[:key_id]}} : {{value[:default]}}) : %var{key.id} - {% else %} - @{{value[:key_id]}} = (%var{key.id}).as({{value[:type]}}) - {% end %} - - {% if value[:presence] %} - @{{value[:key_id]}}_present = %found{key.id} - {% end %} - {% end %} - end - - def to_json(json : ::JSON::Builder) - json.object do - {% for key, value in _properties_ %} - _{{value[:key_id]}} = @{{value[:key_id]}} - - {% unless value[:emit_null] %} - unless _{{value[:key_id]}}.nil? - {% end %} - - json.field({{value[:key] || value[:key_id].stringify}}) do - {% if value[:root] %} - {% if value[:emit_null] %} - if _{{value[:key_id]}}.nil? - nil.to_json(json) - else - {% end %} - - json.object do - json.field({{value[:root]}}) do - {% end %} - - {% if value[:converter] %} - if _{{value[:key_id]}} - {{ value[:converter] }}.to_json(_{{value[:key_id]}}, json) - else - nil.to_json(json) - end - {% else %} - _{{value[:key_id]}}.to_json(json) - {% end %} - - {% if value[:root] %} - {% if value[:emit_null] %} - end - {% end %} - end - end - {% end %} - end - - {% unless value[:emit_null] %} - end - {% end %} - {% end %} - end - end -end diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 6c01d78b..960d994d 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -1,21 +1,21 @@ struct MixVideo - db_mapping({ - title: String, - id: String, - author: String, - ucid: String, - length_seconds: Int32, - index: Int32, - rdid: String, - }) + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property length_seconds : Int32 + property index : Int32 + property rdid : String end struct Mix - db_mapping({ - title: String, - id: String, - videos: Array(MixVideo), - }) + include DB::Serializable + + property title : String + property id : String + property videos : Array(MixVideo) end def fetch_mix(rdid, video_id, cookies = nil, locale = nil) @@ -48,23 +48,22 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) id = item["videoId"].as_s title = item["title"]?.try &.["simpleText"].as_s - if !title - next - end + next if !title + author = item["longBylineText"]["runs"][0]["text"].as_s ucid = item["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s length_seconds = decode_length_seconds(item["lengthText"]["simpleText"].as_s) index = item["navigationEndpoint"]["watchEndpoint"]["index"].as_i - videos << MixVideo.new( - title, - id, - author, - ucid, - length_seconds, - index, - rdid - ) + videos << MixVideo.new({ + title: title, + id: id, + author: author, + ucid: ucid, + length_seconds: length_seconds, + index: index, + rdid: rdid, + }) end if !cookies @@ -74,7 +73,11 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) videos.uniq! { |video| video.id } videos = videos.first(50) - return Mix.new(mix_title, rdid, videos) + return Mix.new({ + title: mix_title, + id: rdid, + videos: videos, + }) end def template_mix(mix) diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index d3064665..9190e4e6 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -1,4 +1,16 @@ struct PlaylistVideo + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property length_seconds : Int32 + property published : Time + property plid : String + property index : Int64 + property live_now : Bool + def to_xml(auto_generated, xml : XML::Builder) xml.element("entry") do xml.element("id") { xml.text "yt:video:#{self.id}" } @@ -78,21 +90,22 @@ struct PlaylistVideo end end end - - db_mapping({ - title: String, - id: String, - author: String, - ucid: String, - length_seconds: Int32, - published: Time, - plid: String, - index: Int64, - live_now: Bool, - }) end struct Playlist + include DB::Serializable + + property title : String + property id : String + property author : String + property author_thumbnail : String + property ucid : String + property description : String + property video_count : Int32 + property views : Int64 + property updated : Time + property thumbnail : String? + def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) json.object do json.field "type", "playlist" @@ -147,19 +160,6 @@ struct Playlist end end - db_mapping({ - title: String, - id: String, - author: String, - author_thumbnail: String, - ucid: String, - description: String, - video_count: Int32, - views: Int64, - updated: Time, - thumbnail: String?, - }) - def privacy PlaylistPrivacy::Public end @@ -176,6 +176,29 @@ enum PlaylistPrivacy end struct InvidiousPlaylist + include DB::Serializable + + property title : String + property id : String + property author : String + property description : String = "" + property video_count : Int32 + property created : Time + property updated : Time + + @[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)] + property privacy : PlaylistPrivacy = PlaylistPrivacy::Private + property index : Array(Int64) + + @[DB::Field(ignore: true)] + property thumbnail_id : String? + + module PlaylistPrivacyConverter + def self.from_rs(rs) + return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8)))) + end + end + def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) json.object do json.field "type", "invidiousPlaylist" @@ -216,26 +239,6 @@ struct InvidiousPlaylist end end - property thumbnail_id - - module PlaylistPrivacyConverter - def self.from_rs(rs) - return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8)))) - end - end - - db_mapping({ - title: String, - id: String, - author: String, - description: {type: String, default: ""}, - video_count: Int32, - created: Time, - updated: Time, - privacy: {type: PlaylistPrivacy, default: PlaylistPrivacy::Private, converter: PlaylistPrivacyConverter}, - index: Array(Int64), - }) - def thumbnail @thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------" "/vi/#{@thumbnail_id}/mqdefault.jpg" @@ -261,17 +264,17 @@ end def create_playlist(db, title, privacy, user) plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}" - playlist = InvidiousPlaylist.new( - title: title.byte_slice(0, 150), - id: plid, - author: user.email, + playlist = InvidiousPlaylist.new({ + title: title.byte_slice(0, 150), + id: plid, + author: user.email, description: "", # Max 5000 characters video_count: 0, - created: Time.utc, - updated: Time.utc, - privacy: privacy, - index: [] of Int64, - ) + created: Time.utc, + updated: Time.utc, + privacy: privacy, + index: [] of Int64, + }) playlist_array = playlist.to_a args = arg_array(playlist_array) @@ -282,17 +285,17 @@ def create_playlist(db, title, privacy, user) end def subscribe_playlist(db, user, playlist) - playlist = InvidiousPlaylist.new( - title: playlist.title.byte_slice(0, 150), - id: playlist.id, - author: user.email, + playlist = InvidiousPlaylist.new({ + title: playlist.title.byte_slice(0, 150), + id: playlist.id, + author: user.email, description: "", # Max 5000 characters video_count: playlist.video_count, - created: Time.utc, - updated: playlist.updated, - privacy: PlaylistPrivacy::Private, - index: [] of Int64, - ) + created: Time.utc, + updated: playlist.updated, + privacy: PlaylistPrivacy::Private, + index: [] of Int64, + }) playlist_array = playlist.to_a args = arg_array(playlist_array) @@ -393,18 +396,18 @@ def fetch_playlist(plid, locale) author = author_info["title"]["runs"][0]["text"]?.try &.as_s || "" ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || "" - return Playlist.new( - title: title, - id: plid, - author: author, + return Playlist.new({ + title: title, + id: plid, + author: author, author_thumbnail: author_thumbnail, - ucid: ucid, - description: description, - video_count: video_count, - views: views, - updated: updated, - thumbnail: thumbnail - ) + ucid: ucid, + description: description, + video_count: video_count, + views: views, + updated: updated, + thumbnail: thumbnail, + }) end def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) @@ -471,17 +474,17 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) length_seconds = 0 end - videos << PlaylistVideo.new( - title: title, - id: video_id, - author: author, - ucid: ucid, + videos << PlaylistVideo.new({ + title: title, + id: video_id, + author: author, + ucid: ucid, length_seconds: length_seconds, - published: Time.utc, - plid: plid, - live_now: live, - index: index - 1 - ) + published: Time.utc, + plid: plid, + live_now: live, + index: index - 1, + }) end end diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 92baed0b..85fd024a 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -1,4 +1,19 @@ struct SearchVideo + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property published : Time + property views : Int64 + property description_html : String + property length_seconds : Int32 + property live_now : Bool + property paid : Bool + property premium : Bool + property premiere_timestamp : Time? + def to_xml(auto_generated, query_params, xml : XML::Builder) query_params["v"] = self.id @@ -99,32 +114,27 @@ struct SearchVideo def is_upcoming premiere_timestamp ? true : false end - - db_mapping({ - title: String, - id: String, - author: String, - ucid: String, - published: Time, - views: Int64, - description_html: String, - length_seconds: Int32, - live_now: Bool, - paid: Bool, - premium: Bool, - premiere_timestamp: Time?, - }) end struct SearchPlaylistVideo - db_mapping({ - title: String, - id: String, - length_seconds: Int32, - }) + include DB::Serializable + + property title : String + property id : String + property length_seconds : Int32 end struct SearchPlaylist + include DB::Serializable + + property title : String + property id : String + property author : String + property ucid : String + property video_count : Int32 + property videos : Array(SearchPlaylistVideo) + property thumbnail : String? + def to_json(locale, json : JSON::Builder) json.object do json.field "type", "playlist" @@ -164,19 +174,19 @@ struct SearchPlaylist end end end - - db_mapping({ - title: String, - id: String, - author: String, - ucid: String, - video_count: Int32, - videos: Array(SearchPlaylistVideo), - thumbnail: String?, - }) end struct SearchChannel + include DB::Serializable + + property author : String + property ucid : String + property author_thumbnail : String + property subscriber_count : Int32 + property video_count : Int32 + property description_html : String + property auto_generated : Bool + def to_json(locale, json : JSON::Builder) json.object do json.field "type", "channel" @@ -216,16 +226,6 @@ struct SearchChannel end end end - - db_mapping({ - author: String, - ucid: String, - author_thumbnail: String, - subscriber_count: Int32, - video_count: Int32, - description_html: String, - auto_generated: Bool, - }) end alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist diff --git a/src/invidious/users.cr b/src/invidious/users.cr index f3cfafa3..46bf8865 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -4,6 +4,20 @@ require "crypto/bcrypt/password" MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" } struct User + include DB::Serializable + + property updated : Time + property notifications : Array(String) + property subscriptions : Array(String) + property email : String + + @[DB::Field(converter: User::PreferencesConverter)] + property preferences : Preferences + property password : String? + property token : String + property watched : Array(String) + property feed_needs_update : Bool? + module PreferencesConverter def self.from_rs(rs) begin @@ -13,31 +27,78 @@ struct User end end end - - db_mapping({ - updated: Time, - notifications: Array(String), - subscriptions: Array(String), - email: String, - preferences: { - type: Preferences, - converter: PreferencesConverter, - }, - password: String?, - token: String, - watched: Array(String), - feed_needs_update: Bool?, - }) end struct Preferences - module ProcessString + include JSON::Serializable + include YAML::Serializable + + property annotations : Bool = CONFIG.default_user_preferences.annotations + property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed + property autoplay : Bool = CONFIG.default_user_preferences.autoplay + + @[JSON::Field(converter: Preferences::StringToArray)] + @[YAML::Field(converter: Preferences::StringToArray)] + property captions : Array(String) = CONFIG.default_user_preferences.captions + + @[JSON::Field(converter: Preferences::StringToArray)] + @[YAML::Field(converter: Preferences::StringToArray)] + property comments : Array(String) = CONFIG.default_user_preferences.comments + property continue : Bool = CONFIG.default_user_preferences.continue + property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay + + @[JSON::Field(converter: Preferences::BoolToString)] + @[YAML::Field(converter: Preferences::BoolToString)] + property dark_mode : String = CONFIG.default_user_preferences.dark_mode + property latest_only : Bool = CONFIG.default_user_preferences.latest_only + property listen : Bool = CONFIG.default_user_preferences.listen + property local : Bool = CONFIG.default_user_preferences.local + + @[JSON::Field(converter: Preferences::ProcessString)] + property locale : String = CONFIG.default_user_preferences.locale + + @[JSON::Field(converter: Preferences::ClampInt)] + property max_results : Int32 = CONFIG.default_user_preferences.max_results + property notifications_only : Bool = CONFIG.default_user_preferences.notifications_only + + @[JSON::Field(converter: Preferences::ProcessString)] + property player_style : String = CONFIG.default_user_preferences.player_style + + @[JSON::Field(converter: Preferences::ProcessString)] + property quality : String = CONFIG.default_user_preferences.quality + property default_home : String = CONFIG.default_user_preferences.default_home + property feed_menu : Array(String) = CONFIG.default_user_preferences.feed_menu + property related_videos : Bool = CONFIG.default_user_preferences.related_videos + + @[JSON::Field(converter: Preferences::ProcessString)] + property sort : String = CONFIG.default_user_preferences.sort + property speed : Float32 = CONFIG.default_user_preferences.speed + property thin_mode : Bool = CONFIG.default_user_preferences.thin_mode + property unseen_only : Bool = CONFIG.default_user_preferences.unseen_only + property video_loop : Bool = CONFIG.default_user_preferences.video_loop + property volume : Int32 = CONFIG.default_user_preferences.volume + + module BoolToString def self.to_json(value : String, json : JSON::Builder) json.string value end def self.from_json(value : JSON::PullParser) : String - HTML.escape(value.read_string[0, 100]) + begin + result = value.read_string + + if result.empty? + CONFIG.default_user_preferences.dark_mode + else + result + end + rescue ex + if value.read_bool + "dark" + else + "light" + end + end end def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) @@ -45,7 +106,20 @@ struct Preferences end def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String - HTML.escape(node.value[0, 100]) + unless node.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{node.class}" + end + + case node.value + when "true" + "dark" + when "false" + "light" + when "" + CONFIG.default_user_preferences.dark_mode + else + node.value + end end end @@ -67,33 +141,130 @@ struct Preferences end end - json_mapping({ - annotations: {type: Bool, default: CONFIG.default_user_preferences.annotations}, - annotations_subscribed: {type: Bool, default: CONFIG.default_user_preferences.annotations_subscribed}, - autoplay: {type: Bool, default: CONFIG.default_user_preferences.autoplay}, - captions: {type: Array(String), default: CONFIG.default_user_preferences.captions, converter: ConfigPreferences::StringToArray}, - comments: {type: Array(String), default: CONFIG.default_user_preferences.comments, converter: ConfigPreferences::StringToArray}, - continue: {type: Bool, default: CONFIG.default_user_preferences.continue}, - continue_autoplay: {type: Bool, default: CONFIG.default_user_preferences.continue_autoplay}, - dark_mode: {type: String, default: CONFIG.default_user_preferences.dark_mode, converter: ConfigPreferences::BoolToString}, - latest_only: {type: Bool, default: CONFIG.default_user_preferences.latest_only}, - listen: {type: Bool, default: CONFIG.default_user_preferences.listen}, - local: {type: Bool, default: CONFIG.default_user_preferences.local}, - locale: {type: String, default: CONFIG.default_user_preferences.locale, converter: ProcessString}, - max_results: {type: Int32, default: CONFIG.default_user_preferences.max_results, converter: ClampInt}, - notifications_only: {type: Bool, default: CONFIG.default_user_preferences.notifications_only}, - player_style: {type: String, default: CONFIG.default_user_preferences.player_style, converter: ProcessString}, - quality: {type: String, default: CONFIG.default_user_preferences.quality, converter: ProcessString}, - default_home: {type: String, default: CONFIG.default_user_preferences.default_home}, - feed_menu: {type: Array(String), default: CONFIG.default_user_preferences.feed_menu}, - related_videos: {type: Bool, default: CONFIG.default_user_preferences.related_videos}, - sort: {type: String, default: CONFIG.default_user_preferences.sort, converter: ProcessString}, - speed: {type: Float32, default: CONFIG.default_user_preferences.speed}, - thin_mode: {type: Bool, default: CONFIG.default_user_preferences.thin_mode}, - unseen_only: {type: Bool, default: CONFIG.default_user_preferences.unseen_only}, - video_loop: {type: Bool, default: CONFIG.default_user_preferences.video_loop}, - volume: {type: Int32, default: CONFIG.default_user_preferences.volume}, - }) + module FamilyConverter + def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder) + case value + when Socket::Family::UNSPEC + yaml.scalar nil + when Socket::Family::INET + yaml.scalar "ipv4" + when Socket::Family::INET6 + yaml.scalar "ipv6" + when Socket::Family::UNIX + raise "Invalid socket family #{value}" + end + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family + if node.is_a?(YAML::Nodes::Scalar) + case node.value.downcase + when "ipv4" + Socket::Family::INET + when "ipv6" + Socket::Family::INET6 + else + Socket::Family::UNSPEC + end + else + node.raise "Expected scalar, not #{node.class}" + end + end + end + + module ProcessString + def self.to_json(value : String, json : JSON::Builder) + json.string value + end + + def self.from_json(value : JSON::PullParser) : String + HTML.escape(value.read_string[0, 100]) + end + + def self.to_yaml(value : String, yaml : YAML::Nodes::Builder) + yaml.scalar value + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String + HTML.escape(node.value[0, 100]) + end + end + + module StringToArray + def self.to_json(value : Array(String), json : JSON::Builder) + json.array do + value.each do |element| + json.string element + end + end + end + + def self.from_json(value : JSON::PullParser) : Array(String) + begin + result = [] of String + value.read_array do + result << HTML.escape(value.read_string[0, 100]) + end + rescue ex + result = [HTML.escape(value.read_string[0, 100]), ""] + end + + result + end + + def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder) + yaml.sequence do + value.each do |element| + yaml.scalar element + end + end + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String) + begin + unless node.is_a?(YAML::Nodes::Sequence) + node.raise "Expected sequence, not #{node.class}" + end + + result = [] of String + node.nodes.each do |item| + unless item.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{item.class}" + end + + result << HTML.escape(item.value[0, 100]) + end + rescue ex + if node.is_a?(YAML::Nodes::Scalar) + result = [HTML.escape(node.value[0, 100]), ""] + else + result = ["", ""] + end + end + + result + end + end + + module StringToCookies + def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder) + (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml) + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies + unless node.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{node.class}" + end + + cookies = HTTP::Cookies.new + node.value.split(";").each do |cookie| + next if cookie.strip.empty? + name, value = cookie.split("=", 2) + cookies << HTTP::Cookie.new(name.strip, value.strip) + end + + cookies + end + end end def get_user(sid, headers, db, refresh = true) @@ -103,8 +274,7 @@ def get_user(sid, headers, db, refresh = true) if refresh && Time.utc - user.updated > 1.minute user, sid = fetch_user(sid, headers, db) user_array = user.to_a - - user_array[4] = user_array[4].to_json + user_array[4] = user_array[4].to_json # User preferences args = arg_array(user_array) db.exec("INSERT INTO users VALUES (#{args}) \ @@ -122,8 +292,7 @@ def get_user(sid, headers, db, refresh = true) else user, sid = fetch_user(sid, headers, db) user_array = user.to_a - - user_array[4] = user_array[4].to_json + user_array[4] = user_array[4].to_json # User preferences args = arg_array(user.to_a) db.exec("INSERT INTO users VALUES (#{args}) \ @@ -166,7 +335,17 @@ def fetch_user(sid, headers, db) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new(Time.utc, [] of String, channels, email, CONFIG.default_user_preferences, nil, token, [] of String, true) + user = User.new({ + updated: Time.utc, + notifications: [] of String, + subscriptions: channels, + email: email, + preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple), + password: nil, + token: token, + watched: [] of String, + feed_needs_update: true, + }) return user, sid end @@ -174,7 +353,17 @@ def create_user(sid, email, password) password = Crypto::Bcrypt::Password.create(password, cost: 10) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new(Time.utc, [] of String, [] of String, email, CONFIG.default_user_preferences, password.to_s, token, [] of String, true) + user = User.new({ + updated: Time.utc, + notifications: [] of String, + subscriptions: [] of String, + email: email, + preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple), + password: password.to_s, + token: token, + watched: [] of String, + feed_needs_update: true, + }) return user, sid end @@ -281,48 +470,6 @@ def subscribe_ajax(channel_id, action, env_headers) end end -# TODO: Playlist stub, sync with YouTube for Google accounts -# def playlist_ajax(video_ids, source_playlist_id, name, privacy, action, env_headers) -# headers = HTTP::Headers.new -# headers["Cookie"] = env_headers["Cookie"] -# -# html = YT_POOL.client &.get("/view_all_playlists", headers) -# -# cookies = HTTP::Cookies.from_headers(headers) -# html.cookies.each do |cookie| -# if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name -# if cookies[cookie.name]? -# cookies[cookie.name] = cookie -# else -# cookies << cookie -# end -# end -# end -# headers = cookies.add_request_headers(headers) -# -# if match = html.body.match(/'XSRF_TOKEN': "(?<session_token>[^"]+)"/) -# session_token = match["session_token"] -# -# headers["content-type"] = "application/x-www-form-urlencoded" -# -# post_req = { -# video_ids: [] of String, -# source_playlist_id: "", -# n: name, -# p: privacy, -# session_token: session_token, -# } -# post_url = "/playlist_ajax?#{action}=1" -# -# response = client.post(post_url, headers, form: post_req) -# if response.status_code == 200 -# return JSON.parse(response.body)["result"]["playlistId"].as_s -# else -# return nil -# end -# end -# end - def get_subscription_feed(db, user, max_results = 40, page = 1) limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE) offset = (page - 1) * limit diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index dea03163..e7751fb0 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -222,30 +222,50 @@ VIDEO_FORMATS = { } struct VideoPreferences - json_mapping({ - annotations: Bool, - autoplay: Bool, - comments: Array(String), - continue: Bool, - continue_autoplay: Bool, - controls: Bool, - listen: Bool, - local: Bool, - preferred_captions: Array(String), - player_style: String, - quality: String, - raw: Bool, - region: String?, - related_videos: Bool, - speed: (Float32 | Float64), - video_end: (Float64 | Int32), - video_loop: Bool, - video_start: (Float64 | Int32), - volume: Int32, - }) + include JSON::Serializable + + property annotations : Bool + property autoplay : Bool + property comments : Array(String) + property continue : Bool + property continue_autoplay : Bool + property controls : Bool + property listen : Bool + property local : Bool + property preferred_captions : Array(String) + property player_style : String + property quality : String + property raw : Bool + property region : String? + property related_videos : Bool + property speed : Float32 | Float64 + property video_end : Float64 | Int32 + property video_loop : Bool + property video_start : Float64 | Int32 + property volume : Int32 end struct Video + include DB::Serializable + + property id : String + + @[DB::Field(converter: Video::JSONConverter)] + property info : Hash(String, JSON::Any) + property updated : Time + + @[DB::Field(ignore: true)] + property captions : Array(Caption)? + + @[DB::Field(ignore: true)] + property adaptive_fmts : Array(Hash(String, JSON::Any))? + + @[DB::Field(ignore: true)] + property fmt_stream : Array(Hash(String, JSON::Any))? + + @[DB::Field(ignore: true)] + property description : String? + module JSONConverter def self.from_rs(rs) JSON.parse(rs.read(String)).as_h @@ -552,6 +572,7 @@ struct Video def fmt_stream return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream + fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) fmt_stream.each do |fmt| if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } @@ -751,30 +772,20 @@ struct Video def session_token : String? info["sessionToken"]?.try &.as_s? end +end - db_mapping({ - id: String, - info: {type: Hash(String, JSON::Any), converter: Video::JSONConverter}, - updated: Time, - }) +struct CaptionName + include JSON::Serializable - @captions : Array(Caption)? - @adaptive_fmts : Array(Hash(String, JSON::Any))? - @fmt_stream : Array(Hash(String, JSON::Any))? + property simpleText : String end struct Caption - json_mapping({ - name: CaptionName, - baseUrl: String, - languageCode: String, - }) -end + include JSON::Serializable -struct CaptionName - json_mapping({ - simpleText: String, - }) + property name : CaptionName + property baseUrl : String + property languageCode : String end class VideoRedirect < Exception @@ -990,7 +1001,12 @@ def fetch_video(id, region) raise info["reason"]?.try &.as_s || "" if !info["videoDetails"]? - video = Video.new(id, info, Time.utc) + video = Video.new({ + id: id, + info: info, + updated: Time.utc, + }) + return video end @@ -1097,27 +1113,27 @@ def process_video_params(query, preferences) controls ||= 1 controls = controls >= 1 - params = VideoPreferences.new( - annotations: annotations, - autoplay: autoplay, - comments: comments, - continue: continue, - continue_autoplay: continue_autoplay, - controls: controls, - listen: listen, - local: local, - player_style: player_style, + params = VideoPreferences.new({ + annotations: annotations, + autoplay: autoplay, + comments: comments, + continue: continue, + continue_autoplay: continue_autoplay, + controls: controls, + listen: listen, + local: local, + player_style: player_style, preferred_captions: preferred_captions, - quality: quality, - raw: raw, - region: region, - related_videos: related_videos, - speed: speed, - video_end: video_end, - video_loop: video_loop, - video_start: video_start, - volume: volume, - ) + quality: quality, + raw: raw, + region: region, + related_videos: related_videos, + speed: speed, + video_end: video_end, + video_loop: video_loop, + video_start: video_start, + volume: volume, + }) return params end |