diff options
| -rw-r--r-- | config/config.example.yml | 2 | ||||
| -rw-r--r-- | docker/Dockerfile | 3 | ||||
| -rw-r--r-- | locales/nb-NO.json | 8 | ||||
| -rw-r--r-- | locales/si.json | 414 | ||||
| -rw-r--r-- | shard.lock | 2 | ||||
| -rw-r--r-- | shard.yml | 2 | ||||
| -rw-r--r-- | spec/helpers_spec.cr | 10 | ||||
| -rw-r--r-- | src/invidious.cr | 17 | ||||
| -rw-r--r-- | src/invidious/helpers/helpers.cr | 30 | ||||
| -rw-r--r-- | src/invidious/search.cr | 11 | ||||
| -rw-r--r-- | src/invidious/users.cr | 14 | ||||
| -rw-r--r-- | src/invidious/videos.cr | 2 |
12 files changed, 478 insertions, 37 deletions
diff --git a/config/config.example.yml b/config/config.example.yml index e83a7515..e8330705 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -6,6 +6,8 @@ db: host: localhost port: 5432 dbname: invidious +# alternatively, the database URL can be provided directly - if both are set then the latter takes precedence +# database_url: postgres://kemal:kemal@localhost:5432/invidious full_refresh: false https_only: false domain: diff --git a/docker/Dockerfile b/docker/Dockerfile index b88a76c0..591c5aa1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,8 +3,7 @@ RUN apk add --no-cache curl sqlite-static WORKDIR /invidious COPY ./shard.yml ./shard.yml COPY ./shard.lock ./shard.lock -RUN shards install && \ - curl -Lo ./lib/lsquic/src/lsquic/ext/liblsquic.a https://github.com/iv-org/lsquic-static-alpine/releases/download/v2.18.1/liblsquic.a +RUN shards install COPY ./src/ ./src/ # TODO: .git folder is required for building – this is destructive. # See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION. diff --git a/locales/nb-NO.json b/locales/nb-NO.json index 6bf5107b..355a1c33 100644 --- a/locales/nb-NO.json +++ b/locales/nb-NO.json @@ -77,8 +77,8 @@ "Thin mode: ": "Tynt modus: ", "Subscription preferences": "Abonnementsinnstillinger", "Show annotations by default for subscribed channels: ": "Vis merknader som forvalg for kanaler det abonneres på? ", - "Redirect homepage to feed: ": "Videresend hjemmeside til flyt: ", - "Number of videos shown in feed: ": "Antall videoer å vise i flyt: ", + "Redirect homepage to feed: ": "Videresend hjemmeside til kilde: ", + "Number of videos shown in feed: ": "Antall videoer å vise i kilde: ", "Sort videos by: ": "Sorter videoer etter: ", "published": "publisert", "published - reverse": "publisert - motsatt", @@ -103,7 +103,7 @@ "Delete account": "Slett konto", "Administrator preferences": "Administratorinnstillinger", "Default homepage: ": "Forvalgt hjemmeside: ", - "Feed menu: ": "Flyt-meny: ", + "Feed menu: ": "Kilde-meny: ", "Top enabled: ": "Topp påskrudd? ", "CAPTCHA enabled: ": "CAPTCHA påskrudd? ", "Login enabled: ": "Innlogging påskrudd? ", @@ -174,7 +174,7 @@ "Password cannot be empty": "Passordet kan ikke være tomt", "Password cannot be longer than 55 characters": "Passordet kan ikke være lengre enn 55 tegn", "Please log in": "Logg inn", - "Invidious Private Feed for `x`": "Invidious personlige flyt for `x`", + "Invidious Private Feed for `x`": "Invidious personlig kilde for `x`", "channel:`x`": "kanal `x`", "Deleted or invalid channel": "Slettet eller ugyldig kanal", "This channel does not exist.": "Denne kanalen finnes ikke.", diff --git a/locales/si.json b/locales/si.json new file mode 100644 index 00000000..0dabe03a --- /dev/null +++ b/locales/si.json @@ -0,0 +1,414 @@ +{ + "`x` subscribers": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "`x` videos": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "`x` playlists": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "LIVE": "සජීව", + "Shared `x` ago": "", + "Unsubscribe": "", + "Subscribe": "", + "View channel on YouTube": "", + "View playlist on YouTube": "", + "newest": "", + "oldest": "", + "popular": "ජනප්රිය", + "last": "", + "Next page": "", + "Previous page": "", + "Clear watch history?": "", + "New password": "", + "New passwords must match": "", + "Cannot change password for Google accounts": "", + "Authorize token?": "", + "Authorize token for `x`?": "", + "Yes": "", + "No": "", + "Import and Export Data": "", + "Import": "", + "Import Invidious data": "", + "Import YouTube subscriptions": "", + "Import FreeTube subscriptions (.db)": "", + "Import NewPipe subscriptions (.json)": "", + "Import NewPipe data (.zip)": "", + "Export": "", + "Export subscriptions as OPML": "", + "Export subscriptions as OPML (for NewPipe & FreeTube)": "", + "Export data as JSON": "", + "Delete account?": "", + "History": "", + "An alternative front-end to YouTube": "", + "JavaScript license information": "", + "source": "", + "Log in": "", + "Log in/register": "", + "Log in with Google": "", + "User ID": "", + "Password": "", + "Time (h:mm:ss):": "", + "Text CAPTCHA": "", + "Image CAPTCHA": "", + "Sign In": "", + "Register": "", + "E-mail": "", + "Google verification code": "", + "Preferences": "", + "Player preferences": "", + "Always loop: ": "", + "Autoplay: ": "", + "Play next by default: ": "", + "Autoplay next video: ": "", + "Listen by default: ": "", + "Proxy videos: ": "", + "Default speed: ": "", + "Preferred video quality: ": "", + "Player volume: ": "", + "Default comments: ": "", + "youtube": "", + "reddit": "", + "Default captions: ": "", + "Fallback captions: ": "", + "Show related videos: ": "", + "Show annotations by default: ": "", + "Visual preferences": "", + "Player style: ": "", + "Dark mode: ": "", + "Theme: ": "", + "dark": "", + "light": "", + "Thin mode: ": "", + "Subscription preferences": "", + "Show annotations by default for subscribed channels: ": "", + "Redirect homepage to feed: ": "", + "Number of videos shown in feed: ": "", + "Sort videos by: ": "", + "published": "", + "published - reverse": "", + "alphabetically": "", + "alphabetically - reverse": "", + "channel name": "", + "channel name - reverse": "", + "Only show latest video from channel: ": "", + "Only show latest unwatched video from channel: ": "", + "Only show unwatched: ": "", + "Only show notifications (if there are any): ": "", + "Enable web notifications": "", + "`x` uploaded a video": "", + "`x` is live": "", + "Data preferences": "", + "Clear watch history": "", + "Import/export data": "", + "Change password": "", + "Manage subscriptions": "", + "Manage tokens": "", + "Watch history": "", + "Delete account": "", + "Administrator preferences": "", + "Default homepage: ": "", + "Feed menu: ": "", + "Top enabled: ": "", + "CAPTCHA enabled: ": "", + "Login enabled: ": "", + "Registration enabled: ": "", + "Report statistics: ": "", + "Save preferences": "", + "Subscription manager": "", + "Token manager": "", + "Token": "", + "`x` subscriptions": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "`x` tokens": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "Import/export": "", + "unsubscribe": "", + "revoke": "", + "Subscriptions": "", + "`x` unseen notifications": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "search": "", + "Log out": "", + "Released under the AGPLv3 by Omar Roth.": "", + "Source available here.": "", + "View JavaScript license information.": "", + "View privacy policy.": "", + "Trending": "", + "Public": "", + "Unlisted": "", + "Private": "", + "View all playlists": "", + "Updated `x` ago": "", + "Delete playlist `x`?": "", + "Delete playlist": "", + "Create playlist": "", + "Title": "", + "Playlist privacy": "", + "Editing playlist `x`": "", + "Watch on YouTube": "", + "Hide annotations": "", + "Show annotations": "", + "Genre: ": "", + "License: ": "", + "Family friendly? ": "", + "Wilson score: ": "", + "Engagement: ": "", + "Whitelisted regions: ": "", + "Blacklisted regions: ": "", + "Shared `x`": "", + "`x` views": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "Premieres in `x`": "", + "Premieres `x`": "", + "Hi! Looks like you have JavaScript turned off. Click here to view comments, keep in mind they may take a bit longer to load.": "", + "View YouTube comments": "", + "View more comments on Reddit": "", + "View `x` comments": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "View Reddit comments": "", + "Hide replies": "", + "Show replies": "", + "Incorrect password": "", + "Quota exceeded, try again in a few hours": "", + "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.": "", + "Invalid TFA code": "", + "Login failed. This may be because two-factor authentication is not turned on for your account.": "", + "Wrong answer": "", + "Erroneous CAPTCHA": "", + "CAPTCHA is a required field": "", + "User ID is a required field": "", + "Password is a required field": "", + "Wrong username or password": "", + "Please sign in using 'Log in with Google'": "", + "Password cannot be empty": "", + "Password cannot be longer than 55 characters": "", + "Please log in": "", + "Invidious Private Feed for `x`": "", + "channel:`x`": "", + "Deleted or invalid channel": "", + "This channel does not exist.": "", + "Could not get channel info.": "", + "Could not fetch comments": "", + "View `x` replies": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "`x` ago": "", + "Load more": "", + "`x` points": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "Could not create mix.": "", + "Empty playlist": "", + "Not a playlist.": "", + "Playlist does not exist.": "", + "Could not pull trending pages.": "", + "Hidden field \"challenge\" is a required field": "", + "Hidden field \"token\" is a required field": "", + "Erroneous challenge": "", + "Erroneous token": "", + "No such user": "", + "Token is expired, please try again": "", + "English": "", + "English (auto-generated)": "", + "Afrikaans": "", + "Albanian": "", + "Amharic": "", + "Arabic": "", + "Armenian": "", + "Azerbaijani": "", + "Bangla": "", + "Basque": "", + "Belarusian": "", + "Bosnian": "", + "Bulgarian": "", + "Burmese": "", + "Catalan": "", + "Cebuano": "", + "Chinese (Simplified)": "", + "Chinese (Traditional)": "", + "Corsican": "", + "Croatian": "", + "Czech": "", + "Danish": "", + "Dutch": "", + "Esperanto": "", + "Estonian": "", + "Filipino": "", + "Finnish": "", + "French": "", + "Galician": "", + "Georgian": "", + "German": "", + "Greek": "", + "Gujarati": "", + "Haitian Creole": "", + "Hausa": "", + "Hawaiian": "", + "Hebrew": "", + "Hindi": "", + "Hmong": "", + "Hungarian": "", + "Icelandic": "", + "Igbo": "", + "Indonesian": "", + "Irish": "", + "Italian": "", + "Japanese": "", + "Javanese": "", + "Kannada": "", + "Kazakh": "", + "Khmer": "", + "Korean": "", + "Kurdish": "", + "Kyrgyz": "", + "Lao": "", + "Latin": "", + "Latvian": "", + "Lithuanian": "", + "Luxembourgish": "", + "Macedonian": "", + "Malagasy": "", + "Malay": "", + "Malayalam": "", + "Maltese": "", + "Maori": "", + "Marathi": "", + "Mongolian": "", + "Nepali": "", + "Norwegian Bokmål": "", + "Nyanja": "", + "Pashto": "", + "Persian": "", + "Polish": "", + "Portuguese": "", + "Punjabi": "", + "Romanian": "", + "Russian": "", + "Samoan": "", + "Scottish Gaelic": "", + "Serbian": "", + "Shona": "", + "Sindhi": "", + "Sinhala": "", + "Slovak": "", + "Slovenian": "", + "Somali": "", + "Southern Sotho": "", + "Spanish": "", + "Spanish (Latin America)": "", + "Sundanese": "", + "Swahili": "", + "Swedish": "", + "Tajik": "", + "Tamil": "", + "Telugu": "", + "Thai": "", + "Turkish": "", + "Ukrainian": "", + "Urdu": "", + "Uzbek": "", + "Vietnamese": "", + "Welsh": "", + "Western Frisian": "", + "Xhosa": "", + "Yiddish": "", + "Yoruba": "", + "Zulu": "", + "`x` years": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "`x` months": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "`x` weeks": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "`x` days": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "`x` hours": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "`x` minutes": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "`x` seconds": { + "([^.,0-9]|^)1([^.,0-9]|$)": "", + "": "" + }, + "Fallback comments: ": "", + "Popular": "", + "Top": "", + "About": "", + "Rating: ": "", + "Language: ": "", + "View as playlist": "", + "Default": "", + "Music": "", + "Gaming": "", + "News": "", + "Movies": "", + "Download": "", + "Download as: ": "", + "%A %B %-d, %Y": "", + "(edited)": "", + "YouTube comment permalink": "", + "permalink": "", + "`x` marked it with a ❤": "", + "Audio mode": "", + "Video mode": "", + "Videos": "", + "Playlists": "", + "Community": "", + "relevance": "", + "rating": "", + "date": "", + "views": "", + "content_type": "", + "duration": "", + "features": "", + "sort": "", + "hour": "", + "today": "", + "week": "", + "month": "", + "year": "", + "video": "", + "channel": "", + "playlist": "", + "movie": "", + "show": "", + "hd": "", + "subtitles": "", + "creative_commons": "", + "3d": "", + "live": "", + "4k": "", + "location": "", + "hdr": "", + "filter": "", + "Current version: ": "" +} @@ -18,7 +18,7 @@ shards: lsquic: git: https://github.com/iv-org/lsquic.cr.git - version: 2.18.1-1 + version: 2.23.1 pg: git: https://github.com/will/crystal-pg.git @@ -27,7 +27,7 @@ dependencies: version: ~> 0.1.3 lsquic: github: iv-org/lsquic.cr - version: ~> 2.18.1-1 + version: ~> 2.23.1 crystal: 0.36.1 diff --git a/spec/helpers_spec.cr b/spec/helpers_spec.cr index d297759e..073d2700 100644 --- a/spec/helpers_spec.cr +++ b/spec/helpers_spec.cr @@ -65,15 +65,15 @@ describe "Helper" do describe "#produce_search_params" do it "correctly produces token for searching with specified filters" do - produce_search_params.should eq("CAASAhAB") + produce_search_params.should eq("CAASAhABSAA%3D") - produce_search_params(sort: "upload_date", content_type: "video").should eq("CAISAhAB") + produce_search_params(sort: "upload_date", content_type: "video").should eq("CAISAhABSAA%3D") - produce_search_params(content_type: "playlist").should eq("CAASAhAD") + produce_search_params(content_type: "playlist").should eq("CAASAhADSAA%3D") - produce_search_params(sort: "date", content_type: "video", features: ["hd", "cc", "purchased", "hdr"]).should eq("CAISCxABIAEwAUgByAEB") + produce_search_params(sort: "date", content_type: "video", features: ["hd", "cc", "purchased", "hdr"]).should eq("CAISCxABIAEwAUgByAEBSAA%3D") - produce_search_params(content_type: "channel").should eq("CAASAhAC") + produce_search_params(content_type: "channel").should eq("CAASAhACSAA%3D") end end diff --git a/src/invidious.cr b/src/invidious.cr index 80f551e9..6d0238a1 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -33,16 +33,7 @@ require "./invidious/jobs/**" CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) -PG_URL = URI.new( - scheme: "postgres", - user: CONFIG.db.user, - password: CONFIG.db.password, - host: CONFIG.db.host, - port: CONFIG.db.port, - path: CONFIG.db.dbname, -) - -PG_DB = DB.open PG_URL +PG_DB = DB.open CONFIG.database_url ARCHIVE_URL = URI.parse("https://archive.org") LOGIN_URL = URI.parse("https://accounts.google.com") PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") @@ -195,7 +186,7 @@ if CONFIG.captcha_key end connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32) -Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(connection_channel, PG_URL) +Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(connection_channel, CONFIG.database_url) Invidious::Jobs.start_all @@ -2566,12 +2557,12 @@ get "/api/v1/search" do |env| content_type ||= "video" begin - search_params = produce_search_params(sort_by, date, content_type, duration, features) + search_params = produce_search_params(page, sort_by, date, content_type, duration, features) rescue ex next error_json(400, ex) end - count, search_results = search(query, page, search_params, region).as(Tuple) + count, search_results = search(query, search_params, region).as(Tuple) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 944d869b..6452b68c 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -64,11 +64,14 @@ end class Config include YAML::Serializable - property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) - property feed_threads : Int32 = 1 # Number of threads to use for updating feeds - property output : String = "STDOUT" # Log file path or STDOUT - property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr - property db : DBConfig # Database configuration + property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) + property feed_threads : Int32 = 1 # Number of threads to use for updating feeds + property output : String = "STDOUT" # Log file path or STDOUT + property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr + property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) + + @[YAML::Field(converter: Preferences::URIConverter)] + property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax property decrypt_polling : Bool = true # Use polling to keep decryption function up to date property full_refresh : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// @@ -170,6 +173,23 @@ class Config end {% end %} + # Build database_url from db.* if it's not set directly + if config.database_url.to_s.empty? + if db = config.db + config.database_url = URI.new( + scheme: "postgres", + user: db.user, + password: db.password, + host: db.host, + port: db.port, + path: db.dbname, + ) + else + puts "Config : Either database_url or db.* is required" + exit(1) + end + end + return config end end diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 1c4bc74e..cf8fd790 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -249,10 +249,10 @@ def channel_search(query, page, channel) return items.size, items end -def search(query, page = 1, search_params = produce_search_params(content_type: "all"), region = nil) +def search(query, search_params = produce_search_params(content_type: "all"), region = nil) return 0, [] of SearchItem if query.empty? - body = YT_POOL.client(region, &.get("/results?q=#{URI.encode_www_form(query)}&page=#{page}&sp=#{search_params}&hl=en").body) + body = YT_POOL.client(region, &.get("/results?search_query=#{URI.encode_www_form(query)}&sp=#{search_params}&hl=en").body) return 0, [] of SearchItem if body.empty? initial_data = extract_initial_data(body) @@ -263,11 +263,12 @@ def search(query, page = 1, search_params = produce_search_params(content_type: return items.size, items end -def produce_search_params(sort : String = "relevance", date : String = "", content_type : String = "", +def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "", duration : String = "", features : Array(String) = [] of String) object = { "1:varint" => 0_i64, "2:embedded" => {} of String => Int64, + "9:varint" => ((page - 1) * 20).to_i64, } case sort @@ -439,10 +440,10 @@ def process_search_query(query, page, user, region) count = 0 end else - search_params = produce_search_params(sort: sort, date: date, content_type: content_type, + search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, duration: duration, features: features) - count, items = search(search_query, page, search_params, region).as(Tuple) + count, items = search(search_query, search_params, region).as(Tuple) end {search_query, count, items, operators} diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 153e3b6a..7a948b76 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -173,6 +173,20 @@ struct Preferences end end + module URIConverter + def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder) + yaml.scalar value.normalize! + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI + if node.is_a?(YAML::Nodes::Scalar) + URI.parse node.value + else + node.raise "Expected scalar, not #{node.class}" + end + end + end + module ProcessString def self.to_json(value : String, json : JSON::Builder) json.string value diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 95d9a80c..e6d4c764 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -764,7 +764,7 @@ struct Video end def engagement : Float64 - ((likes + dislikes) / views).round(4) + (((likes + dislikes) / views) * 100).round(4) end def reason : String? |