diff options
Diffstat (limited to 'src')
36 files changed, 1353 insertions, 482 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 1bdf3097..9f3d5d10 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -27,6 +27,7 @@ require "compress/zip" require "protodec/utils" require "./invidious/database/*" +require "./invidious/database/migrations/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" require "./invidious/frontend/*" @@ -34,6 +35,7 @@ require "./invidious/frontend/*" require "./invidious/*" require "./invidious/channels/*" require "./invidious/user/*" +require "./invidious/search/*" require "./invidious/routes/**" require "./invidious/jobs/**" @@ -102,6 +104,10 @@ Kemal.config.extra_options do |parser| puts SOFTWARE.to_pretty_json exit end + parser.on("--migrate", "Run any migrations (beta, use at your own risk!!") do + Invidious::Database::Migrator.new(PG_DB).migrate + exit + end end Kemal::CLI.new ARGV diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 280de702..93c4c0f7 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -57,20 +57,35 @@ end class Config include YAML::Serializable - property channel_threads : Int32 = 1 # Number of threads to use for crawling videos from channels (for updating subscriptions) - property feed_threads : Int32 = 1 # Number of threads to use for updating feeds - property output : String = "STDOUT" # Log file path or STDOUT - property log_level : LogLevel = LogLevel::Info # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr - property db : DBConfig? = nil # Database configuration with separate parameters (username, hostname, etc) - + # Number of threads to use for crawling videos from channels (for updating subscriptions) + property channel_threads : Int32 = 1 + # Time interval between two executions of the job that crawls channel videos (subscriptions update). + @[YAML::Field(converter: Preferences::TimeSpanConverter)] + property channel_refresh_interval : Time::Span = 30.minutes + # Number of threads to use for updating feeds + property feed_threads : Int32 = 1 + # Log file path or STDOUT + property output : String = "STDOUT" + # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr + property log_level : LogLevel = LogLevel::Info + # Database configuration with separate parameters (username, hostname, etc) + property db : DBConfig? = nil + + # Database configuration using 12-Factor "Database URL" syntax @[YAML::Field(converter: Preferences::URIConverter)] - property database_url : URI = URI.parse("") # Database configuration using 12-Factor "Database URL" syntax - property decrypt_polling : Bool = true # Use polling to keep decryption function up to date - property full_refresh : Bool = false # Used for crawling channels: threads should check all videos uploaded by a channel - property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// - property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions - property domain : String? # Domain to be used for links to resources on the site where an absolute URL is required - property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property database_url : URI = URI.parse("") + # Use polling to keep decryption function up to date + property decrypt_polling : Bool = true + # Used for crawling channels: threads should check all videos uploaded by a channel + property full_refresh : Bool = false + # Used to tell Invidious it is behind a proxy, so links to resources should be https:// + property https_only : Bool? + # HMAC signing key for CSRF tokens and verifying pubsub subscriptions + property hmac_key : String? + # Domain to be used for links to resources on the site where an absolute URL is required + property domain : String? + # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) + property use_pubsub_feeds : Bool | Int32 = false property popular_enabled : Bool = true property captcha_enabled : Bool = true property login_enabled : Bool = true @@ -79,28 +94,42 @@ class Config property admins : Array(String) = [] of String property external_port : Int32? = nil property default_user_preferences : ConfigPreferences = ConfigPreferences.from_yaml("") - property dmca_content : Array(String) = [] of String # For compliance with DMCA, disables download widget using list of video IDs - property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc. - property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards - property banner : String? = nil # Optional banner to be displayed along top of page for announcements, etc. - property hsts : Bool? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely - property disable_proxy : Bool? | Array(String)? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' + # For compliance with DMCA, disables download widget using list of video IDs + property dmca_content : Array(String) = [] of String + # Check table integrity, automatically try to add any missing columns, create tables, etc. + property check_tables : Bool = false + # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards + property cache_annotations : Bool = false + # Optional banner to be displayed along top of page for announcements, etc. + property banner : String? = nil + # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely + property hsts : Bool? = true + # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local' + property disable_proxy : Bool? | Array(String)? = false # URL to the modified source code to be easily AGPL compliant # Will display in the footer, next to the main source code link property modified_source_code_url : String? = nil + # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) @[YAML::Field(converter: Preferences::FamilyConverter)] - property force_resolve : Socket::Family = Socket::Family::UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729) - property port : Int32 = 3000 # Port to listen for connections (overridden by command line argument) - property host_binding : String = "0.0.0.0" # Host to bind (overridden by command line argument) - property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) - property use_quic : Bool = false # Use quic transport for youtube api - + property force_resolve : Socket::Family = Socket::Family::UNSPEC + # Port to listen for connections (overridden by command line argument) + property port : Int32 = 3000 + # Host to bind (overridden by command line argument) + property host_binding : String = "0.0.0.0" + # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) + property pool_size : Int32 = 100 + # Use quic transport for youtube api + property use_quic : Bool = false + + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] - property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format - property captcha_key : String? = nil # Key for Anti-Captcha - property captcha_api_url : String = "https://api.anti-captcha.com" # API URL for Anti-Captcha + property cookies : HTTP::Cookies = HTTP::Cookies.new + # Key for Anti-Captcha + property captcha_key : String? = nil + # API URL for Anti-Captcha + property captcha_api_url : String = "https://api.anti-captcha.com" def disabled?(option) case disabled = CONFIG.disable_proxy diff --git a/src/invidious/database/migration.cr b/src/invidious/database/migration.cr new file mode 100644 index 00000000..921d8f38 --- /dev/null +++ b/src/invidious/database/migration.cr @@ -0,0 +1,38 @@ +abstract class Invidious::Database::Migration + macro inherited + Migrator.migrations << self + end + + @@version : Int64? + + def self.version(version : Int32 | Int64) + @@version = version.to_i64 + end + + getter? completed = false + + def initialize(@db : DB::Database) + end + + abstract def up(conn : DB::Connection) + + def migrate + # migrator already ignores completed migrations + # but this is an extra check to make sure a migration doesn't run twice + return if completed? + + @db.transaction do |txn| + up(txn.connection) + track(txn.connection) + @completed = true + end + end + + def version : Int64 + @@version.not_nil! + end + + private def track(conn : DB::Connection) + conn.exec("INSERT INTO #{Migrator::MIGRATIONS_TABLE} (version) VALUES ($1)", version) + end +end diff --git a/src/invidious/database/migrations/0001_create_channels_table.cr b/src/invidious/database/migrations/0001_create_channels_table.cr new file mode 100644 index 00000000..a1362bcf --- /dev/null +++ b/src/invidious/database/migrations/0001_create_channels_table.cr @@ -0,0 +1,30 @@ +module Invidious::Database::Migrations + class CreateChannelsTable < Migration + version 1 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.channels + ( + id text NOT NULL, + author text, + updated timestamp with time zone, + deleted boolean, + subscribed timestamp with time zone, + CONSTRAINT channels_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.channels TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS channels_id_idx + ON public.channels + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0002_create_videos_table.cr b/src/invidious/database/migrations/0002_create_videos_table.cr new file mode 100644 index 00000000..c2ac84f8 --- /dev/null +++ b/src/invidious/database/migrations/0002_create_videos_table.cr @@ -0,0 +1,28 @@ +module Invidious::Database::Migrations + class CreateVideosTable < Migration + version 2 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE UNLOGGED TABLE IF NOT EXISTS public.videos + ( + id text NOT NULL, + info text, + updated timestamp with time zone, + CONSTRAINT videos_pkey PRIMARY KEY (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.videos TO current_user; + SQL + + conn.exec <<-SQL + CREATE UNIQUE INDEX IF NOT EXISTS id_idx + ON public.videos + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0003_create_channel_videos_table.cr b/src/invidious/database/migrations/0003_create_channel_videos_table.cr new file mode 100644 index 00000000..c9b62e4c --- /dev/null +++ b/src/invidious/database/migrations/0003_create_channel_videos_table.cr @@ -0,0 +1,35 @@ +module Invidious::Database::Migrations + class CreateChannelVideosTable < Migration + version 3 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.channel_videos + ( + id text NOT NULL, + title text, + published timestamp with time zone, + updated timestamp with time zone, + ucid text, + author text, + length_seconds integer, + live_now boolean, + premiere_timestamp timestamp with time zone, + views bigint, + CONSTRAINT channel_videos_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.channel_videos TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx + ON public.channel_videos + USING btree + (ucid COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0004_create_users_table.cr b/src/invidious/database/migrations/0004_create_users_table.cr new file mode 100644 index 00000000..a13ba15f --- /dev/null +++ b/src/invidious/database/migrations/0004_create_users_table.cr @@ -0,0 +1,34 @@ +module Invidious::Database::Migrations + class CreateUsersTable < Migration + version 4 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.users + ( + updated timestamp with time zone, + notifications text[], + subscriptions text[], + email text NOT NULL, + preferences text, + password text, + token text, + watched text[], + feed_needs_update boolean, + CONSTRAINT users_email_key UNIQUE (email) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.users TO current_user; + SQL + + conn.exec <<-SQL + CREATE UNIQUE INDEX IF NOT EXISTS email_unique_idx + ON public.users + USING btree + (lower(email) COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0005_create_session_ids_table.cr b/src/invidious/database/migrations/0005_create_session_ids_table.cr new file mode 100644 index 00000000..13c2228d --- /dev/null +++ b/src/invidious/database/migrations/0005_create_session_ids_table.cr @@ -0,0 +1,28 @@ +module Invidious::Database::Migrations + class CreateSessionIdsTable < Migration + version 5 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.session_ids + ( + id text NOT NULL, + email text, + issued timestamp with time zone, + CONSTRAINT session_ids_pkey PRIMARY KEY (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.session_ids TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS session_ids_id_idx + ON public.session_ids + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0006_create_nonces_table.cr b/src/invidious/database/migrations/0006_create_nonces_table.cr new file mode 100644 index 00000000..cf1229e1 --- /dev/null +++ b/src/invidious/database/migrations/0006_create_nonces_table.cr @@ -0,0 +1,27 @@ +module Invidious::Database::Migrations + class CreateNoncesTable < Migration + version 6 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.nonces + ( + nonce text, + expire timestamp with time zone, + CONSTRAINT nonces_id_key UNIQUE (nonce) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.nonces TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS nonces_nonce_idx + ON public.nonces + USING btree + (nonce COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0007_create_annotations_table.cr b/src/invidious/database/migrations/0007_create_annotations_table.cr new file mode 100644 index 00000000..dcecbc3b --- /dev/null +++ b/src/invidious/database/migrations/0007_create_annotations_table.cr @@ -0,0 +1,20 @@ +module Invidious::Database::Migrations + class CreateAnnotationsTable < Migration + version 7 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.annotations + ( + id text NOT NULL, + annotations xml, + CONSTRAINT annotations_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.annotations TO current_user; + SQL + end + end +end diff --git a/src/invidious/database/migrations/0008_create_playlists_table.cr b/src/invidious/database/migrations/0008_create_playlists_table.cr new file mode 100644 index 00000000..6aa16e1a --- /dev/null +++ b/src/invidious/database/migrations/0008_create_playlists_table.cr @@ -0,0 +1,50 @@ +module Invidious::Database::Migrations + class CreatePlaylistsTable < Migration + version 8 + + def up(conn : DB::Connection) + if !privacy_type_exists?(conn) + conn.exec <<-SQL + CREATE TYPE public.privacy AS ENUM + ( + 'Public', + 'Unlisted', + 'Private' + ); + SQL + end + + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlists + ( + title text, + id text primary key, + author text, + description text, + video_count integer, + created timestamptz, + updated timestamptz, + privacy privacy, + index int8[] + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON public.playlists TO current_user; + SQL + end + + private def privacy_type_exists?(conn : DB::Connection) : Bool + request = <<-SQL + SELECT 1 AS one + FROM pg_type + INNER JOIN pg_namespace ON pg_namespace.oid = pg_type.typnamespace + WHERE pg_namespace.nspname = 'public' + AND pg_type.typname = 'privacy' + LIMIT 1; + SQL + + !conn.query_one?(request, as: Int32).nil? + end + end +end diff --git a/src/invidious/database/migrations/0009_create_playlist_videos_table.cr b/src/invidious/database/migrations/0009_create_playlist_videos_table.cr new file mode 100644 index 00000000..84938b9b --- /dev/null +++ b/src/invidious/database/migrations/0009_create_playlist_videos_table.cr @@ -0,0 +1,27 @@ +module Invidious::Database::Migrations + class CreatePlaylistVideosTable < Migration + version 9 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlist_videos + ( + title text, + id text, + author text, + ucid text, + length_seconds integer, + published timestamptz, + plid text references playlists(id), + index int8, + live_now boolean, + PRIMARY KEY (index,plid) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.playlist_videos TO current_user; + SQL + end + end +end diff --git a/src/invidious/database/migrations/0010_make_videos_unlogged.cr b/src/invidious/database/migrations/0010_make_videos_unlogged.cr new file mode 100644 index 00000000..f5d19683 --- /dev/null +++ b/src/invidious/database/migrations/0010_make_videos_unlogged.cr @@ -0,0 +1,11 @@ +module Invidious::Database::Migrations + class MakeVideosUnlogged < Migration + version 10 + + def up(conn : DB::Connection) + conn.exec <<-SQL + ALTER TABLE public.videos SET UNLOGGED; + SQL + end + end +end diff --git a/src/invidious/database/migrator.cr b/src/invidious/database/migrator.cr new file mode 100644 index 00000000..660c3203 --- /dev/null +++ b/src/invidious/database/migrator.cr @@ -0,0 +1,49 @@ +class Invidious::Database::Migrator + MIGRATIONS_TABLE = "public.invidious_migrations" + + class_getter migrations = [] of Invidious::Database::Migration.class + + def initialize(@db : DB::Database) + end + + def migrate + versions = load_versions + + ran_migration = false + load_migrations.sort_by(&.version) + .each do |migration| + next if versions.includes?(migration.version) + + puts "Running migration: #{migration.class.name}" + migration.migrate + ran_migration = true + end + + puts "No migrations to run." unless ran_migration + end + + def pending_migrations? : Bool + versions = load_versions + + load_migrations.sort_by(&.version) + .any? { |migration| !versions.includes?(migration.version) } + end + + private def load_migrations : Array(Invidious::Database::Migration) + self.class.migrations.map(&.new(@db)) + end + + private def load_versions : Array(Int64) + create_migrations_table + @db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64) + end + + private def create_migrations_table + @db.exec <<-SQL + CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} ( + id bigserial PRIMARY KEY, + version bigint NOT NULL + ) + SQL + end +end diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr index 490d98cd..bfaa3fd5 100644 --- a/src/invidious/exceptions.cr +++ b/src/invidious/exceptions.cr @@ -1,3 +1,11 @@ +# Exception used to hold the bogus UCID during a channel search. +class ChannelSearchException < InfoException + getter channel : String + + def initialize(@channel) + end +end + # Exception used to hold the name of the missing item # Should be used in all parsing functions class BrokenTubeException < Exception diff --git a/src/invidious/frontend/misc.cr b/src/invidious/frontend/misc.cr new file mode 100644 index 00000000..43ba9f5c --- /dev/null +++ b/src/invidious/frontend/misc.cr @@ -0,0 +1,14 @@ +module Invidious::Frontend::Misc + extend self + + def redirect_url(env : HTTP::Server::Context) + prefs = env.get("preferences").as(Preferences) + + if prefs.automatic_instance_redirect + current_page = env.get?("current_page").as(String) + redirect_url = "/redirect?referer=#{current_page}" + else + redirect_url = "https://redirect.invidious.io#{env.request.resource}" + end + end +end diff --git a/src/invidious/frontend/search_filters.cr b/src/invidious/frontend/search_filters.cr new file mode 100644 index 00000000..8ac0af2e --- /dev/null +++ b/src/invidious/frontend/search_filters.cr @@ -0,0 +1,135 @@ +module Invidious::Frontend::SearchFilters + extend self + + # Generate the search filters collapsable widget. + def generate(filters : Search::Filters, query : String, page : Int, locale : String) : String + return String.build(8000) do |str| + str << "<div id='filters'>\n" + str << "\t<details id='filters-collapse'>" + str << "\t\t<summary>" << translate(locale, "search_filters_title") << "</summary>\n" + + str << "\t\t<div id='filters-box'><form action='/search' method='get'>\n" + + str << "\t\t\t<input type='hidden' name='q' value='" << HTML.escape(query) << "'>\n" + str << "\t\t\t<input type='hidden' name='page' value='" << page << "'>\n" + + str << "\t\t\t<div id='filters-flex'>" + + filter_wrapper(date) + filter_wrapper(type) + filter_wrapper(duration) + filter_wrapper(features) + filter_wrapper(sort) + + str << "\t\t\t</div>\n" + + str << "\t\t\t<div id='filters-apply'>" + str << "<button type='submit' class=\"pure-button pure-button-primary\">" + str << translate(locale, "search_filters_apply_button") + str << "</button></div>\n" + + str << "\t\t</form></div>\n" + + str << "\t</details>\n" + str << "</div>\n" + end + end + + # Generate wrapper HTML (`<div>`, filter name, etc...) around the + # `<input>` elements of a search filter + macro filter_wrapper(name) + str << "\t\t\t\t<div class=\"filter-column\"><fieldset>\n" + + str << "\t\t\t\t\t<legend><div class=\"filter-name underlined\">" + str << translate(locale, "search_filters_{{name}}_label") + str << "</div></legend>\n" + + str << "\t\t\t\t\t<div class=\"filter-options\">\n" + make_{{name}}_filter_options(str, filters.{{name}}, locale) + str << "\t\t\t\t\t</div>" + + str << "\t\t\t\t</fieldset></div>\n" + end + + # Generates the HTML for the list of radio buttons of the "date" search filter + def make_date_filter_options(str : String::Builder, value : Search::Filters::Date, locale : String) + {% for value in Invidious::Search::Filters::Date.constants %} + {% date = value.underscore %} + + str << "\t\t\t\t\t\t<div>" + str << "<input type='radio' name='date' id='filter-date-{{date}}' value='{{date}}'" + str << " checked" if value.{{date}}? + str << '>' + + str << "<label for='filter-date-{{date}}'>" + str << translate(locale, "search_filters_date_option_{{date}}") + str << "</label></div>\n" + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "type" search filter + def make_type_filter_options(str : String::Builder, value : Search::Filters::Type, locale : String) + {% for value in Invidious::Search::Filters::Type.constants %} + {% type = value.underscore %} + + str << "\t\t\t\t\t\t<div>" + str << "<input type='radio' name='type' id='filter-type-{{type}}' value='{{type}}'" + str << " checked" if value.{{type}}? + str << '>' + + str << "<label for='filter-type-{{type}}'>" + str << translate(locale, "search_filters_type_option_{{type}}") + str << "</label></div>\n" + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "duration" search filter + def make_duration_filter_options(str : String::Builder, value : Search::Filters::Duration, locale : String) + {% for value in Invidious::Search::Filters::Duration.constants %} + {% duration = value.underscore %} + + str << "\t\t\t\t\t\t<div>" + str << "<input type='radio' name='duration' id='filter-duration-{{duration}}' value='{{duration}}'" + str << " checked" if value.{{duration}}? + str << '>' + + str << "<label for='filter-duration-{{duration}}'>" + str << translate(locale, "search_filters_duration_option_{{duration}}") + str << "</label></div>\n" + {% end %} + end + + # Generates the HTML for the list of checkboxes of the "features" search filter + def make_features_filter_options(str : String::Builder, value : Search::Filters::Features, locale : String) + {% for value in Invidious::Search::Filters::Features.constants %} + {% if value.stringify != "All" && value.stringify != "None" %} + {% feature = value.underscore %} + + str << "\t\t\t\t\t\t<div>" + str << "<input type='checkbox' name='features' id='filter-feature-{{feature}}' value='{{feature}}'" + str << " checked" if value.{{feature}}? + str << '>' + + str << "<label for='filter-feature-{{feature}}'>" + str << translate(locale, "search_filters_features_option_{{feature}}") + str << "</label></div>\n" + {% end %} + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "sort" search filter + def make_sort_filter_options(str : String::Builder, value : Search::Filters::Sort, locale : String) + {% for value in Invidious::Search::Filters::Sort.constants %} + {% sort = value.underscore %} + + str << "\t\t\t\t\t\t<div>" + str << "<input type='radio' name='sort' id='filter-sort-{{sort}}' value='{{sort}}'" + str << " checked" if value.{{sort}}? + str << '>' + + str << "<label for='filter-sort-{{sort}}'>" + str << translate(locale, "search_filters_sort_option_{{sort}}") + str << "</label></div>\n" + {% end %} + end +end diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 6155e561..2eab6263 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -49,7 +49,7 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce issue_template += github_details("Backtrace", HTML.escape(exception.inspect_with_backtrace)) # URLs for the error message below - url_faq = "https://github.com/iv-org/documentation/blob/master/FAQ.md" + url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md" url_search_issues = "https://github.com/iv-org/invidious/issues" url_switch = "https://redirect.invidious.io" + env.request.resource diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 39e183f2..982b97d8 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -29,10 +29,10 @@ LOCALES_LIST = { "pt-BR" => "Português Brasileiro", # Portuguese (Brazil) "pt-PT" => "Português de Portugal", # Portuguese (Portugal) "ro" => "Română", # Romanian - "ru" => "русский", # Russian + "ru" => "Русский", # Russian "sq" => "Shqip", # Albanian - "sr" => "srpski (latinica)", # Serbian (Latin) - "sr_Cyrl" => "српски (ћирилица)", # Serbian (Cyrillic) + "sr" => "Srpski (latinica)", # Serbian (Latin) + "sr_Cyrl" => "Српски (ћирилица)", # Serbian (Cyrillic) "sv-SE" => "Svenska", # Swedish "tr" => "Türkçe", # Turkish "uk" => "Українська", # Ukrainian diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 9d403ddc..8ae5034a 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -51,6 +51,24 @@ def recode_length_seconds(time) end end +def decode_interval(string : String) : Time::Span + rawMinutes = string.try &.to_i32? + + if !rawMinutes + hours = /(?<hours>\d+)h/.match(string).try &.["hours"].try &.to_i32 + hours ||= 0 + + minutes = /(?<minutes>\d+)m(?!s)/.match(string).try &.["minutes"].try &.to_i32 + minutes ||= 0 + + time = Time::Span.new(hours: hours, minutes: minutes) + else + time = Time::Span.new(minutes: rawMinutes) + end + + return time +end + def decode_time(string) time = string.try &.to_f? diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 55fb8154..92681408 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -58,9 +58,8 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob end end - # TODO: make this configurable - LOGGER.debug("RefreshChannelsJob: Done, sleeping for thirty minutes") - sleep 30.minutes + LOGGER.debug("RefreshChannelsJob: Done, sleeping for #{CONFIG.channel_refresh_interval}") + sleep CONFIG.channel_refresh_interval Fiber.yield end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index c4d6643a..8650976d 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -251,18 +251,22 @@ module Invidious::Routes::API::V1::Channels def self.search(env) locale = env.get("preferences").as(Preferences).locale + region = env.params.query["region"]? env.response.content_type = "application/json" - ucid = env.params.url["ucid"] + query = Invidious::Search::Query.new(env.params.query, :channel, region) - query = env.params.query["q"]? - query ||= "" + # Required because we can't (yet) pass multiple parameter to the + # `Search::Query` initializer (in this case, an URL segment) + query.channel = env.params.url["ucid"] - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + begin + search_results = query.process + rescue ex + return error_json(400, ex) + end - search_results = channel_search(query, page, ucid) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 0b0853b1..21451d33 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -5,34 +5,14 @@ module Invidious::Routes::API::V1::Search env.response.content_type = "application/json" - query = env.params.query["q"]? - query ||= "" - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "relevance" - - date = env.params.query["date"]?.try &.downcase - date ||= "" - - duration = env.params.query["duration"]?.try &.downcase - duration ||= "" - - features = env.params.query["features"]?.try &.split(",").map(&.downcase) - features ||= [] of String - - content_type = env.params.query["type"]?.try &.downcase - content_type ||= "video" + query = Invidious::Search::Query.new(env.params.query, :regular, region) begin - search_params = produce_search_params(page, sort_by, date, content_type, duration, features) + search_results = query.process rescue ex return error_json(400, ex) end - search_results = search(query, search_params, region) JSON.build do |json| json.array do search_results.each do |item| @@ -43,20 +23,20 @@ module Invidious::Routes::API::V1::Search end def self.search_suggestions(env) - locale = env.get("preferences").as(Preferences).locale - region = env.params.query["region"]? + preferences = env.get("preferences").as(Preferences) + region = env.params.query["region"]? || preferences.region env.response.content_type = "application/json" - query = env.params.query["q"]? - query ||= "" + query = env.params.query["q"]? || "" begin - headers = HTTP::Headers{":authority" => "suggestqueries.google.com"} - response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body + client = HTTP::Client.new("suggestqueries-clients6.youtube.com") + url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt" + + response = client.get(url).body - body = response[35..-2] - body = JSON.parse(body).as_a + body = JSON.parse(response[5..-1]).as_a suggestions = body[1].as_a[0..-2] JSON.build do |json| diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index dbeb4f97..de981d81 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -212,7 +212,10 @@ module Invidious::Routes::Playlists end def self.add_playlist_items_page(env) - locale = env.get("preferences").as(Preferences).locale + prefs = env.get("preferences").as(Preferences) + locale = prefs.locale + + region = env.params.query["region"]? || prefs.region user = env.get? "user" sid = env.get? "sid" @@ -236,15 +239,10 @@ module Invidious::Routes::Playlists return env.redirect referer end - query = env.params.query["q"]? - if query - begin - search_query, items, operators = process_search_query(query, page, user, region: nil) - videos = items.select(SearchVideo).map(&.as(SearchVideo)) - rescue ex - videos = [] of SearchVideo - end - else + begin + query = Invidious::Search::Query.new(env.params.query, :playlist, region) + videos = query.process.select(SearchVideo).map(&.as(SearchVideo)) + rescue ex videos = [] of SearchVideo end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 3f4c7e5e..e60d0081 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -37,37 +37,29 @@ module Invidious::Routes::Search end def self.search(env) - locale = env.get("preferences").as(Preferences).locale - region = env.params.query["region"]? + prefs = env.get("preferences").as(Preferences) + locale = prefs.locale - query = env.params.query["search_query"]? - query ||= env.params.query["q"]? + region = env.params.query["region"]? || prefs.region + + query = Invidious::Search::Query.new(env.params.query, :regular, region) - if !query || query.empty? + if query.empty? # Display the full page search box implemented in #1977 env.set "search", "" templated "search_homepage", navbar_search: false else - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - user = env.get? "user" begin - search_query, videos, operators = process_search_query(query, page, user, region: region) + videos = query.process rescue ex : ChannelSearchException return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex return error_template(500, ex) end - operator_hash = {} of String => String - operators.each do |operator| - key, value = operator.downcase.split(":") - operator_hash[key] = value - end - - env.set "search", query + env.set "search", query.text templated "search" end end diff --git a/src/invidious/search.cr b/src/invidious/search.cr deleted file mode 100644 index ae106bf6..00000000 --- a/src/invidious/search.cr +++ /dev/null @@ -1,254 +0,0 @@ -class ChannelSearchException < InfoException - getter channel : String - - def initialize(@channel) - end -end - -def channel_search(query, page, channel) : Array(SearchItem) - response = YT_POOL.client &.get("/channel/#{channel}") - - if response.status_code == 404 - response = YT_POOL.client &.get("/user/#{channel}") - response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 - initial_data = extract_initial_data(response.body) - ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) - raise ChannelSearchException.new(channel) if !ucid - else - ucid = channel - end - - continuation = produce_channel_search_continuation(ucid, query, page) - response_json = YoutubeAPI.browse(continuation) - - continuation_items = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem if !continuation_items - - items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| - extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } - end - - return items -end - -def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem) - return [] of SearchItem if query.empty? - - client_config = YoutubeAPI::ClientConfig.new(region: region) - initial_data = YoutubeAPI.search(query, search_params, client_config: client_config) - - return extract_items(initial_data) -end - -def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "", - duration : String = "", features : Array(String) = [] of String) - object = { - "1:varint" => 0_i64, - "2:embedded" => {} of String => Int64, - "9:varint" => ((page - 1) * 20).to_i64, - } - - case sort - when "relevance" - object["1:varint"] = 0_i64 - when "rating" - object["1:varint"] = 1_i64 - when "upload_date", "date" - object["1:varint"] = 2_i64 - when "view_count", "views" - object["1:varint"] = 3_i64 - else - raise "No sort #{sort}" - end - - case date - when "hour" - object["2:embedded"].as(Hash)["1:varint"] = 1_i64 - when "today" - object["2:embedded"].as(Hash)["1:varint"] = 2_i64 - when "week" - object["2:embedded"].as(Hash)["1:varint"] = 3_i64 - when "month" - object["2:embedded"].as(Hash)["1:varint"] = 4_i64 - when "year" - object["2:embedded"].as(Hash)["1:varint"] = 5_i64 - else nil # Ignore - end - - case content_type - when "video" - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - when "channel" - object["2:embedded"].as(Hash)["2:varint"] = 2_i64 - when "playlist" - object["2:embedded"].as(Hash)["2:varint"] = 3_i64 - when "movie" - object["2:embedded"].as(Hash)["2:varint"] = 4_i64 - when "show" - object["2:embedded"].as(Hash)["2:varint"] = 5_i64 - when "all" - # - else - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - end - - case duration - when "short" - object["2:embedded"].as(Hash)["3:varint"] = 1_i64 - when "long" - object["2:embedded"].as(Hash)["3:varint"] = 2_i64 - else nil # Ignore - end - - features.each do |feature| - case feature - when "hd" - object["2:embedded"].as(Hash)["4:varint"] = 1_i64 - when "subtitles" - object["2:embedded"].as(Hash)["5:varint"] = 1_i64 - when "creative_commons", "cc" - object["2:embedded"].as(Hash)["6:varint"] = 1_i64 - when "3d" - object["2:embedded"].as(Hash)["7:varint"] = 1_i64 - when "live", "livestream" - object["2:embedded"].as(Hash)["8:varint"] = 1_i64 - when "purchased" - object["2:embedded"].as(Hash)["9:varint"] = 1_i64 - when "4k" - object["2:embedded"].as(Hash)["14:varint"] = 1_i64 - when "360" - object["2:embedded"].as(Hash)["15:varint"] = 1_i64 - when "location" - object["2:embedded"].as(Hash)["23:varint"] = 1_i64 - when "hdr" - object["2:embedded"].as(Hash)["25:varint"] = 1_i64 - else nil # Ignore - end - end - - if object["2:embedded"].as(Hash).empty? - object.delete("2:embedded") - end - - params = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return params -end - -def produce_channel_search_continuation(ucid, query, page) - if page <= 1 - idx = 0_i64 - else - idx = 30_i64 * (page - 1) - end - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "search", - "6:varint" => 1_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "15:base64" => { - "3:varint" => idx, - }, - "23:varint" => 0_i64, - }, - "11:string" => query, - "35:string" => "browse-feed#{ucid}search", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def process_search_query(query, page, user, region) - if user - user = user.as(Invidious::User) - view_name = "subscriptions_#{sha256(user.email)}" - end - - channel = nil - content_type = "all" - date = "" - duration = "" - features = [] of String - sort = "relevance" - subscriptions = nil - - operators = query.split(" ").select(&.match(/\w+:[\w,]+/)) - operators.each do |operator| - key, value = operator.downcase.split(":") - - case key - when "channel", "user" - channel = operator.split(":")[-1] - when "content_type", "type" - content_type = value - when "date" - date = value - when "duration" - duration = value - when "feature", "features" - features = value.split(",") - when "sort" - sort = value - when "subscriptions" - subscriptions = value == "true" - else - operators.delete(operator) - end - end - - search_query = (query.split(" ") - operators).join(" ") - - if channel - items = channel_search(search_query, page, channel) - elsif subscriptions - if view_name - items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM ( - SELECT *, - to_tsvector(#{view_name}.title) || - to_tsvector(#{view_name}.author) - as document - FROM #{view_name} - ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo) - else - items = [] of ChannelVideo - end - else - search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, - duration: duration, features: features) - - items = search(search_query, search_params, region) - end - - # Light processing to flatten search results out of Categories. - # They should ideally be supported in the future. - items_without_category = [] of SearchItem | ChannelVideo - items.each do |i| - if i.is_a? Category - i.contents.each do |nest_i| - if !nest_i.is_a? Video - items_without_category << nest_i - end - end - else - items_without_category << i - end - end - - {search_query, items_without_category, operators} -end diff --git a/src/invidious/search/ctoken.cr b/src/invidious/search/ctoken.cr new file mode 100644 index 00000000..161065e0 --- /dev/null +++ b/src/invidious/search/ctoken.cr @@ -0,0 +1,32 @@ +def produce_channel_search_continuation(ucid, query, page) + if page <= 1 + idx = 0_i64 + else + idx = 30_i64 * (page - 1) + end + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:base64" => { + "2:string" => "search", + "6:varint" => 1_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "15:base64" => { + "3:varint" => idx, + }, + "23:varint" => 0_i64, + }, + "11:string" => query, + "35:string" => "browse-feed#{ucid}search", + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation +end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr new file mode 100644 index 00000000..c2b5c758 --- /dev/null +++ b/src/invidious/search/filters.cr @@ -0,0 +1,376 @@ +require "protodec/utils" +require "http/params" + +module Invidious::Search + struct Filters + # Values correspond to { "2:embedded": { "1:varint": <X> }} + # except for "None" which is only used by us (= nothing selected) + enum Date + None = 0 + Hour = 1 + Today = 2 + Week = 3 + Month = 4 + Year = 5 + end + + # Values correspond to { "2:embedded": { "2:varint": <X> }} + # except for "All" which is only used by us (= nothing selected) + enum Type + All = 0 + Video = 1 + Channel = 2 + Playlist = 3 + Movie = 4 + + # Has it been removed? + # (Not available on youtube's UI) + Show = 5 + end + + # Values correspond to { "2:embedded": { "3:varint": <X> }} + # except for "None" which is only used by us (= nothing selected) + enum Duration + None = 0 + Short = 1 # "Under 4 minutes" + Long = 2 # "Over 20 minutes" + Medium = 3 # "4 - 20 minutes" + end + + # Note: flag enums automatically generate + # "none" and "all" members + @[Flags] + enum Features + Live + FourK # "4K" + HD + Subtitles # "Subtitles/CC" + CCommons # "Creative Commons" + ThreeSixty # "360°" + VR180 + ThreeD # "3D" + HDR + Location + Purchased + end + + # Values correspond to { "1:varint": <X> } + enum Sort + Relevance = 0 + Rating = 1 + Date = 2 + Views = 3 + end + + # Parameters are sorted as on Youtube + property date : Date + property type : Type + property duration : Duration + property features : Features + property sort : Sort + + def initialize( + *, # All parameters must be named + @date : Date = Date::None, + @type : Type = Type::All, + @duration : Duration = Duration::None, + @features : Features = Features::None, + @sort : Sort = Sort::Relevance + ) + end + + def default? : Bool + return @date.none? && @type.all? && @duration.none? && \ + @features.none? && @sort.relevance? + end + + # ------------------- + # Invidious params + # ------------------- + + def self.parse_features(raw : Array(String)) : Features + # Initialize return variable + features = Features.new(0) + + raw.each do |ft| + case ft.downcase + when "live", "livestream" + features = features | Features::Live + when "4k" then features = features | Features::FourK + when "hd" then features = features | Features::HD + when "subtitles" then features = features | Features::Subtitles + when "creative_commons", "commons", "cc" + features = features | Features::CCommons + when "360" then features = features | Features::ThreeSixty + when "vr180" then features = features | Features::VR180 + when "3d" then features = features | Features::ThreeD + when "hdr" then features = features | Features::HDR + when "location" then features = features | Features::Location + when "purchased" then features = features | Features::Purchased + end + end + + return features + end + + def self.format_features(features : Features) : String + # Directly return an empty string if there are no features + return "" if features.none? + + # Initialize return variable + str = [] of String + + str << "live" if features.live? + str << "4k" if features.four_k? + str << "hd" if features.hd? + str << "subtitles" if features.subtitles? + str << "commons" if features.c_commons? + str << "360" if features.three_sixty? + str << "vr180" if features.vr180? + str << "3d" if features.three_d? + str << "hdr" if features.hdr? + str << "location" if features.location? + str << "purchased" if features.purchased? + + return str.join(',') + end + + def self.from_legacy_filters(str : String) : {Filters, String, String, Bool} + # Split search query on spaces + members = str.split(' ') + + # Output variables + channel = "" + filters = Filters.new + subscriptions = false + + # Array to hold the non-filter members + query = [] of String + + # Parse! + members.each do |substr| + # Separator operators + operators = substr.split(':') + + case operators[0] + when "user", "channel" + next if operators.size != 2 + channel = operators[1] + # + when "type", "content_type" + next if operators.size != 2 + type = Type.parse?(operators[1]) + filters.type = type if !type.nil? + # + when "date" + next if operators.size != 2 + date = Date.parse?(operators[1]) + filters.date = date if !date.nil? + # + when "duration" + next if operators.size != 2 + duration = Duration.parse?(operators[1]) + filters.duration = duration if !duration.nil? + # + when "feature", "features" + next if operators.size != 2 + features = parse_features(operators[1].split(',')) + filters.features = features if !features.nil? + # + when "sort" + next if operators.size != 2 + sort = Sort.parse?(operators[1]) + filters.sort = sort if !sort.nil? + # + when "subscriptions" + next if operators.size != 2 + subscriptions = {"true", "on", "yes", "1"}.any?(&.== operators[1]) + # + else + query << substr + end + end + + # Re-assemble query (without filters) + cleaned_query = query.join(' ') + + return {filters, channel, cleaned_query, subscriptions} + end + + def self.from_iv_params(params : HTTP::Params) : Filters + # Temporary variables + filters = Filters.new + + if type = params["type"]? + filters.type = Type.parse?(type) || Type::All + params.delete("type") + end + + if date = params["date"]? + filters.date = Date.parse?(date) || Date::None + params.delete("date") + end + + if duration = params["duration"]? + filters.duration = Duration.parse?(duration) || Duration::None + params.delete("duration") + end + + features = params.fetch_all("features") + if !features.empty? + # Un-array input so it can be treated as a comma-separated list + features = features[0].split(',') if features.size == 1 + + filters.features = parse_features(features) || Features::None + params.delete_all("features") + end + + if sort = params["sort"]? + filters.sort = Sort.parse?(sort) || Sort::Relevance + params.delete("sort") + end + + return filters + end + + def to_iv_params : HTTP::Params + # Temporary variables + raw_params = {} of String => Array(String) + + raw_params["date"] = [@date.to_s.underscore] if !@date.none? + raw_params["type"] = [@type.to_s.underscore] if !@type.all? + raw_params["sort"] = [@sort.to_s.underscore] if !@sort.relevance? + + if !@duration.none? + raw_params["duration"] = [@duration.to_s.underscore] + end + + if !@features.none? + raw_params["features"] = [Filters.format_features(@features)] + end + + return HTTP::Params.new(raw_params) + end + + # ------------------- + # Youtube params + # ------------------- + + # Produce the youtube search parameters for the + # innertube API (base64-encoded protobuf object). + def to_yt_params(page : Int = 1) : String + # Initialize the embedded protobuf object + embedded = {} of String => Int64 + + # Add these field only if associated parameter is selected + embedded["1:varint"] = @date.to_i64 if !@date.none? + embedded["2:varint"] = @type.to_i64 if !@type.all? + embedded["3:varint"] = @duration.to_i64 if !@duration.none? + + if !@features.none? + # All features have a value of "1" when enabled, and + # the field is omitted when the feature is no selected. + embedded["4:varint"] = 1_i64 if @features.includes?(Features::HD) + embedded["5:varint"] = 1_i64 if @features.includes?(Features::Subtitles) + embedded["6:varint"] = 1_i64 if @features.includes?(Features::CCommons) + embedded["7:varint"] = 1_i64 if @features.includes?(Features::ThreeD) + embedded["8:varint"] = 1_i64 if @features.includes?(Features::Live) + embedded["9:varint"] = 1_i64 if @features.includes?(Features::Purchased) + embedded["14:varint"] = 1_i64 if @features.includes?(Features::FourK) + embedded["15:varint"] = 1_i64 if @features.includes?(Features::ThreeSixty) + embedded["23:varint"] = 1_i64 if @features.includes?(Features::Location) + embedded["25:varint"] = 1_i64 if @features.includes?(Features::HDR) + embedded["26:varint"] = 1_i64 if @features.includes?(Features::VR180) + end + + # Initialize an empty protobuf object + object = {} of String => (Int64 | String | Hash(String, Int64)) + + # As usual, everything can be omitted if it has no value + object["2:embedded"] = embedded if !embedded.empty? + + # Default sort is "relevance", so when this option is selected, + # the associated field can be omitted. + if !@sort.relevance? + object["1:varint"] = @sort.to_i64 + end + + # Add page number (if provided) + if page > 1 + object["9:varint"] = ((page - 1) * 20).to_i64 + end + + # If the object is empty, return an empty string, + # otherwise encode to protobuf then to base64 + return "" if object.empty? + + return object + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + end + + # Function to parse the `sp` URL parameter from Youtube + # search page. It's a base64-encoded protobuf object. + def self.from_yt_params(params : HTTP::Params) : Filters + # Initialize output variable + filters = Filters.new + + # Get parameter, and check emptyness + search_params = params["sp"]? + + if search_params.nil? || search_params.empty? + return filters + end + + # Decode protobuf object + object = search_params + .try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + # Parse items from embedded object + if embedded = object["2:0:embedded"]? + # All the following fields (date, type, duration) are optional. + if date = embedded["1:0:varint"]? + filters.date = Date.from_value?(date.as_i) || Date::None + end + + if type = embedded["2:0:varint"]? + filters.type = Type.from_value?(type.as_i) || Type::All + end + + if duration = embedded["3:0:varint"]? + filters.duration = Duration.from_value?(duration.as_i) || Duration::None + end + + # All features should have a value of "1" when enabled, and + # the field should be omitted when the feature is no selected. + features = 0 + features += (embedded["4:0:varint"]?.try &.as_i == 1_i64) ? Features::HD.value : 0 + features += (embedded["5:0:varint"]?.try &.as_i == 1_i64) ? Features::Subtitles.value : 0 + features += (embedded["6:0:varint"]?.try &.as_i == 1_i64) ? Features::CCommons.value : 0 + features += (embedded["7:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeD.value : 0 + features += (embedded["8:0:varint"]?.try &.as_i == 1_i64) ? Features::Live.value : 0 + features += (embedded["9:0:varint"]?.try &.as_i == 1_i64) ? Features::Purchased.value : 0 + features += (embedded["14:0:varint"]?.try &.as_i == 1_i64) ? Features::FourK.value : 0 + features += (embedded["15:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeSixty.value : 0 + features += (embedded["23:0:varint"]?.try &.as_i == 1_i64) ? Features::Location.value : 0 + features += (embedded["25:0:varint"]?.try &.as_i == 1_i64) ? Features::HDR.value : 0 + features += (embedded["26:0:varint"]?.try &.as_i == 1_i64) ? Features::VR180.value : 0 + + filters.features = Features.from_value?(features) || Features::None + end + + if sort = object["1:0:varint"]? + filters.sort = Sort.from_value?(sort.as_i) || Sort::Relevance + end + + # Remove URL parameter and return result + params.delete("sp") + return filters + end + end +end diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr new file mode 100644 index 00000000..d1409c06 --- /dev/null +++ b/src/invidious/search/processors.cr @@ -0,0 +1,64 @@ +module Invidious::Search + module Processors + extend self + + # Regular search (`/search` endpoint) + def regular(query : Query) : Array(SearchItem) + search_params = query.filters.to_yt_params(page: query.page) + + client_config = YoutubeAPI::ClientConfig.new(region: query.region) + initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) + + return extract_items(initial_data) + end + + # Search a youtube channel + # TODO: clean code, and rely more on YoutubeAPI + def channel(query : Query) : Array(SearchItem) + response = YT_POOL.client &.get("/channel/#{query.channel}") + + if response.status_code == 404 + response = YT_POOL.client &.get("/user/#{query.channel}") + response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404 + initial_data = extract_initial_data(response.body) + ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) + raise ChannelSearchException.new(query.channel) if !ucid + else + ucid = query.channel + end + + continuation = produce_channel_search_continuation(ucid, query.text, query.page) + response_json = YoutubeAPI.browse(continuation) + + continuation_items = response_json["onResponseReceivedActions"]? + .try &.[0]["appendContinuationItemsAction"]["continuationItems"] + + return [] of SearchItem if !continuation_items + + items = [] of SearchItem + continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| + extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } + end + + return items + end + + # Search inside of user subscriptions + def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo) + view_name = "subscriptions_#{sha256(user.email)}" + + return PG_DB.query_all(" + SELECT id,title,published,updated,ucid,author,length_seconds + FROM ( + SELECT *, + to_tsvector(#{view_name}.title) || + to_tsvector(#{view_name}.author) + as document + FROM #{view_name} + ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", + query.text, (query.page - 1) * 20, + as: ChannelVideo + ) + end + end +end diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr new file mode 100644 index 00000000..1c2b37d2 --- /dev/null +++ b/src/invidious/search/query.cr @@ -0,0 +1,148 @@ +module Invidious::Search + class Query + enum Type + # Types related to YouTube + Regular # Youtube search page + Channel # Youtube channel search box + + # Types specific to Invidious + Subscriptions # Search user subscriptions + Playlist # "Add playlist item" search + end + + @type : Type = Type::Regular + + @raw_query : String + @query : String = "" + + property filters : Filters = Filters.new + property page : Int32 + property region : String? + property channel : String = "" + + # Return true if @raw_query is either `nil` or empty + private def empty_raw_query? + return @raw_query.empty? + end + + # Same as `empty_raw_query?`, but named for external use + def empty? + return self.empty_raw_query? + end + + # Getter for the query string. + # It is named `text` to reduce confusion (`search_query.text` makes more + # sense than `search_query.query`) + def text + return @query + end + + # Initialize a new search query. + # Parameters are used to get the query string, the page number + # and the search filters (if any). Type tells this function + # where it is being called from (See `Type` above). + def initialize( + params : HTTP::Params, + @type : Type = Type::Regular, + @region : String? = nil + ) + # Get the raw search query string (common to all search types). In + # Regular search mode, also look for the `search_query` URL parameter + if @type.regular? + @raw_query = params["q"]? || params["search_query"]? || "" + else + @raw_query = params["q"]? || "" + end + + # Get the page number (also common to all search types) + @page = params["page"]?.try &.to_i? || 1 + + # Stop here is raw query in empty + # NOTE: maybe raise in the future? + return if self.empty_raw_query? + + # Specific handling + case @type + when .playlist?, .channel? + # In "add playlist item" mode, filters are parsed from the query + # string itself (legacy), and the channel is ignored. + # + # In "channel search" mode, filters are ignored, but we still parse + # the query prevent transmission of legacy filters to youtube. + # + @filters, @query, @channel, _ = Filters.from_legacy_filters(@raw_query || "") + # + when .subscriptions?, .regular? + if params["sp"]? + # Parse the `sp` URL parameter (youtube compatibility) + @filters = Filters.from_yt_params(params) + @query = @raw_query || "" + else + # Parse invidious URL parameters (sort, date, etc...) + @filters = Filters.from_iv_params(params) + @channel = params["channel"]? || "" + + if @filters.default? && @raw_query.includes?(':') + # Parse legacy filters from query + @filters, @query, @channel, subs = Filters.from_legacy_filters(@raw_query || "") + else + @query = @raw_query || "" + end + + if !@channel.empty? + # Switch to channel search mode (filters will be ignored) + @type = Type::Channel + elsif subs + # Switch to subscriptions search mode + @type = Type::Subscriptions + end + end + end + end + + # Run the search query using the corresponding search processor. + # Returns either the results or an empty array of `SearchItem`. + def process(user : Invidious::User? = nil) : Array(SearchItem) | Array(ChannelVideo) + items = [] of SearchItem + + # Don't bother going further if search query is empty + return items if self.empty_raw_query? + + case @type + when .regular?, .playlist? + items = unnest_items(Processors.regular(self)) + # + when .channel? + items = Processors.channel(self) + # + when .subscriptions? + if user + items = Processors.subscriptions(self, user.as(Invidious::User)) + end + end + + return items + end + + # TODO: clean code + private def unnest_items(all_items) : Array(SearchItem) + items = [] of SearchItem + + # Light processing to flatten search results out of Categories. + # They should ideally be supported in the future. + all_items.each do |i| + if i.is_a? Category + i.contents.each do |nest_i| + if !nest_i.is_a? Video + items << nest_i + end + end + else + items << i + end + end + + return items + end + end +end diff --git a/src/invidious/user/cookies.cr b/src/invidious/user/cookies.cr index 99df1b07..65e079ec 100644 --- a/src/invidious/user/cookies.cr +++ b/src/invidious/user/cookies.cr @@ -17,7 +17,8 @@ struct Invidious::User value: sid, expires: Time.utc + 2.years, secure: SECURE, - http_only: true + http_only: true, + samesite: HTTP::Cookie::SameSite::Strict ) end @@ -30,7 +31,8 @@ struct Invidious::User value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, secure: SECURE, - http_only: true + http_only: false, + samesite: HTTP::Cookie::SameSite::Strict ) end end diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index 8ecbe7a0..b3059403 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -257,4 +257,18 @@ struct Preferences cookies end end + + module TimeSpanConverter + def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder) + return yaml.scalar value.total_minutes.to_i32 + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span + if node.is_a?(YAML::Nodes::Scalar) + return decode_interval(node.value) + else + node.raise "Expected scalar, not #{node.class}" + end + end + end end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 81fce5b8..31ae90c7 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -585,7 +585,7 @@ struct Video def allowed_regions info - .dig("microformat", "playerMicroformatRenderer", "availableCountries") + .dig?("microformat", "playerMicroformatRenderer", "availableCountries") .try &.as_a.map &.as_s || [] of String end @@ -876,7 +876,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) if context_screen == "embed" - client_config.client_type = YoutubeAPI::ClientType::WebScreenEmbed + client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed end player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) @@ -1094,6 +1094,10 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) end return video +rescue DB::Error + # Avoid common `DB::PoolRetryAttemptsExceeded` error and friends + # Note: All DB errors inherit from `DB::Error` + return fetch_video(id, region) end def fetch_video(id, region) diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr index ad50909a..22870317 100644 --- a/src/invidious/views/add_playlist_items.ecr +++ b/src/invidious/views/add_playlist_items.ecr @@ -11,7 +11,9 @@ <legend><a href="/playlist?list=<%= playlist.id %>"><%= translate(locale, "Editing playlist `x`", %|"#{HTML.escape(playlist.title)}"|) %></a></legend> <fieldset> - <input class="pure-input-1" type="search" name="q" <% if query %>value="<%= HTML.escape(query) %>"<% else %>placeholder="<%= translate(locale, "Search for videos") %>"<% end %>> + <input class="pure-input-1" type="search" name="q" + <% if query %>value="<%= HTML.escape(query.text) %>"<% end %> + placeholder="<%= translate(locale, "Search for videos") %>"> <input type="hidden" name="list" value="<%= plid %>"> </fieldset> </form> @@ -38,10 +40,11 @@ </div> <% if query %> + <%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%> <div class="pure-g h-box"> <div class="pure-u-1 pure-u-lg-1-5"> - <% if page > 1 %> - <a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page - 1 %>"> + <% if query.page > 1 %> + <a href="/add_playlist_items?list=<%= plid %>&q=<%= query_encoded %>&page=<%= page - 1 %>"> <%= translate(locale, "Previous page") %> </a> <% end %> @@ -49,7 +52,7 @@ <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if videos.size >= 20 %> - <a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page + 1 %>"> + <a href="/add_playlist_items?list=<%= plid %>&q=<%= query_encoded %>&page=<%= page + 1 %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index 45bbdefc..7110703e 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -1,147 +1,62 @@ <% content_for "header" do %> -<title><%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious</title> +<title><%= query.text.size > 30 ? HTML.escape(query.text[0,30].rstrip(".")) + "…" : HTML.escape(query.text) %> - Invidious</title> +<link rel="stylesheet" href="/css/search.css?v=<%= ASSET_COMMIT %>"> <% end %> -<% search_query_encoded = env.get?("search").try { |x| URI.encode_www_form(x.as(String), space_to_plus: true) } %> +<%- + search_query_encoded = URI.encode_www_form(query.text, space_to_plus: true) + filter_params = query.filters.to_iv_params -<!-- Search redirection and filtering UI --> -<% if videos.size == 0 %> - <h3 style="text-align: center"> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Broken? Try another Invidious Instance!") %></a> - </h3> -<% else %> - <details id="filters"> - <summary> - <h3 style="display:inline"> <%= translate(locale, "filter") %> </h3> - </summary> - <div id="filters" class="pure-g h-box"> - <div class="pure-u-1-3 pure-u-md-1-5"> - <b><%= translate(locale, "date") %></b> - <hr/> - <% ["hour", "today", "week", "month", "year"].each do |date| %> - <div class="pure-u-1 pure-md-1-5"> - <% if operator_hash.fetch("date", "all") == date %> - <b><%= translate(locale, date) %></b> - <% else %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?date:[a-z]+/, "") + " date:" + date) %>&page=<%= page %>"> - <%= translate(locale, date) %> - </a> - <% end %> - </div> - <% end %> - </div> - <div class="pure-u-1-3 pure-u-md-1-5"> - <b><%= translate(locale, "content_type") %></b> - <hr/> - <% ["video", "channel", "playlist", "movie", "show"].each do |content_type| %> - <div class="pure-u-1 pure-md-1-5"> - <% if operator_hash.fetch("content_type", "all") == content_type %> - <b><%= translate(locale, content_type) %></b> - <% else %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?content_type:[a-z]+/, "") + " content_type:" + content_type) %>&page=<%= page %>"> - <%= translate(locale, content_type) %> - </a> - <% end %> - </div> - <% end %> - </div> - <div class="pure-u-1-3 pure-u-md-1-5"> - <b><%= translate(locale, "duration") %></b> - <hr/> - <% ["short", "long"].each do |duration| %> - <div class="pure-u-1 pure-md-1-5"> - <% if operator_hash.fetch("duration", "all") == duration %> - <b><%= translate(locale, duration) %></b> - <% else %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?duration:[a-z]+/, "") + " duration:" + duration) %>&page=<%= page %>"> - <%= translate(locale, duration) %> - </a> - <% end %> - </div> - <% end %> - </div> - <div class="pure-u-1-3 pure-u-md-1-5"> - <b><%= translate(locale, "features") %></b> - <hr/> - <% ["hd", "subtitles", "creative_commons", "3d", "live", "purchased", "4k", "360", "location", "hdr"].each do |feature| %> - <div class="pure-u-1 pure-md-1-5"> - <% if operator_hash.fetch("features", "all").includes?(feature) %> - <b><%= translate(locale, feature) %></b> - <% elsif operator_hash.has_key?("features") %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/features:/, "features:" + feature + ",")) %>&page=<%= page %>"> - <%= translate(locale, feature) %> - </a> - <% else %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil! + " features:" + feature) %>&page=<%= page %>"> - <%= translate(locale, feature) %> - </a> - <% end %> - </div> - <% end %> - </div> - <div class="pure-u-1-3 pure-u-md-1-5"> - <b><%= translate(locale, "sort") %></b> - <hr/> - <% ["relevance", "rating", "date", "views"].each do |sort| %> - <div class="pure-u-1 pure-md-1-5"> - <% if operator_hash.fetch("sort", "relevance") == sort %> - <b><%= translate(locale, sort) %></b> - <% else %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?sort:[a-z]+/, "") + " sort:" + sort) %>&page=<%= page %>"> - <%= translate(locale, sort) %> - </a> - <% end %> - </div> - <% end %> - </div> - </div> - </details> -<% end %> + url_prev_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page - 1}" + url_next_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page + 1}" -<% if videos.size == 0 %> - <hr style="margin: 0;"/> -<% else %> - <hr/> -<% end %> + redirect_url = Invidious::Frontend::Misc.redirect_url(env) +-%> + +<!-- Search redirection and filtering UI --> +<%= Invidious::Frontend::SearchFilters.generate(query.filters, query.text, query.page, locale) %> +<hr/> <div class="pure-g h-box v-box"> <div class="pure-u-1 pure-u-lg-1-5"> - <% if page > 1 %> - <a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>"> - <%= translate(locale, "Previous page") %> - </a> - <% end %> + <%- if query.page > 1 -%> + <a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a> + <%- end -%> </div> <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if videos.size >= 20 %> - <a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>"> - <%= translate(locale, "Next page") %> - </a> - <% end %> + <%- if videos.size >= 20 -%> + <a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a> + <%- end -%> </div> </div> +<%- if videos.empty? -%> +<div class="h-box no-results-error"> + <div> + <%= translate(locale, "search_message_no_results") %><br/><br/> + <%= translate(locale, "search_message_change_filters_or_query") %><br/><br/> + <%= translate(locale, "search_message_use_another_instance", redirect_url) %> + </div> +</div> +<%- else -%> <div class="pure-g"> - <% videos.each do |item| %> + <%- videos.each do |item| -%> <%= rendered "components/item" %> - <% end %> + <%- end -%> </div> +<%- end -%> <div class="pure-g h-box"> <div class="pure-u-1 pure-u-lg-1-5"> - <% if page > 1 %> - <a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>"> - <%= translate(locale, "Previous page") %> - </a> - <% end %> + <%- if query.page > 1 -%> + <a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a> + <%- end -%> </div> <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if videos.size >= 20 %> - <a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>"> - <%= translate(locale, "Next page") %> - </a> - <% end %> + <%- if videos.size >= 20 -%> + <a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a> + <%- end -%> </div> </div> diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 5bbd9213..2678ac6c 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -14,6 +14,7 @@ module YoutubeAPI Android AndroidEmbeddedPlayer AndroidScreenEmbed + TvHtml5ScreenEmbed end # List of hard-coded values used by the different clients @@ -60,6 +61,12 @@ module YoutubeAPI api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", screen: "EMBED", }, + ClientType::TvHtml5ScreenEmbed => { + name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", + version: "2.0", + api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + screen: "EMBED", + }, } #################################################################### @@ -401,7 +408,7 @@ module YoutubeAPI client_config ||= DEFAULT_CLIENT_CONFIG # Query parameters - url = "#{endpoint}?key=#{client_config.api_key}" + url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false" headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", |