diff options
| author | Samantaz Fox <coding@samantaz.fr> | 2022-04-08 23:49:03 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-04-08 23:49:03 +0200 |
| commit | 4900ce24fac163d801a56af1fcf0f4c207448adf (patch) | |
| tree | dd135e34e9c0177d9813bd4e50fdf6053ec68fa4 /src | |
| parent | 91b079f4dfd78e25b6da238f872afe30ff9516bf (diff) | |
| parent | 62d7abdd9e699779a7e74ed5569aa6d631004210 (diff) | |
| download | invidious-4900ce24fac163d801a56af1fcf0f4c207448adf.tar.gz invidious-4900ce24fac163d801a56af1fcf0f4c207448adf.tar.bz2 invidious-4900ce24fac163d801a56af1fcf0f4c207448adf.zip | |
Merge pull request #2956 from SamantazFox/search-filters
Overhaul search filters
Diffstat (limited to 'src')
| -rw-r--r-- | src/invidious.cr | 1 | ||||
| -rw-r--r-- | src/invidious/exceptions.cr | 8 | ||||
| -rw-r--r-- | src/invidious/frontend/misc.cr | 14 | ||||
| -rw-r--r-- | src/invidious/frontend/search_filters.cr | 135 | ||||
| -rw-r--r-- | src/invidious/routes/api/v1/channels.cr | 16 | ||||
| -rw-r--r-- | src/invidious/routes/api/v1/search.cr | 24 | ||||
| -rw-r--r-- | src/invidious/routes/playlists.cr | 18 | ||||
| -rw-r--r-- | src/invidious/routes/search.cr | 24 | ||||
| -rw-r--r-- | src/invidious/search.cr | 254 | ||||
| -rw-r--r-- | src/invidious/search/ctoken.cr | 32 | ||||
| -rw-r--r-- | src/invidious/search/filters.cr | 376 | ||||
| -rw-r--r-- | src/invidious/search/processors.cr | 64 | ||||
| -rw-r--r-- | src/invidious/search/query.cr | 148 | ||||
| -rw-r--r-- | src/invidious/views/add_playlist_items.ecr | 11 | ||||
| -rw-r--r-- | src/invidious/views/search.ecr | 159 |
15 files changed, 850 insertions, 434 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index a470c6b6..9f3d5d10 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -35,6 +35,7 @@ require "./invidious/frontend/*" require "./invidious/*" require "./invidious/channels/*" require "./invidious/user/*" +require "./invidious/search/*" require "./invidious/routes/**" require "./invidious/jobs/**" diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr index 490d98cd..bfaa3fd5 100644 --- a/src/invidious/exceptions.cr +++ b/src/invidious/exceptions.cr @@ -1,3 +1,11 @@ +# Exception used to hold the bogus UCID during a channel search. +class ChannelSearchException < InfoException + getter channel : String + + def initialize(@channel) + end +end + # Exception used to hold the name of the missing item # Should be used in all parsing functions class BrokenTubeException < Exception diff --git a/src/invidious/frontend/misc.cr b/src/invidious/frontend/misc.cr new file mode 100644 index 00000000..43ba9f5c --- /dev/null +++ b/src/invidious/frontend/misc.cr @@ -0,0 +1,14 @@ +module Invidious::Frontend::Misc + extend self + + def redirect_url(env : HTTP::Server::Context) + prefs = env.get("preferences").as(Preferences) + + if prefs.automatic_instance_redirect + current_page = env.get?("current_page").as(String) + redirect_url = "/redirect?referer=#{current_page}" + else + redirect_url = "https://redirect.invidious.io#{env.request.resource}" + end + end +end diff --git a/src/invidious/frontend/search_filters.cr b/src/invidious/frontend/search_filters.cr new file mode 100644 index 00000000..68f27b4f --- /dev/null +++ b/src/invidious/frontend/search_filters.cr @@ -0,0 +1,135 @@ +module Invidious::Frontend::SearchFilters + extend self + + # Generate the search filters collapsable widget. + def generate(filters : Search::Filters, query : String, page : Int, locale : String) : String + return String.build(8000) do |str| + str << "<div id='filters'>\n" + str << "\t<details id='filters-collapse'>" + str << "\t\t<summary>" << translate(locale, "search_filters_title") << "</summary>\n" + + str << "\t\t<div id='filters-box'><form action='/search' method='get'>\n" + + str << "\t\t\t<input type='hidden' name='q' value='" << HTML.escape(query) << "'>\n" + str << "\t\t\t<input type='hidden' name='page' value='" << page << "'>\n" + + str << "\t\t\t<div id='filters-flex'>" + + filter_wrapper(date) + filter_wrapper(type) + filter_wrapper(duration) + filter_wrapper(features) + filter_wrapper(sort) + + str << "\t\t\t</div>\n" + + str << "\t\t\t<div id='filters-apply'>" + str << "<button type='submit' class=\"pure-button pure-button-primary\">" + str << translate(locale, "search_filters_apply_button") + str << "</button></div>\n" + + str << "\t\t</form></div>\n" + + str << "\t</details>\n" + str << "</div>\n" + end + end + + # Generate wrapper HTML (`<div>`, filter name, etc...) around the + # `<input>` elements of a search filter + macro filter_wrapper(name) + str << "\t\t\t\t<div class=\"filter-column\"><fieldset>\n" + + str << "\t\t\t\t\t<legend><div class=\"filter-name underlined\">" + str << translate(locale, "search_filters_{{name}}_label") + str << "</div></legend>\n" + + str << "\t\t\t\t\t<div class=\"filter-options\">\n" + make_{{name}}_filter_options(str, filters.{{name}}, locale) + str << "\t\t\t\t\t</div>" + + str << "\t\t\t\t</fieldset></div>\n" + end + + # Generates the HTML for the list of radio buttons of the "date" search filter + def make_date_filter_options(str : String::Builder, value : Search::Filters::Date, locale : String) + {% for value in Invidious::Search::Filters::Date.constants %} + {% date = value.underscore %} + + str << "\t\t\t\t\t\t<div>" + str << "<input type='radio' name='date' id='filter-date-{{date}}' value='{{date}}'" + str << " checked" if value.{{date}}? + str << '>' + + str << "<label for='filter-date-{{date}}'>" + str << translate(locale, "search_filters_date_option_{{date}}") + str << "</label></div>\n" + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "type" search filter + def make_type_filter_options(str : String::Builder, value : Search::Filters::Type, locale : String) + {% for value in Invidious::Search::Filters::Type.constants %} + {% type = value.underscore %} + + str << "\t\t\t\t\t\t<div>" + str << "<input type='radio' name='type' id='filter-type-{{type}}' value='{{type}}'" + str << " checked" if value.{{type}}? + str << '>' + + str << "<label for='filter-type-{{type}}'>" + str << translate(locale, "search_filters_type_option_{{type}}") + str << "</label></div>\n" + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "duration" search filter + def make_duration_filter_options(str : String::Builder, value : Search::Filters::Duration, locale : String) + {% for value in Invidious::Search::Filters::Duration.constants %} + {% duration = value.underscore %} + + str << "\t\t\t\t\t\t<div>" + str << "<input type='radio' name='duration' id='filter-duration-{{duration}}' value='{{duration}}'" + str << " checked" if value.{{duration}}? + str << '>' + + str << "<label for='filter-duration-{{duration}}'>" + str << translate(locale, "search_filters_duration_option_{{duration}}") + str << "</label></div>\n" + {% end %} + end + + # Generates the HTML for the list of checkboxes of the "features" search filter + def make_features_filter_options(str : String::Builder, value : Search::Filters::Features, locale : String) + {% for value in Invidious::Search::Filters::Features.constants %} + {% if value.stringify != "All" && value.stringify != "None" %} + {% feature = value.underscore %} + + str << "\t\t\t\t\t\t<div>" + str << "<input type='checkbox' name='features' id='filter-features-{{feature}}' value='{{feature}}'" + str << " checked" if value.{{feature}}? + str << '>' + + str << "<label for='filter-feature-{{feature}}'>" + str << translate(locale, "search_filters_features_option_{{feature}}") + str << "</label></div>\n" + {% end %} + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "sort" search filter + def make_sort_filter_options(str : String::Builder, value : Search::Filters::Sort, locale : String) + {% for value in Invidious::Search::Filters::Sort.constants %} + {% sort = value.underscore %} + + str << "\t\t\t\t\t\t<div>" + str << "<input type='radio' name='sort' id='filter-sort-{{sort}}' value='{{sort}}'" + str << " checked" if value.{{sort}}? + str << '>' + + str << "<label for='filter-sort-{{sort}}'>" + str << translate(locale, "search_filters_sort_option_{{sort}}") + str << "</label></div>\n" + {% end %} + end +end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index c4d6643a..8650976d 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -251,18 +251,22 @@ module Invidious::Routes::API::V1::Channels def self.search(env) locale = env.get("preferences").as(Preferences).locale + region = env.params.query["region"]? env.response.content_type = "application/json" - ucid = env.params.url["ucid"] + query = Invidious::Search::Query.new(env.params.query, :channel, region) - query = env.params.query["q"]? - query ||= "" + # Required because we can't (yet) pass multiple parameter to the + # `Search::Query` initializer (in this case, an URL segment) + query.channel = env.params.url["ucid"] - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + begin + search_results = query.process + rescue ex + return error_json(400, ex) + end - search_results = channel_search(query, page, ucid) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 5666460d..21451d33 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -5,34 +5,14 @@ module Invidious::Routes::API::V1::Search env.response.content_type = "application/json" - query = env.params.query["q"]? - query ||= "" - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "relevance" - - date = env.params.query["date"]?.try &.downcase - date ||= "" - - duration = env.params.query["duration"]?.try &.downcase - duration ||= "" - - features = env.params.query["features"]?.try &.split(",").map(&.downcase) - features ||= [] of String - - content_type = env.params.query["type"]?.try &.downcase - content_type ||= "video" + query = Invidious::Search::Query.new(env.params.query, :regular, region) begin - search_params = produce_search_params(page, sort_by, date, content_type, duration, features) + search_results = query.process rescue ex return error_json(400, ex) end - search_results = search(query, search_params, region) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index dbeb4f97..de981d81 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -212,7 +212,10 @@ module Invidious::Routes::Playlists end def self.add_playlist_items_page(env) - locale = env.get("preferences").as(Preferences).locale + prefs = env.get("preferences").as(Preferences) + locale = prefs.locale + + region = env.params.query["region"]? || prefs.region user = env.get? "user" sid = env.get? "sid" @@ -236,15 +239,10 @@ module Invidious::Routes::Playlists return env.redirect referer end - query = env.params.query["q"]? - if query - begin - search_query, items, operators = process_search_query(query, page, user, region: nil) - videos = items.select(SearchVideo).map(&.as(SearchVideo)) - rescue ex - videos = [] of SearchVideo - end - else + begin + query = Invidious::Search::Query.new(env.params.query, :playlist, region) + videos = query.process.select(SearchVideo).map(&.as(SearchVideo)) + rescue ex videos = [] of SearchVideo end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 3f4c7e5e..e60d0081 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -37,37 +37,29 @@ module Invidious::Routes::Search end def self.search(env) - locale = env.get("preferences").as(Preferences).locale - region = env.params.query["region"]? + prefs = env.get("preferences").as(Preferences) + locale = prefs.locale - query = env.params.query["search_query"]? - query ||= env.params.query["q"]? + region = env.params.query["region"]? || prefs.region + + query = Invidious::Search::Query.new(env.params.query, :regular, region) - if !query || query.empty? + if query.empty? # Display the full page search box implemented in #1977 env.set "search", "" templated "search_homepage", navbar_search: false else - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - user = env.get? "user" begin - search_query, videos, operators = process_search_query(query, page, user, region: region) + videos = query.process rescue ex : ChannelSearchException return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex return error_template(500, ex) end - operator_hash = {} of String => String - operators.each do |operator| - key, value = operator.downcase.split(":") - operator_hash[key] = value - end - - env.set "search", query + env.set "search", query.text templated "search" end end diff --git a/src/invidious/search.cr b/src/invidious/search.cr deleted file mode 100644 index ae106bf6..00000000 --- a/src/invidious/search.cr +++ /dev/null @@ -1,254 +0,0 @@ -class ChannelSearchException < InfoException - getter channel : String - - def initialize(@channel) - end -end - -def channel_search(query, page, channel) : Array(SearchItem) - response = YT_POOL.client &.get("/channel/#{channel}") - - if response.status_code == 404 - response = YT_POOL.client &.get("/user/#{channel}") - response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 - initial_data = extract_initial_data(response.body) - ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) - raise ChannelSearchException.new(channel) if !ucid - else - ucid = channel - end - - continuation = produce_channel_search_continuation(ucid, query, page) - response_json = YoutubeAPI.browse(continuation) - - continuation_items = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem if !continuation_items - - items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| - extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } - end - - return items -end - -def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem) - return [] of SearchItem if query.empty? - - client_config = YoutubeAPI::ClientConfig.new(region: region) - initial_data = YoutubeAPI.search(query, search_params, client_config: client_config) - - return extract_items(initial_data) -end - -def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "", - duration : String = "", features : Array(String) = [] of String) - object = { - "1:varint" => 0_i64, - "2:embedded" => {} of String => Int64, - "9:varint" => ((page - 1) * 20).to_i64, - } - - case sort - when "relevance" - object["1:varint"] = 0_i64 - when "rating" - object["1:varint"] = 1_i64 - when "upload_date", "date" - object["1:varint"] = 2_i64 - when "view_count", "views" - object["1:varint"] = 3_i64 - else - raise "No sort #{sort}" - end - - case date - when "hour" - object["2:embedded"].as(Hash)["1:varint"] = 1_i64 - when "today" - object["2:embedded"].as(Hash)["1:varint"] = 2_i64 - when "week" - object["2:embedded"].as(Hash)["1:varint"] = 3_i64 - when "month" - object["2:embedded"].as(Hash)["1:varint"] = 4_i64 - when "year" - object["2:embedded"].as(Hash)["1:varint"] = 5_i64 - else nil # Ignore - end - - case content_type - when "video" - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - when "channel" - object["2:embedded"].as(Hash)["2:varint"] = 2_i64 - when "playlist" - object["2:embedded"].as(Hash)["2:varint"] = 3_i64 - when "movie" - object["2:embedded"].as(Hash)["2:varint"] = 4_i64 - when "show" - object["2:embedded"].as(Hash)["2:varint"] = 5_i64 - when "all" - # - else - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - end - - case duration - when "short" - object["2:embedded"].as(Hash)["3:varint"] = 1_i64 - when "long" - object["2:embedded"].as(Hash)["3:varint"] = 2_i64 - else nil # Ignore - end - - features.each do |feature| - case feature - when "hd" - object["2:embedded"].as(Hash)["4:varint"] = 1_i64 - when "subtitles" - object["2:embedded"].as(Hash)["5:varint"] = 1_i64 - when "creative_commons", "cc" - object["2:embedded"].as(Hash)["6:varint"] = 1_i64 - when "3d" - object["2:embedded"].as(Hash)["7:varint"] = 1_i64 - when "live", "livestream" - object["2:embedded"].as(Hash)["8:varint"] = 1_i64 - when "purchased" - object["2:embedded"].as(Hash)["9:varint"] = 1_i64 - when "4k" - object["2:embedded"].as(Hash)["14:varint"] = 1_i64 - when "360" - object["2:embedded"].as(Hash)["15:varint"] = 1_i64 - when "location" - object["2:embedded"].as(Hash)["23:varint"] = 1_i64 - when "hdr" - object["2:embedded"].as(Hash)["25:varint"] = 1_i64 - else nil # Ignore - end - end - - if object["2:embedded"].as(Hash).empty? - object.delete("2:embedded") - end - - params = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return params -end - -def produce_channel_search_continuation(ucid, query, page) - if page <= 1 - idx = 0_i64 - else - idx = 30_i64 * (page - 1) - end - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "search", - "6:varint" => 1_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "15:base64" => { - "3:varint" => idx, - }, - "23:varint" => 0_i64, - }, - "11:string" => query, - "35:string" => "browse-feed#{ucid}search", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def process_search_query(query, page, user, region) - if user - user = user.as(Invidious::User) - view_name = "subscriptions_#{sha256(user.email)}" - end - - channel = nil - content_type = "all" - date = "" - duration = "" - features = [] of String - sort = "relevance" - subscriptions = nil - - operators = query.split(" ").select(&.match(/\w+:[\w,]+/)) - operators.each do |operator| - key, value = operator.downcase.split(":") - - case key - when "channel", "user" - channel = operator.split(":")[-1] - when "content_type", "type" - content_type = value - when "date" - date = value - when "duration" - duration = value - when "feature", "features" - features = value.split(",") - when "sort" - sort = value - when "subscriptions" - subscriptions = value == "true" - else - operators.delete(operator) - end - end - - search_query = (query.split(" ") - operators).join(" ") - - if channel - items = channel_search(search_query, page, channel) - elsif subscriptions - if view_name - items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM ( - SELECT *, - to_tsvector(#{view_name}.title) || - to_tsvector(#{view_name}.author) - as document - FROM #{view_name} - ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo) - else - items = [] of ChannelVideo - end - else - search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, - duration: duration, features: features) - - items = search(search_query, search_params, region) - end - - # Light processing to flatten search results out of Categories. - # They should ideally be supported in the future. - items_without_category = [] of SearchItem | ChannelVideo - items.each do |i| - if i.is_a? Category - i.contents.each do |nest_i| - if !nest_i.is_a? Video - items_without_category << nest_i - end - end - else - items_without_category << i - end - end - - {search_query, items_without_category, operators} -end diff --git a/src/invidious/search/ctoken.cr b/src/invidious/search/ctoken.cr new file mode 100644 index 00000000..161065e0 --- /dev/null +++ b/src/invidious/search/ctoken.cr @@ -0,0 +1,32 @@ +def produce_channel_search_continuation(ucid, query, page) + if page <= 1 + idx = 0_i64 + else + idx = 30_i64 * (page - 1) + end + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:base64" => { + "2:string" => "search", + "6:varint" => 1_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "15:base64" => { + "3:varint" => idx, + }, + "23:varint" => 0_i64, + }, + "11:string" => query, + "35:string" => "browse-feed#{ucid}search", + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation +end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr new file mode 100644 index 00000000..c2b5c758 --- /dev/null +++ b/src/invidious/search/filters.cr @@ -0,0 +1,376 @@ +require "protodec/utils" +require "http/params" + +module Invidious::Search + struct Filters + # Values correspond to { "2:embedded": { "1:varint": <X> }} + # except for "None" which is only used by us (= nothing selected) + enum Date + None = 0 + Hour = 1 + Today = 2 + Week = 3 + Month = 4 + Year = 5 + end + + # Values correspond to { "2:embedded": { "2:varint": <X> }} + # except for "All" which is only used by us (= nothing selected) + enum Type + All = 0 + Video = 1 + Channel = 2 + Playlist = 3 + Movie = 4 + + # Has it been removed? + # (Not available on youtube's UI) + Show = 5 + end + + # Values correspond to { "2:embedded": { "3:varint": <X> }} + # except for "None" which is only used by us (= nothing selected) + enum Duration + None = 0 + Short = 1 # "Under 4 minutes" + Long = 2 # "Over 20 minutes" + Medium = 3 # "4 - 20 minutes" + end + + # Note: flag enums automatically generate + # "none" and "all" members + @[Flags] + enum Features + Live + FourK # "4K" + HD + Subtitles # "Subtitles/CC" + CCommons # "Creative Commons" + ThreeSixty # "360°" + VR180 + ThreeD # "3D" + HDR + Location + Purchased + end + + # Values correspond to { "1:varint": <X> } + enum Sort + Relevance = 0 + Rating = 1 + Date = 2 + Views = 3 + end + + # Parameters are sorted as on Youtube + property date : Date + property type : Type + property duration : Duration + property features : Features + property sort : Sort + + def initialize( + *, # All parameters must be named + @date : Date = Date::None, + @type : Type = Type::All, + @duration : Duration = Duration::None, + @features : Features = Features::None, + @sort : Sort = Sort::Relevance + ) + end + + def default? : Bool + return @date.none? && @type.all? && @duration.none? && \ + @features.none? && @sort.relevance? + end + + # ------------------- + # Invidious params + # ------------------- + + def self.parse_features(raw : Array(String)) : Features + # Initialize return variable + features = Features.new(0) + + raw.each do |ft| + case ft.downcase + when "live", "livestream" + features = features | Features::Live + when "4k" then features = features | Features::FourK + when "hd" then features = features | Features::HD + when "subtitles" then features = features | Features::Subtitles + when "creative_commons", "commons", "cc" + features = features | Features::CCommons + when "360" then features = features | Features::ThreeSixty + when "vr180" then features = features | Features::VR180 + when "3d" then features = features | Features::ThreeD + when "hdr" then features = features | Features::HDR + when "location" then features = features | Features::Location + when "purchased" then features = features | Features::Purchased + end + end + + return features + end + + def self.format_features(features : Features) : String + # Directly return an empty string if there are no features + return "" if features.none? + + # Initialize return variable + str = [] of String + + str << "live" if features.live? + str << "4k" if features.four_k? + str << "hd" if features.hd? + str << "subtitles" if features.subtitles? + str << "commons" if features.c_commons? + str << "360" if features.three_sixty? + str << "vr180" if features.vr180? + str << "3d" if features.three_d? + str << "hdr" if features.hdr? + str << "location" if features.location? + str << "purchased" if features.purchased? + + return str.join(',') + end + + def self.from_legacy_filters(str : String) : {Filters, String, String, Bool} + # Split search query on spaces + members = str.split(' ') + + # Output variables + channel = "" + filters = Filters.new + subscriptions = false + + # Array to hold the non-filter members + query = [] of String + + # Parse! + members.each do |substr| + # Separator operators + operators = substr.split(':') + + case operators[0] + when "user", "channel" + next if operators.size != 2 + channel = operators[1] + # + when "type", "content_type" + next if operators.size != 2 + type = Type.parse?(operators[1]) + filters.type = type if !type.nil? + # + when "date" + next if operators.size != 2 + date = Date.parse?(operators[1]) + filters.date = date if !date.nil? + # + when "duration" + next if operators.size != 2 + duration = Duration.parse?(operators[1]) + filters.duration = duration if !duration.nil? + # + when "feature", "features" + next if operators.size != 2 + features = parse_features(operators[1].split(',')) + filters.features = features if !features.nil? + # + when "sort" + next if operators.size != 2 + sort = Sort.parse?(operators[1]) + filters.sort = sort if !sort.nil? + # + when "subscriptions" + next if operators.size != 2 + subscriptions = {"true", "on", "yes", "1"}.any?(&.== operators[1]) + # + else + query << substr + end + end + + # Re-assemble query (without filters) + cleaned_query = query.join(' ') + + return {filters, channel, cleaned_query, subscriptions} + end + + def self.from_iv_params(params : HTTP::Params) : Filters + # Temporary variables + filters = Filters.new + + if type = params["type"]? + filters.type = Type.parse?(type) || Type::All + params.delete("type") + end + + if date = params["date"]? + filters.date = Date.parse?(date) || Date::None + params.delete("date") + end + + if duration = params["duration"]? + filters.duration = Duration.parse?(duration) || Duration::None + params.delete("duration") + end + + features = params.fetch_all("features") + if !features.empty? + # Un-array input so it can be treated as a comma-separated list + features = features[0].split(',') if features.size == 1 + + filters.features = parse_features(features) || Features::None + params.delete_all("features") + end + + if sort = params["sort"]? + filters.sort = Sort.parse?(sort) || Sort::Relevance + params.delete("sort") + end + + return filters + end + + def to_iv_params : HTTP::Params + # Temporary variables + raw_params = {} of String => Array(String) + + raw_params["date"] = [@date.to_s.underscore] if !@date.none? + raw_params["type"] = [@type.to_s.underscore] if !@type.all? + raw_params["sort"] = [@sort.to_s.underscore] if !@sort.relevance? + + if !@duration.none? + raw_params["duration"] = [@duration.to_s.underscore] + end + + if !@features.none? + raw_params["features"] = [Filters.format_features(@features)] + end + + return HTTP::Params.new(raw_params) + end + + # ------------------- + # Youtube params + # ------------------- + + # Produce the youtube search parameters for the + # innertube API (base64-encoded protobuf object). + def to_yt_params(page : Int = 1) : String + # Initialize the embedded protobuf object + embedded = {} of String => Int64 + + # Add these field only if associated parameter is selected + embedded["1:varint"] = @date.to_i64 if !@date.none? + embedded["2:varint"] = @type.to_i64 if !@type.all? + embedded["3:varint"] = @duration.to_i64 if !@duration.none? + + if !@features.none? + # All features have a value of "1" when enabled, and + # the field is omitted when the feature is no selected. + embedded["4:varint"] = 1_i64 if @features.includes?(Features::HD) + embedded["5:varint"] = 1_i64 if @features.includes?(Features::Subtitles) + embedded["6:varint"] = 1_i64 if @features.includes?(Features::CCommons) + embedded["7:varint"] = 1_i64 if @features.includes?(Features::ThreeD) + embedded["8:varint"] = 1_i64 if @features.includes?(Features::Live) + embedded["9:varint"] = 1_i64 if @features.includes?(Features::Purchased) + embedded["14:varint"] = 1_i64 if @features.includes?(Features::FourK) + embedded["15:varint"] = 1_i64 if @features.includes?(Features::ThreeSixty) + embedded["23:varint"] = 1_i64 if @features.includes?(Features::Location) + embedded["25:varint"] = 1_i64 if @features.includes?(Features::HDR) + embedded["26:varint"] = 1_i64 if @features.includes?(Features::VR180) + end + + # Initialize an empty protobuf object + object = {} of String => (Int64 | String | Hash(String, Int64)) + + # As usual, everything can be omitted if it has no value + object["2:embedded"] = embedded if !embedded.empty? + + # Default sort is "relevance", so when this option is selected, + # the associated field can be omitted. + if !@sort.relevance? + object["1:varint"] = @sort.to_i64 + end + + # Add page number (if provided) + if page > 1 + object["9:varint"] = ((page - 1) * 20).to_i64 + end + + # If the object is empty, return an empty string, + # otherwise encode to protobuf then to base64 + return "" if object.empty? + + return object + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + end + + # Function to parse the `sp` URL parameter from Youtube + # search page. It's a base64-encoded protobuf object. + def self.from_yt_params(params : HTTP::Params) : Filters + # Initialize output variable + filters = Filters.new + + # Get parameter, and check emptyness + search_params = params["sp"]? + + if search_params.nil? || search_params.empty? + return filters + end + + # Decode protobuf object + object = search_params + .try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + # Parse items from embedded object + if embedded = object["2:0:embedded"]? + # All the following fields (date, type, duration) are optional. + if date = embedded["1:0:varint"]? + filters.date = Date.from_value?(date.as_i) || Date::None + end + + if type = embedded["2:0:varint"]? + filters.type = Type.from_value?(type.as_i) || Type::All + end + + if duration = embedded["3:0:varint"]? + filters.duration = Duration.from_value?(duration.as_i) || Duration::None + end + + # All features should have a value of "1" when enabled, and + # the field should be omitted when the feature is no selected. + features = 0 + features += (embedded["4:0:varint"]?.try &.as_i == 1_i64) ? Features::HD.value : 0 + features += (embedded["5:0:varint"]?.try &.as_i == 1_i64) ? Features::Subtitles.value : 0 + features += (embedded["6:0:varint"]?.try &.as_i == 1_i64) ? Features::CCommons.value : 0 + features += (embedded["7:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeD.value : 0 + features += (embedded["8:0:varint"]?.try &.as_i == 1_i64) ? Features::Live.value : 0 + features += (embedded["9:0:varint"]?.try &.as_i == 1_i64) ? Features::Purchased.value : 0 + features += (embedded["14:0:varint"]?.try &.as_i == 1_i64) ? Features::FourK.value : 0 + features += (embedded["15:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeSixty.value : 0 + features += (embedded["23:0:varint"]?.try &.as_i == 1_i64) ? Features::Location.value : 0 + features += (embedded["25:0:varint"]?.try &.as_i == 1_i64) ? Features::HDR.value : 0 + features += (embedded["26:0:varint"]?.try &.as_i == 1_i64) ? Features::VR180.value : 0 + + filters.features = Features.from_value?(features) || Features::None + end + + if sort = object["1:0:varint"]? + filters.sort = Sort.from_value?(sort.as_i) || Sort::Relevance + end + + # Remove URL parameter and return result + params.delete("sp") + return filters + end + end +end diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr new file mode 100644 index 00000000..d1409c06 --- /dev/null +++ b/src/invidious/search/processors.cr @@ -0,0 +1,64 @@ +module Invidious::Search + module Processors + extend self + + # Regular search (`/search` endpoint) + def regular(query : Query) : Array(SearchItem) + search_params = query.filters.to_yt_params(page: query.page) + + client_config = YoutubeAPI::ClientConfig.new(region: query.region) + initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) + + return extract_items(initial_data) + end + + # Search a youtube channel + # TODO: clean code, and rely more on YoutubeAPI + def channel(query : Query) : Array(SearchItem) + response = YT_POOL.client &.get("/channel/#{query.channel}") + + if response.status_code == 404 + response = YT_POOL.client &.get("/user/#{query.channel}") + response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404 + initial_data = extract_initial_data(response.body) + ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) + raise ChannelSearchException.new(query.channel) if !ucid + else + ucid = query.channel + end + + continuation = produce_channel_search_continuation(ucid, query.text, query.page) + response_json = YoutubeAPI.browse(continuation) + + continuation_items = response_json["onResponseReceivedActions"]? + .try &.[0]["appendContinuationItemsAction"]["continuationItems"] + + return [] of SearchItem if !continuation_items + + items = [] of SearchItem + continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| + extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } + end + + return items + end + + # Search inside of user subscriptions + def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo) + view_name = "subscriptions_#{sha256(user.email)}" + + return PG_DB.query_all(" + SELECT id,title,published,updated,ucid,author,length_seconds + FROM ( + SELECT *, + to_tsvector(#{view_name}.title) || + to_tsvector(#{view_name}.author) + as document + FROM #{view_name} + ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", + query.text, (query.page - 1) * 20, + as: ChannelVideo + ) + end + end +end diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr new file mode 100644 index 00000000..1c2b37d2 --- /dev/null +++ b/src/invidious/search/query.cr @@ -0,0 +1,148 @@ +module Invidious::Search + class Query + enum Type + # Types related to YouTube + Regular # Youtube search page + Channel # Youtube channel search box + + # Types specific to Invidious + Subscriptions # Search user subscriptions + Playlist # "Add playlist item" search + end + + @type : Type = Type::Regular + + @raw_query : String + @query : String = "" + + property filters : Filters = Filters.new + property page : Int32 + property region : String? + property channel : String = "" + + # Return true if @raw_query is either `nil` or empty + private def empty_raw_query? + return @raw_query.empty? + end + + # Same as `empty_raw_query?`, but named for external use + def empty? + return self.empty_raw_query? + end + + # Getter for the query string. + # It is named `text` to reduce confusion (`search_query.text` makes more + # sense than `search_query.query`) + def text + return @query + end + + # Initialize a new search query. + # Parameters are used to get the query string, the page number + # and the search filters (if any). Type tells this function + # where it is being called from (See `Type` above). + def initialize( + params : HTTP::Params, + @type : Type = Type::Regular, + @region : String? = nil + ) + # Get the raw search query string (common to all search types). In + # Regular search mode, also look for the `search_query` URL parameter + if @type.regular? + @raw_query = params["q"]? || params["search_query"]? || "" + else + @raw_query = params["q"]? || "" + end + + # Get the page number (also common to all search types) + @page = params["page"]?.try &.to_i? || 1 + + # Stop here is raw query in empty + # NOTE: maybe raise in the future? + return if self.empty_raw_query? + + # Specific handling + case @type + when .playlist?, .channel? + # In "add playlist item" mode, filters are parsed from the query + # string itself (legacy), and the channel is ignored. + # + # In "channel search" mode, filters are ignored, but we still parse + # the query prevent transmission of legacy filters to youtube. + # + @filters, @query, @channel, _ = Filters.from_legacy_filters(@raw_query || "") + # + when .subscriptions?, .regular? + if params["sp"]? + # Parse the `sp` URL parameter (youtube compatibility) + @filters = Filters.from_yt_params(params) + @query = @raw_query || "" + else + # Parse invidious URL parameters (sort, date, etc...) + @filters = Filters.from_iv_params(params) + @channel = params["channel"]? || "" + + if @filters.default? && @raw_query.includes?(':') + # Parse legacy filters from query + @filters, @query, @channel, subs = Filters.from_legacy_filters(@raw_query || "") + else + @query = @raw_query || "" + end + + if !@channel.empty? + # Switch to channel search mode (filters will be ignored) + @type = Type::Channel + elsif subs + # Switch to subscriptions search mode + @type = Type::Subscriptions + end + end + end + end + + # Run the search query using the corresponding search processor. + # Returns either the results or an empty array of `SearchItem`. + def process(user : Invidious::User? = nil) : Array(SearchItem) | Array(ChannelVideo) + items = [] of SearchItem + + # Don't bother going further if search query is empty + return items if self.empty_raw_query? + + case @type + when .regular?, .playlist? + items = unnest_items(Processors.regular(self)) + # + when .channel? + items = Processors.channel(self) + # + when .subscriptions? + if user + items = Processors.subscriptions(self, user.as(Invidious::User)) + end + end + + return items + end + + # TODO: clean code + private def unnest_items(all_items) : Array(SearchItem) + items = [] of SearchItem + + # Light processing to flatten search results out of Categories. + # They should ideally be supported in the future. + all_items.each do |i| + if i.is_a? Category + i.contents.each do |nest_i| + if !nest_i.is_a? Video + items << nest_i + end + end + else + items << i + end + end + + return items + end + end +end diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr index ad50909a..22870317 100644 --- a/src/invidious/views/add_playlist_items.ecr +++ b/src/invidious/views/add_playlist_items.ecr @@ -11,7 +11,9 @@ <legend><a href="/playlist?list=<%= playlist.id %>"><%= translate(locale, "Editing playlist `x`", %|"#{HTML.escape(playlist.title)}"|) %></a></legend> <fieldset> - <input class="pure-input-1" type="search" name="q" <% if query %>value="<%= HTML.escape(query) %>"<% else %>placeholder="<%= translate(locale, "Search for videos") %>"<% end %>> + <input class="pure-input-1" type="search" name="q" + <% if query %>value="<%= HTML.escape(query.text) %>"<% end %> + placeholder="<%= translate(locale, "Search for videos") %>"> <input type="hidden" name="list" value="<%= plid %>"> </fieldset> </form> @@ -38,10 +40,11 @@ </div> <% if query %> + <%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%> <div class="pure-g h-box"> <div class="pure-u-1 pure-u-lg-1-5"> - <% if page > 1 %> - <a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page - 1 %>"> + <% if query.page > 1 %> + <a href="/add_playlist_items?list=<%= plid %>&q=<%= query_encoded %>&page=<%= page - 1 %>"> <%= translate(locale, "Previous page") %> </a> <% end %> @@ -49,7 +52,7 @@ <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> <% if videos.size >= 20 %> - <a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page + 1 %>"> + <a href="/add_playlist_items?list=<%= plid %>&q=<%= query_encoded %>&page=<%= page + 1 %>"> <%= translate(locale, "Next page") %> </a> <% end %> diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index 45bbdefc..7110703e 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -1,147 +1,62 @@ <% content_for "header" do %> -<title><%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious</title> +<title><%= query.text.size > 30 ? HTML.escape(query.text[0,30].rstrip(".")) + "…" : HTML.escape(query.text) %> - Invidious</title> +<link rel="stylesheet" href="/css/search.css?v=<%= ASSET_COMMIT %>"> <% end %> -<% search_query_encoded = env.get?("search").try { |x| URI.encode_www_form(x.as(String), space_to_plus: true) } %> +<%- + search_query_encoded = URI.encode_www_form(query.text, space_to_plus: true) + filter_params = query.filters.to_iv_params -<!-- Search redirection and filtering UI --> -<% if videos.size == 0 %> - <h3 style="text-align: center"> - <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Broken? Try another Invidious Instance!") %></a> - </h3> -<% else %> - <details id="filters"> - <summary> - <h3 style="display:inline"> <%= translate(locale, "filter") %> </h3> - </summary> - <div id="filters" class="pure-g h-box"> - <div class="pure-u-1-3 pure-u-md-1-5"> - <b><%= translate(locale, "date") %></b> - <hr/> - <% ["hour", "today", "week", "month", "year"].each do |date| %> - <div class="pure-u-1 pure-md-1-5"> - <% if operator_hash.fetch("date", "all") == date %> - <b><%= translate(locale, date) %></b> - <% else %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?date:[a-z]+/, "") + " date:" + date) %>&page=<%= page %>"> - <%= translate(locale, date) %> - </a> - <% end %> - </div> - <% end %> - </div> - <div class="pure-u-1-3 pure-u-md-1-5"> - <b><%= translate(locale, "content_type") %></b> - <hr/> - <% ["video", "channel", "playlist", "movie", "show"].each do |content_type| %> - <div class="pure-u-1 pure-md-1-5"> - <% if operator_hash.fetch("content_type", "all") == content_type %> - <b><%= translate(locale, content_type) %></b> - <% else %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?content_type:[a-z]+/, "") + " content_type:" + content_type) %>&page=<%= page %>"> - <%= translate(locale, content_type) %> - </a> - <% end %> - </div> - <% end %> - </div> - <div class="pure-u-1-3 pure-u-md-1-5"> - <b><%= translate(locale, "duration") %></b> - <hr/> - <% ["short", "long"].each do |duration| %> - <div class="pure-u-1 pure-md-1-5"> - <% if operator_hash.fetch("duration", "all") == duration %> - <b><%= translate(locale, duration) %></b> - <% else %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?duration:[a-z]+/, "") + " duration:" + duration) %>&page=<%= page %>"> - <%= translate(locale, duration) %> - </a> - <% end %> - </div> - <% end %> - </div> - <div class="pure-u-1-3 pure-u-md-1-5"> - <b><%= translate(locale, "features") %></b> - <hr/> - <% ["hd", "subtitles", "creative_commons", "3d", "live", "purchased", "4k", "360", "location", "hdr"].each do |feature| %> - <div class="pure-u-1 pure-md-1-5"> - <% if operator_hash.fetch("features", "all").includes?(feature) %> - <b><%= translate(locale, feature) %></b> - <% elsif operator_hash.has_key?("features") %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/features:/, "features:" + feature + ",")) %>&page=<%= page %>"> - <%= translate(locale, feature) %> - </a> - <% else %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil! + " features:" + feature) %>&page=<%= page %>"> - <%= translate(locale, feature) %> - </a> - <% end %> - </div> - <% end %> - </div> - <div class="pure-u-1-3 pure-u-md-1-5"> - <b><%= translate(locale, "sort") %></b> - <hr/> - <% ["relevance", "rating", "date", "views"].each do |sort| %> - <div class="pure-u-1 pure-md-1-5"> - <% if operator_hash.fetch("sort", "relevance") == sort %> - <b><%= translate(locale, sort) %></b> - <% else %> - <a href="/search?q=<%= URI.encode_www_form(query.not_nil!.gsub(/ ?sort:[a-z]+/, "") + " sort:" + sort) %>&page=<%= page %>"> - <%= translate(locale, sort) %> - </a> - <% end %> - </div> - <% end %> - </div> - </div> - </details> -<% end %> + url_prev_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page - 1}" + url_next_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page + 1}" -<% if videos.size == 0 %> - <hr style="margin: 0;"/> -<% else %> - <hr/> -<% end %> + redirect_url = Invidious::Frontend::Misc.redirect_url(env) +-%> + +<!-- Search redirection and filtering UI --> +<%= Invidious::Frontend::SearchFilters.generate(query.filters, query.text, query.page, locale) %> +<hr/> <div class="pure-g h-box v-box"> <div class="pure-u-1 pure-u-lg-1-5"> - <% if page > 1 %> - <a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>"> - <%= translate(locale, "Previous page") %> - </a> - <% end %> + <%- if query.page > 1 -%> + <a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a> + <%- end -%> </div> <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if videos.size >= 20 %> - <a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>"> - <%= translate(locale, "Next page") %> - </a> - <% end %> + <%- if videos.size >= 20 -%> + <a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a> + <%- end -%> </div> </div> +<%- if videos.empty? -%> +<div class="h-box no-results-error"> + <div> + <%= translate(locale, "search_message_no_results") %><br/><br/> + <%= translate(locale, "search_message_change_filters_or_query") %><br/><br/> + <%= translate(locale, "search_message_use_another_instance", redirect_url) %> + </div> +</div> +<%- else -%> <div class="pure-g"> - <% videos.each do |item| %> + <%- videos.each do |item| -%> <%= rendered "components/item" %> - <% end %> + <%- end -%> </div> +<%- end -%> <div class="pure-g h-box"> <div class="pure-u-1 pure-u-lg-1-5"> - <% if page > 1 %> - <a href="/search?q=<%= search_query_encoded %>&page=<%= page - 1 %>"> - <%= translate(locale, "Previous page") %> - </a> - <% end %> + <%- if query.page > 1 -%> + <a href="<%= url_prev_page %>"><%= translate(locale, "Previous page") %></a> + <%- end -%> </div> <div class="pure-u-1 pure-u-lg-3-5"></div> <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right"> - <% if videos.size >= 20 %> - <a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>"> - <%= translate(locale, "Next page") %> - </a> - <% end %> + <%- if videos.size >= 20 -%> + <a href="<%= url_next_page %>"><%= translate(locale, "Next page") %></a> + <%- end -%> </div> </div> |
