From f9b8bc006f0375d4f7d24f2e671d0f8ab38059dd Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 7 Mar 2022 00:52:54 +0100 Subject: Create a search processors module --- src/invidious.cr | 1 + src/invidious/routes/api/v1/channels.cr | 2 +- src/invidious/search.cr | 47 +++------------------------- src/invidious/search/processors.cr | 54 +++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 44 deletions(-) create mode 100644 src/invidious/search/processors.cr (limited to 'src') diff --git a/src/invidious.cr b/src/invidious.cr index a470c6b6..9f3d5d10 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -35,6 +35,7 @@ require "./invidious/frontend/*" require "./invidious/*" require "./invidious/channels/*" require "./invidious/user/*" +require "./invidious/search/*" require "./invidious/routes/**" require "./invidious/jobs/**" diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index c4d6643a..c4395353 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -262,7 +262,7 @@ module Invidious::Routes::API::V1::Channels page = env.params.query["page"]?.try &.to_i? page ||= 1 - search_results = channel_search(query, page, ucid) + search_results = Invidious::Search::Processors.channel(query, page, ucid) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/search.cr b/src/invidious/search.cr index ae106bf6..af854653 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -5,35 +5,6 @@ class ChannelSearchException < InfoException end end -def channel_search(query, page, channel) : Array(SearchItem) - response = YT_POOL.client &.get("/channel/#{channel}") - - if response.status_code == 404 - response = YT_POOL.client &.get("/user/#{channel}") - response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 - initial_data = extract_initial_data(response.body) - ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) - raise ChannelSearchException.new(channel) if !ucid - else - ucid = channel - end - - continuation = produce_channel_search_continuation(ucid, query, page) - response_json = YoutubeAPI.browse(continuation) - - continuation_items = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem if !continuation_items - - items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| - extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } - end - - return items -end - def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem) return [] of SearchItem if query.empty? @@ -175,11 +146,6 @@ def produce_channel_search_continuation(ucid, query, page) end def process_search_query(query, page, user, region) - if user - user = user.as(Invidious::User) - view_name = "subscriptions_#{sha256(user.email)}" - end - channel = nil content_type = "all" date = "" @@ -215,16 +181,11 @@ def process_search_query(query, page, user, region) search_query = (query.split(" ") - operators).join(" ") if channel - items = channel_search(search_query, page, channel) + items = Invidious::Search::Processors.channel(search_query, page, channel) elsif subscriptions - if view_name - items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM ( - SELECT *, - to_tsvector(#{view_name}.title) || - to_tsvector(#{view_name}.author) - as document - FROM #{view_name} - ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo) + if user + user = user.as(Invidious::User) + items = Invidious::Search::Processors.subscriptions(query, page, user) else items = [] of ChannelVideo end diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr new file mode 100644 index 00000000..c5327f34 --- /dev/null +++ b/src/invidious/search/processors.cr @@ -0,0 +1,54 @@ +module Invidious::Search + module Processors + extend self + + # Search a youtube channel + # TODO: clean code, and rely more on YoutubeAPI + def channel(query, page, channel) : Array(SearchItem) + response = YT_POOL.client &.get("/channel/#{channel}") + + if response.status_code == 404 + response = YT_POOL.client &.get("/user/#{channel}") + response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 + initial_data = extract_initial_data(response.body) + ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) + raise ChannelSearchException.new(channel) if !ucid + else + ucid = channel + end + + continuation = produce_channel_search_continuation(ucid, query, page) + response_json = YoutubeAPI.browse(continuation) + + continuation_items = response_json["onResponseReceivedActions"]? + .try &.[0]["appendContinuationItemsAction"]["continuationItems"] + + return [] of SearchItem if !continuation_items + + items = [] of SearchItem + continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| + extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } + end + + return items + end + + # Search inside of user subscriptions + def subscriptions(query, page, user : Invidious::User) : Array(ChannelVideo) + view_name = "subscriptions_#{sha256(user.email)}" + + return PG_DB.query_all(" + SELECT id,title,published,updated,ucid,author,length_seconds + FROM ( + SELECT *, + to_tsvector(#{view_name}.title) || + to_tsvector(#{view_name}.author) + as document + FROM #{view_name} + ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", + query, (page - 1) * 20, + as: ChannelVideo + ) + end + end +end -- cgit v1.2.3 From 80417281c437f10dc6653648d6def00c8ba167d6 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 3 Mar 2022 22:32:34 +0100 Subject: Add a struct for search filters --- src/invidious/search/filters.cr | 79 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 src/invidious/search/filters.cr (limited to 'src') diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr new file mode 100644 index 00000000..75ac287a --- /dev/null +++ b/src/invidious/search/filters.cr @@ -0,0 +1,79 @@ +module Invidious::Search + struct Filters + # Values correspond to { "2:embedded": { "1:varint": }} + # except for "None" which is only used by us (= nothing selected) + enum Date + None = 0 + Hour = 1 + Today = 2 + Week = 3 + Month = 4 + Year = 5 + end + + # Values correspond to { "2:embedded": { "2:varint": }} + # except for "All" which is only used by us (= nothing selected) + enum Type + All = 0 + Video = 1 + Channel = 2 + Playlist = 3 + Movie = 4 + + # Has it been removed? + # (Not available on youtube's UI) + Show = 5 + end + + # Values correspond to { "2:embedded": { "3:varint": }} + # except for "None" which is only used by us (= nothing selected) + enum Duration + None = 0 + Short = 1 # "Under 4 minutes" + Long = 2 # "Over 20 minutes" + Medium = 3 # "4 - 20 minutes" + end + + # Note: flag enums automatically generate + # "none" and "all" members + @[Flags] + enum Features + Live + FourK # "4K" + HD + Subtitles # "Subtitles/CC" + CCommons # "Creative Commons" + ThreeSixty # "360°" + VR180 + ThreeD # "3D" + HDR + Location + Purchased + end + + # Values correspond to { "1:varint": } + enum Sort + Relevance = 0 + Rating = 1 + Date = 2 + Views = 3 + end + + # Parameters are sorted as on Youtube + property date : Date + property type : Type + property duration : Duration + property features : Features + property sort : Sort + + def initialize( + *, # All parameters must be named + @date : Date = Date::None, + @type : Type = Type::All, + @duration : Duration = Duration::None, + @features : Features = Features::None, + @sort : Sort = Sort::Relevance + ) + end + end +end -- cgit v1.2.3 From c01a29fe76c78d403e80d1e9000046c45ac97a72 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 3 Mar 2022 22:37:02 +0100 Subject: Add a function to build youtube search filters (it aims at replacing produce_search_params) --- spec/invidious/helpers_spec.cr | 14 ----- spec/invidious/search/yt_filters_spec.cr | 92 ++++++++++++++++++++++++++++++++ src/invidious/search/filters.cr | 60 +++++++++++++++++++++ 3 files changed, 152 insertions(+), 14 deletions(-) create mode 100644 spec/invidious/search/yt_filters_spec.cr (limited to 'src') diff --git a/spec/invidious/helpers_spec.cr b/spec/invidious/helpers_spec.cr index b2436989..5ecebef3 100644 --- a/spec/invidious/helpers_spec.cr +++ b/spec/invidious/helpers_spec.cr @@ -29,20 +29,6 @@ Spectator.describe "Helper" do end end - describe "#produce_search_params" do - it "correctly produces token for searching with specified filters" do - expect(produce_search_params).to eq("CAASAhABSAA%3D") - - expect(produce_search_params(sort: "upload_date", content_type: "video")).to eq("CAISAhABSAA%3D") - - expect(produce_search_params(content_type: "playlist")).to eq("CAASAhADSAA%3D") - - expect(produce_search_params(sort: "date", content_type: "video", features: ["hd", "cc", "purchased", "hdr"])).to eq("CAISCxABIAEwAUgByAEBSAA%3D") - - expect(produce_search_params(content_type: "channel")).to eq("CAASAhACSAA%3D") - end - end - describe "#produce_comment_continuation" do it "correctly produces a continuation token for comments" do expect(produce_comment_continuation("_cE8xSu6swE", "ADSJ_i2qvJeFtL0htmS5_K5Ctj3eGFVBMWL9Wd42o3kmUL6_mAzdLp85-liQZL0mYr_16BhaggUqX652Sv9JqV6VXinShSP-ZT6rL4NolPBaPXVtJsO5_rA_qE3GubAuLFw9uzIIXU2-HnpXbdgPLWTFavfX206hqWmmpHwUOrmxQV_OX6tYkM3ux3rPAKCDrT8eWL7MU3bLiNcnbgkW8o0h8KYLL_8BPa8LcHbTv8pAoNkjerlX1x7K4pqxaXPoyz89qNlnh6rRx6AXgAzzoHH1dmcyQ8CIBeOHg-m4i8ZxdX4dP88XWrIFg-jJGhpGP8JUMDgZgavxVx225hUEYZMyrLGler5em4FgbG62YWC51moLDLeYEA")).to eq("EkMSC19jRTh4U3U2c3dFyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyjAMK9gJBRFNKX2kycXZKZUZ0TDBodG1TNV9LNUN0ajNlR0ZWQk1XTDlXZDQybzNrbVVMNl9tQXpkTHA4NS1saVFaTDBtWXJfMTZCaGFnZ1VxWDY1MlN2OUpxVjZWWGluU2hTUC1aVDZyTDROb2xQQmFQWFZ0SnNPNV9yQV9xRTNHdWJBdUxGdzl1eklJWFUyLUhucFhiZGdQTFdURmF2ZlgyMDZocVdtbXBId1VPcm14UVZfT1g2dFlrTTN1eDNyUEFLQ0RyVDhlV0w3TVUzYkxpTmNuYmdrVzhvMGg4S1lMTF84QlBhOExjSGJUdjhwQW9Oa2plcmxYMXg3SzRwcXhhWFBveXo4OXFObG5oNnJSeDZBWGdBenpvSEgxZG1jeVE4Q0lCZU9IZy1tNGk4WnhkWDRkUDg4WFdySUZnLWpKR2hwR1A4SlVNRGdaZ2F2eFZ4MjI1aFVFWVpNeXJMR2xlcjVlbTRGZ2JHNjJZV0M1MW1vTERMZVlFQSIPIgtfY0U4eFN1NnN3RTAAKBQ%3D") diff --git a/spec/invidious/search/yt_filters_spec.cr b/spec/invidious/search/yt_filters_spec.cr new file mode 100644 index 00000000..27357058 --- /dev/null +++ b/spec/invidious/search/yt_filters_spec.cr @@ -0,0 +1,92 @@ +require "../../../src/invidious/search/filters" + +require "http/params" +require "spectator" + +Spectator.configure do |config| + config.fail_blank + config.randomize +end + +# Encoded filter values are extracted from the search +# page of Youtube with any browser devtools HTML inspector. + +DATE_FILTERS = { + Invidious::Search::Filters::Date::Hour => "EgIIAQ%3D%3D", + Invidious::Search::Filters::Date::Today => "EgIIAg%3D%3D", + Invidious::Search::Filters::Date::Week => "EgIIAw%3D%3D", + Invidious::Search::Filters::Date::Month => "EgIIBA%3D%3D", + Invidious::Search::Filters::Date::Year => "EgIIBQ%3D%3D", +} + +TYPE_FILTERS = { + Invidious::Search::Filters::Type::Video => "EgIQAQ%3D%3D", + Invidious::Search::Filters::Type::Channel => "EgIQAg%3D%3D", + Invidious::Search::Filters::Type::Playlist => "EgIQAw%3D%3D", + Invidious::Search::Filters::Type::Movie => "EgIQBA%3D%3D", +} + +DURATION_FILTERS = { + Invidious::Search::Filters::Duration::Short => "EgIYAQ%3D%3D", + Invidious::Search::Filters::Duration::Medium => "EgIYAw%3D%3D", + Invidious::Search::Filters::Duration::Long => "EgIYAg%3D%3D", +} + +FEATURE_FILTERS = { + Invidious::Search::Filters::Features::Live => "EgJAAQ%3D%3D", + Invidious::Search::Filters::Features::FourK => "EgJwAQ%3D%3D", + Invidious::Search::Filters::Features::HD => "EgIgAQ%3D%3D", + Invidious::Search::Filters::Features::Subtitles => "EgIoAQ%3D%3D", + Invidious::Search::Filters::Features::CCommons => "EgIwAQ%3D%3D", + Invidious::Search::Filters::Features::ThreeSixty => "EgJ4AQ%3D%3D", + Invidious::Search::Filters::Features::VR180 => "EgPQAQE%3D", + Invidious::Search::Filters::Features::ThreeD => "EgI4AQ%3D%3D", + Invidious::Search::Filters::Features::HDR => "EgPIAQE%3D", + Invidious::Search::Filters::Features::Location => "EgO4AQE%3D", + Invidious::Search::Filters::Features::Purchased => "EgJIAQ%3D%3D", +} + +SORT_FILTERS = { + Invidious::Search::Filters::Sort::Relevance => "", + Invidious::Search::Filters::Sort::Date => "CAI%3D", + Invidious::Search::Filters::Sort::Views => "CAM%3D", + Invidious::Search::Filters::Sort::Rating => "CAE%3D", +} + +Spectator.describe Invidious::Search::Filters do + # ------------------- + # Encode YT params + # ------------------- + + describe "#to_yt_params" do + sample DATE_FILTERS do |value, result| + it "Encodes upload date filter '#{value}'" do + expect(described_class.new(date: value).to_yt_params).to eq(result) + end + end + + sample TYPE_FILTERS do |value, result| + it "Encodes content type filter '#{value}'" do + expect(described_class.new(type: value).to_yt_params).to eq(result) + end + end + + sample DURATION_FILTERS do |value, result| + it "Encodes duration filter '#{value}'" do + expect(described_class.new(duration: value).to_yt_params).to eq(result) + end + end + + sample FEATURE_FILTERS do |value, result| + it "Encodes feature filter '#{value}'" do + expect(described_class.new(features: value).to_yt_params).to eq(result) + end + end + + sample SORT_FILTERS do |value, result| + it "Encodes sort filter '#{value}'" do + expect(described_class.new(sort: value).to_yt_params).to eq(result) + end + end + end +end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index 75ac287a..f1fd2695 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -1,3 +1,6 @@ +require "protodec/utils" +require "http/params" + module Invidious::Search struct Filters # Values correspond to { "2:embedded": { "1:varint": }} @@ -74,6 +77,63 @@ module Invidious::Search @features : Features = Features::None, @sort : Sort = Sort::Relevance ) + # ------------------- + # Youtube params + # ------------------- + + # Produce the youtube search parameters for the + # innertube API (base64-encoded protobuf object). + def to_yt_params(page : Int = 1) : String + # Initialize the embedded protobuf object + embedded = {} of String => Int64 + + # Add these field only if associated parameter is selected + embedded["1:varint"] = @date.to_i64 if !@date.none? + embedded["2:varint"] = @type.to_i64 if !@type.all? + embedded["3:varint"] = @duration.to_i64 if !@duration.none? + + if !@features.none? + # All features have a value of "1" when enabled, and + # the field is omitted when the feature is no selected. + embedded["4:varint"] = 1_i64 if @features.includes?(Features::HD) + embedded["5:varint"] = 1_i64 if @features.includes?(Features::Subtitles) + embedded["6:varint"] = 1_i64 if @features.includes?(Features::CCommons) + embedded["7:varint"] = 1_i64 if @features.includes?(Features::ThreeD) + embedded["8:varint"] = 1_i64 if @features.includes?(Features::Live) + embedded["9:varint"] = 1_i64 if @features.includes?(Features::Purchased) + embedded["14:varint"] = 1_i64 if @features.includes?(Features::FourK) + embedded["15:varint"] = 1_i64 if @features.includes?(Features::ThreeSixty) + embedded["23:varint"] = 1_i64 if @features.includes?(Features::Location) + embedded["25:varint"] = 1_i64 if @features.includes?(Features::HDR) + embedded["26:varint"] = 1_i64 if @features.includes?(Features::VR180) + end + + # Initialize an empty protobuf object + object = {} of String => (Int64 | String | Hash(String, Int64)) + + # As usual, everything can be omitted if it has no value + object["2:embedded"] = embedded if !embedded.empty? + + # Default sort is "relevance", so when this option is selected, + # the associated field can be omitted. + if !@sort.relevance? + object["1:varint"] = @sort.to_i64 + end + + # Add page number (if provided) + if page > 1 + object["9:varint"] = ((page - 1) * 20).to_i64 + end + + # If the object is empty, return an empty string, + # otherwise encode to protobuf then to base64 + return "" if object.empty? + + return object + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } end end end -- cgit v1.2.3 From 75c9dbaf6bec907f73c606be2a1710c9d3a68fc3 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 3 Mar 2022 23:29:13 +0100 Subject: Add a function to parse youtube search parameters --- spec/invidious/search/yt_filters_spec.cr | 51 ++++++++++++++++++++++++++ src/invidious/search/filters.cr | 62 ++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) (limited to 'src') diff --git a/spec/invidious/search/yt_filters_spec.cr b/spec/invidious/search/yt_filters_spec.cr index 27357058..bf7f21e7 100644 --- a/spec/invidious/search/yt_filters_spec.cr +++ b/spec/invidious/search/yt_filters_spec.cr @@ -89,4 +89,55 @@ Spectator.describe Invidious::Search::Filters do end end end + + # ------------------- + # Decode YT params + # ------------------- + + describe "#from_yt_params" do + sample DATE_FILTERS do |value, encoded| + it "Decodes upload date filter '#{value}'" do + params = HTTP::Params.parse("sp=#{encoded}") + + expect(described_class.from_yt_params(params)) + .to eq(described_class.new(date: value)) + end + end + + sample TYPE_FILTERS do |value, encoded| + it "Decodes content type filter '#{value}'" do + params = HTTP::Params.parse("sp=#{encoded}") + + expect(described_class.from_yt_params(params)) + .to eq(described_class.new(type: value)) + end + end + + sample DURATION_FILTERS do |value, encoded| + it "Decodes duration filter '#{value}'" do + params = HTTP::Params.parse("sp=#{encoded}") + + expect(described_class.from_yt_params(params)) + .to eq(described_class.new(duration: value)) + end + end + + sample FEATURE_FILTERS do |value, encoded| + it "Decodes feature filter '#{value}'" do + params = HTTP::Params.parse("sp=#{encoded}") + + expect(described_class.from_yt_params(params)) + .to eq(described_class.new(features: value)) + end + end + + sample SORT_FILTERS do |value, encoded| + it "Decodes sort filter '#{value}'" do + params = HTTP::Params.parse("sp=#{encoded}") + + expect(described_class.from_yt_params(params)) + .to eq(described_class.new(sort: value)) + end + end + end end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index f1fd2695..5c478257 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -135,5 +135,67 @@ module Invidious::Search .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } end + + # Function to parse the `sp` URL parameter from Youtube + # search page. It's a base64-encoded protobuf object. + def self.from_yt_params(params : HTTP::Params) : Filters + # Initialize output variable + filters = Filters.new + + # Get parameter, and check emptyness + search_params = params["sp"]? + + if search_params.nil? || search_params.empty? + return filters + end + + # Decode protobuf object + object = search_params + .try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } + + # Parse items from embedded object + if embedded = object["2:0:embedded"]? + # All the following fields (date, type, duration) are optional. + if date = embedded["1:0:varint"]? + filters.date = Date.from_value?(date.as_i) || Date::None + end + + if type = embedded["2:0:varint"]? + filters.type = Type.from_value?(type.as_i) || Type::All + end + + if duration = embedded["3:0:varint"]? + filters.duration = Duration.from_value?(duration.as_i) || Duration::None + end + + # All features should have a value of "1" when enabled, and + # the field should be omitted when the feature is no selected. + features = 0 + features += (embedded["4:0:varint"]?.try &.as_i == 1_i64) ? Features::HD.value : 0 + features += (embedded["5:0:varint"]?.try &.as_i == 1_i64) ? Features::Subtitles.value : 0 + features += (embedded["6:0:varint"]?.try &.as_i == 1_i64) ? Features::CCommons.value : 0 + features += (embedded["7:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeD.value : 0 + features += (embedded["8:0:varint"]?.try &.as_i == 1_i64) ? Features::Live.value : 0 + features += (embedded["9:0:varint"]?.try &.as_i == 1_i64) ? Features::Purchased.value : 0 + features += (embedded["14:0:varint"]?.try &.as_i == 1_i64) ? Features::FourK.value : 0 + features += (embedded["15:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeSixty.value : 0 + features += (embedded["23:0:varint"]?.try &.as_i == 1_i64) ? Features::Location.value : 0 + features += (embedded["25:0:varint"]?.try &.as_i == 1_i64) ? Features::HDR.value : 0 + features += (embedded["26:0:varint"]?.try &.as_i == 1_i64) ? Features::VR180.value : 0 + + filters.features = Features.from_value?(features) || Features::None + end + + if sort = object["1:0:varint"]? + filters.sort = Sort.from_value?(sort.as_i) || Sort::Relevance + end + + # Remove URL parameter and return result + params.delete("sp") + return filters + end end end -- cgit v1.2.3 From c888524523195382a5da171545441eaf0662ab01 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 4 Mar 2022 00:56:33 +0100 Subject: Add a function to parse invidious legacy search filters --- spec/invidious/search/iv_filters_spec.cr | 178 +++++++++++++++++++++++++++++++ src/invidious/search/filters.cr | 115 ++++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 spec/invidious/search/iv_filters_spec.cr (limited to 'src') diff --git a/spec/invidious/search/iv_filters_spec.cr b/spec/invidious/search/iv_filters_spec.cr new file mode 100644 index 00000000..6e8e6f3d --- /dev/null +++ b/spec/invidious/search/iv_filters_spec.cr @@ -0,0 +1,178 @@ +require "../../../src/invidious/search/filters" + +require "http/params" +require "spectator" + +Spectator.configure do |config| + config.fail_blank + config.randomize +end + +FEATURES_TEXT = { + Invidious::Search::Filters::Features::Live => "live", + Invidious::Search::Filters::Features::FourK => "4k", + Invidious::Search::Filters::Features::HD => "hd", + Invidious::Search::Filters::Features::Subtitles => "subtitles", + Invidious::Search::Filters::Features::CCommons => "commons", + Invidious::Search::Filters::Features::ThreeSixty => "360", + Invidious::Search::Filters::Features::VR180 => "vr180", + Invidious::Search::Filters::Features::ThreeD => "3d", + Invidious::Search::Filters::Features::HDR => "hdr", + Invidious::Search::Filters::Features::Location => "location", + Invidious::Search::Filters::Features::Purchased => "purchased", +} + +Spectator.describe Invidious::Search::Filters do + # ------------------- + # Decode (legacy) + # ------------------- + + describe "#from_legacy_filters" do + it "Decodes channel: filter" do + query = "test channel:UC123456 request" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new) + expect(chan).to eq("UC123456") + expect(qury).to eq("test request") + expect(subs).to be_false + end + + it "Decodes user: filter" do + query = "user:LinusTechTips broke something (again)" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new) + expect(chan).to eq("LinusTechTips") + expect(qury).to eq("broke something (again)") + expect(subs).to be_false + end + + it "Decodes type: filter" do + Invidious::Search::Filters::Type.each do |value| + query = "Eiffel 65 - Blue [1 Hour] type:#{value}" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(type: value)) + expect(chan).to eq("") + expect(qury).to eq("Eiffel 65 - Blue [1 Hour]") + expect(subs).to be_false + end + end + + it "Decodes content_type: filter" do + Invidious::Search::Filters::Type.each do |value| + query = "I like to watch content_type:#{value}" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(type: value)) + expect(chan).to eq("") + expect(qury).to eq("I like to watch") + expect(subs).to be_false + end + end + + it "Decodes date: filter" do + Invidious::Search::Filters::Date.each do |value| + query = "This date:#{value} is old!" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(date: value)) + expect(chan).to eq("") + expect(qury).to eq("This is old!") + expect(subs).to be_false + end + end + + it "Decodes duration: filter" do + Invidious::Search::Filters::Duration.each do |value| + query = "This duration:#{value} is old!" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(duration: value)) + expect(chan).to eq("") + expect(qury).to eq("This is old!") + expect(subs).to be_false + end + end + + it "Decodes feature: filter" do + Invidious::Search::Filters::Features.each do |value| + string = FEATURES_TEXT[value] + query = "I like my precious feature:#{string} ^^" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(features: value)) + expect(chan).to eq("") + expect(qury).to eq("I like my precious ^^") + expect(subs).to be_false + end + end + + it "Decodes features: filter" do + query = "This search has many features:vr180,cc,hdr :o" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + features = Invidious::Search::Filters::Features.flags(HDR, VR180, CCommons) + + expect(fltr).to eq(described_class.new(features: features)) + expect(chan).to eq("") + expect(qury).to eq("This search has many :o") + expect(subs).to be_false + end + + it "Decodes sort: filter" do + Invidious::Search::Filters::Sort.each do |value| + query = "Computer? sort:#{value} my files!" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new(sort: value)) + expect(chan).to eq("") + expect(qury).to eq("Computer? my files!") + expect(subs).to be_false + end + end + + it "Decodes subscriptions: filter" do + query = "enable subscriptions:true" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new) + expect(chan).to eq("") + expect(qury).to eq("enable") + expect(subs).to be_true + end + + it "Ignores junk data" do + query = "duration:I sort:like type:cleaning features:stuff date:up!" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new) + expect(chan).to eq("") + expect(qury).to eq("") + expect(subs).to be_false + end + + it "Keeps unknown keys" do + query = "to:be or:not to:be" + + fltr, chan, qury, subs = described_class.from_legacy_filters(query) + + expect(fltr).to eq(described_class.new) + expect(chan).to eq("") + expect(qury).to eq("to:be or:not to:be") + expect(subs).to be_false + end + end +end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index 5c478257..c5e91aae 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -77,6 +77,121 @@ module Invidious::Search @features : Features = Features::None, @sort : Sort = Sort::Relevance ) + end + + # ------------------- + # Invidious params + # ------------------- + + def self.parse_features(raw : Array(String)) : Features + # Initialize return variable + features = Features.new(0) + + raw.each do |ft| + case ft.downcase + when "live", "livestream" + features = features | Features::Live + when "4k" then features = features | Features::FourK + when "hd" then features = features | Features::HD + when "subtitles" then features = features | Features::Subtitles + when "creative_commons", "commons", "cc" + features = features | Features::CCommons + when "360" then features = features | Features::ThreeSixty + when "vr180" then features = features | Features::VR180 + when "3d" then features = features | Features::ThreeD + when "hdr" then features = features | Features::HDR + when "location" then features = features | Features::Location + when "purchased" then features = features | Features::Purchased + end + end + + return features + end + + def self.format_features(features : Features) : String + # Directly return an empty string if there are no features + return "" if features.none? + + # Initialize return variable + str = [] of String + + str << "live" if features.live? + str << "4k" if features.four_k? + str << "hd" if features.hd? + str << "subtitles" if features.subtitles? + str << "commons" if features.c_commons? + str << "360" if features.three_sixty? + str << "vr180" if features.vr180? + str << "3d" if features.three_d? + str << "hdr" if features.hdr? + str << "location" if features.location? + str << "purchased" if features.purchased? + + return str.join(',') + end + + def self.from_legacy_filters(str : String) : {Filters, String, String, Bool} + # Split search query on spaces + members = str.split(' ') + + # Output variables + channel = "" + filters = Filters.new + subscriptions = false + + # Array to hold the non-filter members + query = [] of String + + # Parse! + members.each do |substr| + # Separator operators + operators = substr.split(':') + + case operators[0] + when "user", "channel" + next if operators.size != 2 + channel = operators[1] + # + when "type", "content_type" + next if operators.size != 2 + type = Type.parse?(operators[1]) + filters.type = type if !type.nil? + # + when "date" + next if operators.size != 2 + date = Date.parse?(operators[1]) + filters.date = date if !date.nil? + # + when "duration" + next if operators.size != 2 + duration = Duration.parse?(operators[1]) + filters.duration = duration if !duration.nil? + # + when "feature", "features" + next if operators.size != 2 + features = parse_features(operators[1].split(',')) + filters.features = features if !features.nil? + # + when "sort" + next if operators.size != 2 + sort = Sort.parse?(operators[1]) + filters.sort = sort if !sort.nil? + # + when "subscriptions" + next if operators.size != 2 + subscriptions = {"true", "on", "yes", "1"}.any?(&.== operators[1]) + # + else + query << substr + end + end + + # Re-assemble query (without filters) + cleaned_query = query.join(' ') + + return {filters, channel, cleaned_query, subscriptions} + end + # ------------------- # Youtube params # ------------------- -- cgit v1.2.3 From fb2a331f79fcc42ac2c17ea349943ab2ba6ad0fe Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 6 Mar 2022 02:27:45 +0100 Subject: Add a function to parse search filters from invidious URL params --- spec/invidious/search/iv_filters_spec.cr | 79 ++++++++++++++++++++++++++++++++ src/invidious/search/filters.cr | 36 +++++++++++++++ 2 files changed, 115 insertions(+) (limited to 'src') diff --git a/spec/invidious/search/iv_filters_spec.cr b/spec/invidious/search/iv_filters_spec.cr index 6e8e6f3d..ebf01719 100644 --- a/spec/invidious/search/iv_filters_spec.cr +++ b/spec/invidious/search/iv_filters_spec.cr @@ -175,4 +175,83 @@ Spectator.describe Invidious::Search::Filters do expect(subs).to be_false end end + + # ------------------- + # Decode (URL) + # ------------------- + + describe "#from_iv_params" do + it "Decodes type= filter" do + Invidious::Search::Filters::Type.each do |value| + params = HTTP::Params.parse("type=#{value}") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(type: value)) + end + end + + it "Decodes date= filter" do + Invidious::Search::Filters::Date.each do |value| + params = HTTP::Params.parse("date=#{value}") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(date: value)) + end + end + + it "Decodes duration= filter" do + Invidious::Search::Filters::Duration.each do |value| + params = HTTP::Params.parse("duration=#{value}") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(duration: value)) + end + end + + it "Decodes features= filter (single)" do + Invidious::Search::Filters::Features.each do |value| + string = described_class.format_features(value) + params = HTTP::Params.parse("features=#{string}") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(features: value)) + end + end + + it "Decodes features= filter (multiple - comma separated)" do + features = Invidious::Search::Filters::Features.flags(HDR, VR180, CCommons) + params = HTTP::Params.parse("features=vr180%2Ccc%2Chdr") # %2C is a comma + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(features: features)) + end + + it "Decodes features= filter (multiple - URL parameters)" do + features = Invidious::Search::Filters::Features.flags(ThreeSixty, HD, FourK) + params = HTTP::Params.parse("features=4k&features=360&features=hd") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(features: features)) + end + + it "Decodes sort= filter" do + Invidious::Search::Filters::Sort.each do |value| + params = HTTP::Params.parse("sort=#{value}") + + expect(described_class.from_iv_params(params)) + .to eq(described_class.new(sort: value)) + end + end + + it "Ignores junk data" do + params = HTTP::Params.parse("foo=bar&sort=views&answer=42&type=channel") + + expect(described_class.from_iv_params(params)).to eq( + described_class.new( + sort: Invidious::Search::Filters::Sort::Views, + type: Invidious::Search::Filters::Type::Channel + ) + ) + end + end end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index c5e91aae..d7154d21 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -192,6 +192,42 @@ module Invidious::Search return {filters, channel, cleaned_query, subscriptions} end + def self.from_iv_params(params : HTTP::Params) : Filters + # Temporary variables + filters = Filters.new + + if type = params["type"]? + filters.type = Type.parse?(type) || Type::All + params.delete("type") + end + + if date = params["date"]? + filters.date = Date.parse?(date) || Date::None + params.delete("date") + end + + if duration = params["duration"]? + filters.duration = Duration.parse?(duration) || Duration::None + params.delete("duration") + end + + features = params.fetch_all("features") + if !features.empty? + # Un-array input so it can be treated as a comma-separated list + features = features[0].split(',') if features.size == 1 + + filters.features = parse_features(features) || Features::None + params.delete_all("features") + end + + if sort = params["sort"]? + filters.sort = Sort.parse?(sort) || Sort::Relevance + params.delete("sort") + end + + return filters + end + # ------------------- # Youtube params # ------------------- -- cgit v1.2.3 From 6991d0851fae9d9abff1a714a5bd72ccaac7dec4 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 12 Mar 2022 19:56:52 +0100 Subject: Add a function to generate HTTP::Params from Filters --- spec/invidious/search/iv_filters_spec.cr | 114 +++++++++++++++++++++++++++++++ src/invidious/search/filters.cr | 19 ++++++ 2 files changed, 133 insertions(+) (limited to 'src') diff --git a/spec/invidious/search/iv_filters_spec.cr b/spec/invidious/search/iv_filters_spec.cr index ebf01719..b0897a63 100644 --- a/spec/invidious/search/iv_filters_spec.cr +++ b/spec/invidious/search/iv_filters_spec.cr @@ -254,4 +254,118 @@ Spectator.describe Invidious::Search::Filters do ) end end + + # ------------------- + # Encode (URL) + # ------------------- + + describe "#to_iv_params" do + it "Encodes date filter" do + Invidious::Search::Filters::Date.each do |value| + filters = described_class.new(date: value) + params = filters.to_iv_params + + if value.none? + expect("#{params}").to eq("") + else + expect("#{params}").to eq("date=#{value.to_s.underscore}") + end + end + end + + it "Encodes type filter" do + Invidious::Search::Filters::Type.each do |value| + filters = described_class.new(type: value) + params = filters.to_iv_params + + if value.all? + expect("#{params}").to eq("") + else + expect("#{params}").to eq("type=#{value.to_s.underscore}") + end + end + end + + it "Encodes duration filter" do + Invidious::Search::Filters::Duration.each do |value| + filters = described_class.new(duration: value) + params = filters.to_iv_params + + if value.none? + expect("#{params}").to eq("") + else + expect("#{params}").to eq("duration=#{value.to_s.underscore}") + end + end + end + + it "Encodes features filter (single)" do + Invidious::Search::Filters::Features.each do |value| + string = described_class.format_features(value) + filters = described_class.new(features: value) + + expect("#{filters.to_iv_params}") + .to eq("features=" + FEATURES_TEXT[value]) + end + end + + it "Encodes features filter (multiple)" do + features = Invidious::Search::Filters::Features.flags(Subtitles, Live, ThreeSixty) + filters = described_class.new(features: features) + + expect("#{filters.to_iv_params}") + .to eq("features=live%2Csubtitles%2C360") # %2C is a comma + end + + it "Encodes sort filter" do + Invidious::Search::Filters::Sort.each do |value| + filters = described_class.new(sort: value) + params = filters.to_iv_params + + if value.relevance? + expect("#{params}").to eq("") + else + expect("#{params}").to eq("sort=#{value.to_s.underscore}") + end + end + end + + it "Encodes multiple filters" do + filters = described_class.new( + date: Invidious::Search::Filters::Date::Today, + duration: Invidious::Search::Filters::Duration::Medium, + features: Invidious::Search::Filters::Features.flags(Location, Purchased), + sort: Invidious::Search::Filters::Sort::Relevance + ) + + params = filters.to_iv_params + + # Check the `date` param + expect(params).to have_key("date") + expect(params.fetch_all("date")).to contain_exactly("today") + + # Check the `type` param + expect(params).to_not have_key("type") + expect(params["type"]?).to be_nil + + # Check the `duration` param + expect(params).to have_key("duration") + expect(params.fetch_all("duration")).to contain_exactly("medium") + + # Check the `features` param + expect(params).to have_key("features") + expect(params.fetch_all("features")).to contain_exactly("location,purchased") + + # Check the `sort` param + expect(params).to_not have_key("sort") + expect(params["sort"]?).to be_nil + + # Check if there aren't other parameters + params.delete("date") + params.delete("duration") + params.delete("features") + + expect(params).to be_empty + end + end end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index d7154d21..8f4ada6c 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -228,6 +228,25 @@ module Invidious::Search return filters end + def to_iv_params : HTTP::Params + # Temporary variables + raw_params = {} of String => Array(String) + + raw_params["date"] = [@date.to_s.underscore] if !@date.none? + raw_params["type"] = [@type.to_s.underscore] if !@type.all? + raw_params["sort"] = [@sort.to_s.underscore] if !@sort.relevance? + + if !@duration.none? + raw_params["duration"] = [@duration.to_s.underscore] + end + + if !@features.none? + raw_params["features"] = [Filters.format_features(@features)] + end + + return HTTP::Params.new(raw_params) + end + # ------------------- # Youtube params # ------------------- -- cgit v1.2.3 From 1e3425fdee91f1b25f67d1e03872b68e978bc6e0 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 9 Mar 2022 22:21:53 +0100 Subject: Add filters UI HTML generator --- assets/css/search.css | 91 +++++++++++++++++++++ locales/en-US.json | 69 +++++++++------- src/invidious/frontend/search_filters.cr | 135 +++++++++++++++++++++++++++++++ 3 files changed, 264 insertions(+), 31 deletions(-) create mode 100644 assets/css/search.css create mode 100644 src/invidious/frontend/search_filters.cr (limited to 'src') diff --git a/assets/css/search.css b/assets/css/search.css new file mode 100644 index 00000000..ad2b0b16 --- /dev/null +++ b/assets/css/search.css @@ -0,0 +1,91 @@ +summary { + /* This should hide the marker */ + display: block; + + font-size: 1.17em; + font-weight: bold; + margin: 0 auto 10px auto; +} + +summary::-webkit-details-marker, +summary::marker { display: none; } + +summary:before { + border-radius: 5px; + content: "[ + ]"; + margin: -2px 10px 0 10px; + padding: 1px 0 3px 0; + text-align: center; + width: 40px; +} + +details[open] > summary:before { content: "[ ‒ ]"; } + + +#filters-box { + background: #373737; + padding: 10px 20px 20px 10px; + margin: 10px 15px; +} +#filters-flex { + display: flex; + flex-wrap: wrap; + flex-direction: row; + align-items: flex-start; + align-content: flex-start; + justify-content: flex-start; +} + + +fieldset, legend { + display: contents !important; + border: none !important; + margin: 0 !important; + padding: 0 !important; +} + + +.filter-column { + display: inline-block; + display: inline-flex; + width: max-content; + min-width: max-content; + max-width: 16em; + margin: 15px; + flex-grow: 2; + flex-basis: auto; + flex-direction: column; +} +.filter-name, .filter-options { + display: block; + padding: 5px 10px; + margin: 0; + text-align: start; +} + +/* TODO: move that to the main file */ +.underlined { + border-bottom: 1px solid; + margin-bottom: 20px; +} + + +.filter-options div { margin: 6px 0; } +.filter-options div * { vertical-align: middle; } +.filter-options label { margin: 0 10px; } + + +#filters-apply { text-align: end; } + + +@media only screen and (max-width: 800px) { + summary { font-size: 1.30em; } + #filters-box { + margin: 10px 0 0 0; + padding: 0; + } + #filters-apply { + text-align: center; + padding: 15px; + } +} diff --git a/locales/en-US.json b/locales/en-US.json index a78d8062..03df88b6 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -404,37 +404,44 @@ "Videos": "Videos", "Playlists": "Playlists", "Community": "Community", - "relevance": "Relevance", - "rating": "Rating", - "date": "Upload date", - "views": "View count", - "content_type": "Type", - "duration": "Duration", - "features": "Features", - "sort": "Sort By", - "hour": "Last Hour", - "today": "Today", - "week": "This week", - "month": "This month", - "year": "This year", - "video": "Video", - "channel": "Channel", - "playlist": "Playlist", - "movie": "Movie", - "show": "Show", - "short": "Short (< 4 minutes)", - "long": "Long (> 20 minutes)", - "hd": "HD", - "subtitles": "Subtitles/CC", - "creative_commons": "Creative Commons", - "3d": "3D", - "live": "Live", - "4k": "4K", - "location": "Location", - "hdr": "HDR", - "purchased": "Purchased", - "360": "360°", - "filter": "Filter", + "search_filters_title": "Filters", + "search_filters_date_label": "Upload date", + "search_filters_date_option_none": "Any date", + "search_filters_date_option_hour": "Last Hour", + "search_filters_date_option_today": "Today", + "search_filters_date_option_week": "This week", + "search_filters_date_option_month": "This month", + "search_filters_date_option_year": "This year", + "search_filters_type_label": "Type", + "search_filters_type_option_all": "Any type", + "search_filters_type_option_video": "Video", + "search_filters_type_option_channel": "Channel", + "search_filters_type_option_playlist": "Playlist", + "search_filters_type_option_movie": "Movie", + "search_filters_type_option_show": "Show", + "search_filters_duration_label": "Duration", + "search_filters_duration_option_none": "Any duration", + "search_filters_duration_option_short": "Short (< 4 minutes)", + "search_filters_duration_option_medium": "Medium (4 - 20 minutes)", + "search_filters_duration_option_long": "Long (> 20 minutes)", + "search_filters_features_label": "Features", + "search_filters_features_option_live": "Live", + "search_filters_features_option_four_k": "4K", + "search_filters_features_option_hd": "HD", + "search_filters_features_option_subtitles": "Subtitles/CC", + "search_filters_features_option_c_commons": "Creative Commons", + "search_filters_features_option_three_sixty": "360°", + "search_filters_features_option_vr180": "VR180", + "search_filters_features_option_three_d": "3D", + "search_filters_features_option_hdr": "HDR", + "search_filters_features_option_location": "Location", + "search_filters_features_option_purchased": "Purchased", + "search_filters_sort_label": "Sort By", + "search_filters_sort_option_relevance": "Relevance", + "search_filters_sort_option_rating": "Rating", + "search_filters_sort_option_date": "Upload Date", + "search_filters_sort_option_views": "View count", + "search_filters_apply_button": "Apply selected filters", "Current version: ": "Current version: ", "next_steps_error_message": "After which you should try to: ", "next_steps_error_message_refresh": "Refresh", diff --git a/src/invidious/frontend/search_filters.cr b/src/invidious/frontend/search_filters.cr new file mode 100644 index 00000000..68f27b4f --- /dev/null +++ b/src/invidious/frontend/search_filters.cr @@ -0,0 +1,135 @@ +module Invidious::Frontend::SearchFilters + extend self + + # Generate the search filters collapsable widget. + def generate(filters : Search::Filters, query : String, page : Int, locale : String) : String + return String.build(8000) do |str| + str << "
\n" + str << "\t
" + str << "\t\t" << translate(locale, "search_filters_title") << "\n" + + str << "\t\t
\n" + + str << "\t\t\t\n" + str << "\t\t\t\n" + + str << "\t\t\t
" + + filter_wrapper(date) + filter_wrapper(type) + filter_wrapper(duration) + filter_wrapper(features) + filter_wrapper(sort) + + str << "\t\t\t
\n" + + str << "\t\t\t
" + str << "
\n" + + str << "\t\t
\n" + + str << "\t
\n" + str << "
\n" + end + end + + # Generate wrapper HTML (`
`, filter name, etc...) around the + # `` elements of a search filter + macro filter_wrapper(name) + str << "\t\t\t\t
\n" + + str << "\t\t\t\t\t
" + str << translate(locale, "search_filters_{{name}}_label") + str << "
\n" + + str << "\t\t\t\t\t
\n" + make_{{name}}_filter_options(str, filters.{{name}}, locale) + str << "\t\t\t\t\t
" + + str << "\t\t\t\t
\n" + end + + # Generates the HTML for the list of radio buttons of the "date" search filter + def make_date_filter_options(str : String::Builder, value : Search::Filters::Date, locale : String) + {% for value in Invidious::Search::Filters::Date.constants %} + {% date = value.underscore %} + + str << "\t\t\t\t\t\t
" + str << "' + + str << "
\n" + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "type" search filter + def make_type_filter_options(str : String::Builder, value : Search::Filters::Type, locale : String) + {% for value in Invidious::Search::Filters::Type.constants %} + {% type = value.underscore %} + + str << "\t\t\t\t\t\t
" + str << "' + + str << "
\n" + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "duration" search filter + def make_duration_filter_options(str : String::Builder, value : Search::Filters::Duration, locale : String) + {% for value in Invidious::Search::Filters::Duration.constants %} + {% duration = value.underscore %} + + str << "\t\t\t\t\t\t
" + str << "' + + str << "
\n" + {% end %} + end + + # Generates the HTML for the list of checkboxes of the "features" search filter + def make_features_filter_options(str : String::Builder, value : Search::Filters::Features, locale : String) + {% for value in Invidious::Search::Filters::Features.constants %} + {% if value.stringify != "All" && value.stringify != "None" %} + {% feature = value.underscore %} + + str << "\t\t\t\t\t\t
" + str << "' + + str << "
\n" + {% end %} + {% end %} + end + + # Generates the HTML for the list of radio buttons of the "sort" search filter + def make_sort_filter_options(str : String::Builder, value : Search::Filters::Sort, locale : String) + {% for value in Invidious::Search::Filters::Sort.constants %} + {% sort = value.underscore %} + + str << "\t\t\t\t\t\t
" + str << "' + + str << "
\n" + {% end %} + end +end -- cgit v1.2.3 From a813955ad39c254240dcb02344d94d03d0bbd6b2 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 26 Mar 2022 22:18:42 +0100 Subject: Add Search::Query class to handle search queries --- src/invidious/search/filters.cr | 5 ++ src/invidious/search/query.cr | 149 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 src/invidious/search/query.cr (limited to 'src') diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index 8f4ada6c..0e8438b9 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -79,6 +79,11 @@ module Invidious::Search ) end + def is_default? : Bool + return @date.none? && @type.all? && @duration.none? && \ + @features.none? && @sort.relevance? + end + # ------------------- # Invidious params # ------------------- diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr new file mode 100644 index 00000000..4d76b083 --- /dev/null +++ b/src/invidious/search/query.cr @@ -0,0 +1,149 @@ +module Invidious::Search + class Query + enum Type + # Types related to YouTube + Regular # Youtube search page + Channel # Youtube channel search box + + # Types specific to Invidious + Subscriptions # Search user subscriptions + Playlist # "Add playlist item" search + end + + @type : Type = Type::Regular + + @raw_query : String + @query : String = "" + + property filters : Filters = Filters.new + property page : Int32 + property region : String? + property channel : String = "" + + # Return true if @raw_query is either `nil` or empty + private def empty_raw_query? + return @raw_query.empty? + end + + # Same as `empty_raw_query?`, but named for external use + def empty? + return self.empty_raw_query? + end + + # Getter for the query string. + # It is named `text` to reduce confusion (`search_query.text` makes more + # sense than `search_query.query`) + def text + return @query + end + + # Initialize a new search query. + # Parameters are used to get the query string, the page number + # and the search filters (if any). Type tells this function + # where it is being called from (See `Type` above). + def initialize( + params : HTTP::Params, + @type : Type = Type::Regular, + @region : String? = nil + ) + # Get the raw search query string (common to all search types). In + # Regular search mode, also look for the `search_query` URL parameter + if @type.regular? + @raw_query = params["q"]? || params["search_query"]? || "" + else + @raw_query = params["q"]? || "" + end + + # Get the page number (also common to all search types) + @page = params["page"]?.try &.to_i? || 1 + + # Stop here is raw query in empty + # NOTE: maybe raise in the future? + return if self.empty_raw_query? + + # Specific handling + case @type + when .playlist?, .channel? + # In "add playlist item" mode, filters are parsed from the query + # string itself (legacy), and the channel is ignored. + # + # In "channel search" mode, filters are ignored, but we still parse + # the query prevent transmission of legacy filters to youtube. + # + @filters, @query, @channel, _ = Filters.from_legacy_filters(@raw_query || "") + # + when .subscriptions?, .regular? + if params["sp"]? + # Parse the `sp` URL parameter (youtube compatibility) + @filters = Filters.from_yt_params(params) + @query = @raw_query || "" + else + # Parse invidious URL parameters (sort, date, etc...) + @filters = Filters.from_iv_params(params) + @channel = params["channel"]? || "" + + if @filters.default? && @raw_query.includes?(':') + # Parse legacy filters from query + @filters, @query, @channel, subs = Filters.from_legacy_filters(@raw_query || "") + else + @query = @raw_query || "" + end + + if !@channel.empty? + # Switch to channel search mode (filters will be ignored) + @type = Type::Channel + elsif subs + # Switch to subscriptions search mode + @type = Type::Subscriptions + end + end + end + end + + # Run the search query using the corresponding search processor. + # Returns either the results or an empty array of `SearchItem`. + def process(user : Invidious::User? = nil) : Array(SearchItem) | Array(ChannelVideo) + items = [] of SearchItem + + # Don't bother going further if search query is empty + return items if self.empty_raw_query? + + case @type + when .regular?, .playlist? + all_items = search(@query, @filters, @page, @region) + items = unnest_items(all_items) + # + when .channel? + items = Processors.channel(@query, @page, @channel) + # + when .subscriptions? + if user + items = Processors.subscriptions(self, user.as(Invidious::User)) + end + end + + return items + end + + # TODO: clean code + private def unnest_items(all_items) : Array(SearchItem) + items = [] of SearchItem + + # Light processing to flatten search results out of Categories. + # They should ideally be supported in the future. + all_items.each do |i| + if i.is_a? Category + i.contents.each do |nest_i| + if !nest_i.is_a? Video + items << nest_i + end + end + else + items << i + end + end + + return items + end + end +end -- cgit v1.2.3 From d93a7b315db42474aac4a8e27c3745dc4b5abdeb Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 26 Mar 2022 20:15:02 +0100 Subject: Make use of Search::Query/Filters and associated HTML generator --- src/invidious/routes/api/v1/channels.cr | 16 +-- src/invidious/routes/api/v1/search.cr | 24 +---- src/invidious/routes/playlists.cr | 18 ++-- src/invidious/routes/search.cr | 24 ++--- src/invidious/search.cr | 150 +---------------------------- src/invidious/search/filters.cr | 2 +- src/invidious/search/processors.cr | 28 ++++-- src/invidious/search/query.cr | 5 +- src/invidious/views/add_playlist_items.ecr | 11 ++- src/invidious/views/search.ecr | 140 +++++---------------------- 10 files changed, 87 insertions(+), 331 deletions(-) (limited to 'src') diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index c4395353..8650976d 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -251,18 +251,22 @@ module Invidious::Routes::API::V1::Channels def self.search(env) locale = env.get("preferences").as(Preferences).locale + region = env.params.query["region"]? env.response.content_type = "application/json" - ucid = env.params.url["ucid"] + query = Invidious::Search::Query.new(env.params.query, :channel, region) - query = env.params.query["q"]? - query ||= "" + # Required because we can't (yet) pass multiple parameter to the + # `Search::Query` initializer (in this case, an URL segment) + query.channel = env.params.url["ucid"] - page = env.params.query["page"]?.try &.to_i? - page ||= 1 + begin + search_results = query.process + rescue ex + return error_json(400, ex) + end - search_results = Invidious::Search::Processors.channel(query, page, ucid) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 5666460d..21451d33 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -5,34 +5,14 @@ module Invidious::Routes::API::V1::Search env.response.content_type = "application/json" - query = env.params.query["q"]? - query ||= "" - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - sort_by = env.params.query["sort_by"]?.try &.downcase - sort_by ||= "relevance" - - date = env.params.query["date"]?.try &.downcase - date ||= "" - - duration = env.params.query["duration"]?.try &.downcase - duration ||= "" - - features = env.params.query["features"]?.try &.split(",").map(&.downcase) - features ||= [] of String - - content_type = env.params.query["type"]?.try &.downcase - content_type ||= "video" + query = Invidious::Search::Query.new(env.params.query, :regular, region) begin - search_params = produce_search_params(page, sort_by, date, content_type, duration, features) + search_results = query.process rescue ex return error_json(400, ex) end - search_results = search(query, search_params, region) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index dbeb4f97..de981d81 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -212,7 +212,10 @@ module Invidious::Routes::Playlists end def self.add_playlist_items_page(env) - locale = env.get("preferences").as(Preferences).locale + prefs = env.get("preferences").as(Preferences) + locale = prefs.locale + + region = env.params.query["region"]? || prefs.region user = env.get? "user" sid = env.get? "sid" @@ -236,15 +239,10 @@ module Invidious::Routes::Playlists return env.redirect referer end - query = env.params.query["q"]? - if query - begin - search_query, items, operators = process_search_query(query, page, user, region: nil) - videos = items.select(SearchVideo).map(&.as(SearchVideo)) - rescue ex - videos = [] of SearchVideo - end - else + begin + query = Invidious::Search::Query.new(env.params.query, :playlist, region) + videos = query.process.select(SearchVideo).map(&.as(SearchVideo)) + rescue ex videos = [] of SearchVideo end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 3f4c7e5e..e60d0081 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -37,37 +37,29 @@ module Invidious::Routes::Search end def self.search(env) - locale = env.get("preferences").as(Preferences).locale - region = env.params.query["region"]? + prefs = env.get("preferences").as(Preferences) + locale = prefs.locale - query = env.params.query["search_query"]? - query ||= env.params.query["q"]? + region = env.params.query["region"]? || prefs.region + + query = Invidious::Search::Query.new(env.params.query, :regular, region) - if !query || query.empty? + if query.empty? # Display the full page search box implemented in #1977 env.set "search", "" templated "search_homepage", navbar_search: false else - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - user = env.get? "user" begin - search_query, videos, operators = process_search_query(query, page, user, region: region) + videos = query.process rescue ex : ChannelSearchException return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex return error_template(500, ex) end - operator_hash = {} of String => String - operators.each do |operator| - key, value = operator.downcase.split(":") - operator_hash[key] = value - end - - env.set "search", query + env.set "search", query.text templated "search" end end diff --git a/src/invidious/search.cr b/src/invidious/search.cr index af854653..e4c21bd4 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -5,113 +5,6 @@ class ChannelSearchException < InfoException end end -def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem) - return [] of SearchItem if query.empty? - - client_config = YoutubeAPI::ClientConfig.new(region: region) - initial_data = YoutubeAPI.search(query, search_params, client_config: client_config) - - return extract_items(initial_data) -end - -def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "", - duration : String = "", features : Array(String) = [] of String) - object = { - "1:varint" => 0_i64, - "2:embedded" => {} of String => Int64, - "9:varint" => ((page - 1) * 20).to_i64, - } - - case sort - when "relevance" - object["1:varint"] = 0_i64 - when "rating" - object["1:varint"] = 1_i64 - when "upload_date", "date" - object["1:varint"] = 2_i64 - when "view_count", "views" - object["1:varint"] = 3_i64 - else - raise "No sort #{sort}" - end - - case date - when "hour" - object["2:embedded"].as(Hash)["1:varint"] = 1_i64 - when "today" - object["2:embedded"].as(Hash)["1:varint"] = 2_i64 - when "week" - object["2:embedded"].as(Hash)["1:varint"] = 3_i64 - when "month" - object["2:embedded"].as(Hash)["1:varint"] = 4_i64 - when "year" - object["2:embedded"].as(Hash)["1:varint"] = 5_i64 - else nil # Ignore - end - - case content_type - when "video" - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - when "channel" - object["2:embedded"].as(Hash)["2:varint"] = 2_i64 - when "playlist" - object["2:embedded"].as(Hash)["2:varint"] = 3_i64 - when "movie" - object["2:embedded"].as(Hash)["2:varint"] = 4_i64 - when "show" - object["2:embedded"].as(Hash)["2:varint"] = 5_i64 - when "all" - # - else - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - end - - case duration - when "short" - object["2:embedded"].as(Hash)["3:varint"] = 1_i64 - when "long" - object["2:embedded"].as(Hash)["3:varint"] = 2_i64 - else nil # Ignore - end - - features.each do |feature| - case feature - when "hd" - object["2:embedded"].as(Hash)["4:varint"] = 1_i64 - when "subtitles" - object["2:embedded"].as(Hash)["5:varint"] = 1_i64 - when "creative_commons", "cc" - object["2:embedded"].as(Hash)["6:varint"] = 1_i64 - when "3d" - object["2:embedded"].as(Hash)["7:varint"] = 1_i64 - when "live", "livestream" - object["2:embedded"].as(Hash)["8:varint"] = 1_i64 - when "purchased" - object["2:embedded"].as(Hash)["9:varint"] = 1_i64 - when "4k" - object["2:embedded"].as(Hash)["14:varint"] = 1_i64 - when "360" - object["2:embedded"].as(Hash)["15:varint"] = 1_i64 - when "location" - object["2:embedded"].as(Hash)["23:varint"] = 1_i64 - when "hdr" - object["2:embedded"].as(Hash)["25:varint"] = 1_i64 - else nil # Ignore - end - end - - if object["2:embedded"].as(Hash).empty? - object.delete("2:embedded") - end - - params = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return params -end - def produce_channel_search_continuation(ucid, query, page) if page <= 1 idx = 0_i64 @@ -146,41 +39,10 @@ def produce_channel_search_continuation(ucid, query, page) end def process_search_query(query, page, user, region) - channel = nil - content_type = "all" - date = "" - duration = "" - features = [] of String - sort = "relevance" - subscriptions = nil - - operators = query.split(" ").select(&.match(/\w+:[\w,]+/)) - operators.each do |operator| - key, value = operator.downcase.split(":") - - case key - when "channel", "user" - channel = operator.split(":")[-1] - when "content_type", "type" - content_type = value - when "date" - date = value - when "duration" - duration = value - when "feature", "features" - features = value.split(",") - when "sort" - sort = value - when "subscriptions" - subscriptions = value == "true" - else - operators.delete(operator) - end - end + # Parse legacy query + filters, channel, search_query, subscriptions = Invidious::Search::Filters.from_legacy_filters(query) - search_query = (query.split(" ") - operators).join(" ") - - if channel + if !channel.nil? && !channel.empty? items = Invidious::Search::Processors.channel(search_query, page, channel) elsif subscriptions if user @@ -190,9 +52,7 @@ def process_search_query(query, page, user, region) items = [] of ChannelVideo end else - search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, - duration: duration, features: features) - + search_params = filters.to_yt_params(page: page) items = search(search_query, search_params, region) end @@ -211,5 +71,5 @@ def process_search_query(query, page, user, region) end end - {search_query, items_without_category, operators} + {search_query, items_without_category, filters} end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index 0e8438b9..c2b5c758 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -79,7 +79,7 @@ module Invidious::Search ) end - def is_default? : Bool + def default? : Bool return @date.none? && @type.all? && @duration.none? && \ @features.none? && @sort.relevance? end diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr index c5327f34..d1409c06 100644 --- a/src/invidious/search/processors.cr +++ b/src/invidious/search/processors.cr @@ -2,22 +2,32 @@ module Invidious::Search module Processors extend self + # Regular search (`/search` endpoint) + def regular(query : Query) : Array(SearchItem) + search_params = query.filters.to_yt_params(page: query.page) + + client_config = YoutubeAPI::ClientConfig.new(region: query.region) + initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) + + return extract_items(initial_data) + end + # Search a youtube channel # TODO: clean code, and rely more on YoutubeAPI - def channel(query, page, channel) : Array(SearchItem) - response = YT_POOL.client &.get("/channel/#{channel}") + def channel(query : Query) : Array(SearchItem) + response = YT_POOL.client &.get("/channel/#{query.channel}") if response.status_code == 404 - response = YT_POOL.client &.get("/user/#{channel}") - response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 + response = YT_POOL.client &.get("/user/#{query.channel}") + response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404 initial_data = extract_initial_data(response.body) ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) - raise ChannelSearchException.new(channel) if !ucid + raise ChannelSearchException.new(query.channel) if !ucid else - ucid = channel + ucid = query.channel end - continuation = produce_channel_search_continuation(ucid, query, page) + continuation = produce_channel_search_continuation(ucid, query.text, query.page) response_json = YoutubeAPI.browse(continuation) continuation_items = response_json["onResponseReceivedActions"]? @@ -34,7 +44,7 @@ module Invidious::Search end # Search inside of user subscriptions - def subscriptions(query, page, user : Invidious::User) : Array(ChannelVideo) + def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo) view_name = "subscriptions_#{sha256(user.email)}" return PG_DB.query_all(" @@ -46,7 +56,7 @@ module Invidious::Search as document FROM #{view_name} ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", - query, (page - 1) * 20, + query.text, (query.page - 1) * 20, as: ChannelVideo ) end diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index 4d76b083..1c2b37d2 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -110,11 +110,10 @@ module Invidious::Search case @type when .regular?, .playlist? - all_items = search(@query, @filters, @page, @region) - items = unnest_items(all_items) + items = unnest_items(Processors.regular(self)) # when .channel? - items = Processors.channel(@query, @page, @channel) + items = Processors.channel(self) # when .subscriptions? if user diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr index ad50909a..22870317 100644 --- a/src/invidious/views/add_playlist_items.ecr +++ b/src/invidious/views/add_playlist_items.ecr @@ -11,7 +11,9 @@ <%= translate(locale, "Editing playlist `x`", %|"#{HTML.escape(playlist.title)}"|) %>
- value="<%= HTML.escape(query) %>"<% else %>placeholder="<%= translate(locale, "Search for videos") %>"<% end %>> + value="<%= HTML.escape(query.text) %>"<% end %> + placeholder="<%= translate(locale, "Search for videos") %>">
@@ -38,10 +40,11 @@
<% if query %> + <%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%>
- <% if page > 1 %> - + <% if query.page > 1 %> + <%= translate(locale, "Previous page") %> <% end %> @@ -49,7 +52,7 @@
<% if videos.size >= 20 %> - + <%= translate(locale, "Next page") %> <% end %> diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index 45bbdefc..f1f6ab20 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -1,124 +1,38 @@ <% content_for "header" do %> -<%= search_query.not_nil!.size > 30 ? HTML.escape(query.not_nil![0,30].rstrip(".") + "...") : HTML.escape(query.not_nil!) %> - Invidious +<%= query.text.size > 30 ? HTML.escape(query.text[0,30].rstrip(".")) + "…" : HTML.escape(query.text) %> - Invidious + <% end %> -<% search_query_encoded = env.get?("search").try { |x| URI.encode_www_form(x.as(String), space_to_plus: true) } %> +<%- + search_query_encoded = URI.encode_www_form(query.text, space_to_plus: true) + filter_params = query.filters.to_iv_params + + url_prev_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page - 1}" + url_next_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page + 1}" +-%> <% if videos.size == 0 %>

"><%= translate(locale, "Broken? Try another Invidious Instance!") %>

-<% else %> -
- -

<%= translate(locale, "filter") %>

- -
-
- <%= translate(locale, "date") %> -
- <% ["hour", "today", "week", "month", "year"].each do |date| %> -
- <% if operator_hash.fetch("date", "all") == date %> - <%= translate(locale, date) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, date) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "content_type") %> -
- <% ["video", "channel", "playlist", "movie", "show"].each do |content_type| %> -
- <% if operator_hash.fetch("content_type", "all") == content_type %> - <%= translate(locale, content_type) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, content_type) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "duration") %> -
- <% ["short", "long"].each do |duration| %> -
- <% if operator_hash.fetch("duration", "all") == duration %> - <%= translate(locale, duration) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, duration) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "features") %> -
- <% ["hd", "subtitles", "creative_commons", "3d", "live", "purchased", "4k", "360", "location", "hdr"].each do |feature| %> -
- <% if operator_hash.fetch("features", "all").includes?(feature) %> - <%= translate(locale, feature) %> - <% elsif operator_hash.has_key?("features") %> - &page=<%= page %>"> - <%= translate(locale, feature) %> - - <% else %> - &page=<%= page %>"> - <%= translate(locale, feature) %> - - <% end %> -
- <% end %> -
-
- <%= translate(locale, "sort") %> -
- <% ["relevance", "rating", "date", "views"].each do |sort| %> -
- <% if operator_hash.fetch("sort", "relevance") == sort %> - <%= translate(locale, sort) %> - <% else %> - &page=<%= page %>"> - <%= translate(locale, sort) %> - - <% end %> -
- <% end %> -
-
-
-<% end %> +<%- else -%> + <%= Invidious::Frontend::SearchFilters.generate(query.filters, query.text, query.page, locale) %> +<%- end -%> -<% if videos.size == 0 %> -
-<% else %> -
-<% end %> +<% if videos.size == 0 %>
<% else %>
<% end %>
- <% if page > 1 %> - - <%= translate(locale, "Previous page") %> - - <% end %> + <%- if query.page > 1 -%> + <%= translate(locale, "Previous page") %> + <%- end -%>
- <% if videos.size >= 20 %> - - <%= translate(locale, "Next page") %> - - <% end %> + <%- if videos.size >= 20 -%> + <%= translate(locale, "Next page") %> + <%- end -%>
@@ -130,18 +44,14 @@
- <% if page > 1 %> - - <%= translate(locale, "Previous page") %> - - <% end %> + <%- if query.page > 1 -%> + <%= translate(locale, "Previous page") %> + <%- end -%>
- <% if videos.size >= 20 %> - - <%= translate(locale, "Next page") %> - - <% end %> + <%- if videos.size >= 20 -%> + <%= translate(locale, "Next page") %> + <%- end -%>
-- cgit v1.2.3 From af029177666486d7524a7d1fae03aed91b58e556 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 27 Mar 2022 01:20:00 +0100 Subject: Code cleanup --- .ameba.yml | 4 --- spec/spec_helper.cr | 2 +- src/invidious/exceptions.cr | 8 +++++ src/invidious/search.cr | 75 ------------------------------------------ src/invidious/search/ctoken.cr | 32 ++++++++++++++++++ 5 files changed, 41 insertions(+), 80 deletions(-) delete mode 100644 src/invidious/search.cr create mode 100644 src/invidious/search/ctoken.cr (limited to 'src') diff --git a/.ameba.yml b/.ameba.yml index 247705e8..96cbc8f0 100644 --- a/.ameba.yml +++ b/.ameba.yml @@ -77,10 +77,6 @@ Metrics/CyclomaticComplexity: # process_video_params(query, preferences) => [20/10] - src/invidious/videos.cr - # produce_search_params(page, sort, ...) => [29/10] - # process_search_query(query, page, ...) => [14/10] - - src/invidious/search.cr - #src/invidious/playlists.cr:327:5 diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 09320750..6c492e2f 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -8,7 +8,7 @@ require "../src/invidious/channels/*" require "../src/invidious/videos" require "../src/invidious/comments" require "../src/invidious/playlists" -require "../src/invidious/search" +require "../src/invidious/search/ctoken" require "../src/invidious/trending" require "spectator" diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr index 490d98cd..bfaa3fd5 100644 --- a/src/invidious/exceptions.cr +++ b/src/invidious/exceptions.cr @@ -1,3 +1,11 @@ +# Exception used to hold the bogus UCID during a channel search. +class ChannelSearchException < InfoException + getter channel : String + + def initialize(@channel) + end +end + # Exception used to hold the name of the missing item # Should be used in all parsing functions class BrokenTubeException < Exception diff --git a/src/invidious/search.cr b/src/invidious/search.cr deleted file mode 100644 index e4c21bd4..00000000 --- a/src/invidious/search.cr +++ /dev/null @@ -1,75 +0,0 @@ -class ChannelSearchException < InfoException - getter channel : String - - def initialize(@channel) - end -end - -def produce_channel_search_continuation(ucid, query, page) - if page <= 1 - idx = 0_i64 - else - idx = 30_i64 * (page - 1) - end - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "search", - "6:varint" => 1_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "15:base64" => { - "3:varint" => idx, - }, - "23:varint" => 0_i64, - }, - "11:string" => query, - "35:string" => "browse-feed#{ucid}search", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def process_search_query(query, page, user, region) - # Parse legacy query - filters, channel, search_query, subscriptions = Invidious::Search::Filters.from_legacy_filters(query) - - if !channel.nil? && !channel.empty? - items = Invidious::Search::Processors.channel(search_query, page, channel) - elsif subscriptions - if user - user = user.as(Invidious::User) - items = Invidious::Search::Processors.subscriptions(query, page, user) - else - items = [] of ChannelVideo - end - else - search_params = filters.to_yt_params(page: page) - items = search(search_query, search_params, region) - end - - # Light processing to flatten search results out of Categories. - # They should ideally be supported in the future. - items_without_category = [] of SearchItem | ChannelVideo - items.each do |i| - if i.is_a? Category - i.contents.each do |nest_i| - if !nest_i.is_a? Video - items_without_category << nest_i - end - end - else - items_without_category << i - end - end - - {search_query, items_without_category, filters} -end diff --git a/src/invidious/search/ctoken.cr b/src/invidious/search/ctoken.cr new file mode 100644 index 00000000..161065e0 --- /dev/null +++ b/src/invidious/search/ctoken.cr @@ -0,0 +1,32 @@ +def produce_channel_search_continuation(ucid, query, page) + if page <= 1 + idx = 0_i64 + else + idx = 30_i64 * (page - 1) + end + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:base64" => { + "2:string" => "search", + "6:varint" => 1_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "15:base64" => { + "3:varint" => idx, + }, + "23:varint" => 0_i64, + }, + "11:string" => query, + "35:string" => "browse-feed#{ucid}search", + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation +end -- cgit v1.2.3 From 62d7abdd9e699779a7e74ed5569aa6d631004210 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 6 Apr 2022 22:23:22 +0200 Subject: Add a user friendly message for when no results are found --- assets/css/search.css | 9 +++++++++ locales/en-US.json | 4 +++- src/invidious/frontend/misc.cr | 14 ++++++++++++++ src/invidious/views/search.ecr | 27 ++++++++++++++++----------- 4 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 src/invidious/frontend/misc.cr (limited to 'src') diff --git a/assets/css/search.css b/assets/css/search.css index 226207a5..a5996362 100644 --- a/assets/css/search.css +++ b/assets/css/search.css @@ -69,6 +69,15 @@ fieldset, legend { #filters-apply { text-align: end; } +/* Error message */ + +.no-results-error { + text-align: center; + line-height: 180%; + font-size: 110%; + padding: 15px 15px 125px 15px; +} + /* Responsive rules */ @media only screen and (max-width: 800px) { diff --git a/locales/en-US.json b/locales/en-US.json index 03df88b6..58098929 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -175,7 +175,9 @@ "Show less": "Show less", "Watch on YouTube": "Watch on YouTube", "Switch Invidious Instance": "Switch Invidious Instance", - "Broken? Try another Invidious Instance": "Broken? Try another Invidious Instance", + "search_message_no_results": "No results found.", + "search_message_change_filters_or_query": "Try widening your search query and/or changing the filters.", + "search_message_use_another_instance": " You can also search on another instance.", "Hide annotations": "Hide annotations", "Show annotations": "Show annotations", "Genre: ": "Genre: ", diff --git a/src/invidious/frontend/misc.cr b/src/invidious/frontend/misc.cr new file mode 100644 index 00000000..43ba9f5c --- /dev/null +++ b/src/invidious/frontend/misc.cr @@ -0,0 +1,14 @@ +module Invidious::Frontend::Misc + extend self + + def redirect_url(env : HTTP::Server::Context) + prefs = env.get("preferences").as(Preferences) + + if prefs.automatic_instance_redirect + current_page = env.get?("current_page").as(String) + redirect_url = "/redirect?referer=#{current_page}" + else + redirect_url = "https://redirect.invidious.io#{env.request.resource}" + end + end +end diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index f1f6ab20..7110703e 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -9,18 +9,13 @@ url_prev_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page - 1}" url_next_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page + 1}" + + redirect_url = Invidious::Frontend::Misc.redirect_url(env) -%> -<% if videos.size == 0 %> -

- "><%= translate(locale, "Broken? Try another Invidious Instance!") %> -

-<%- else -%> - <%= Invidious::Frontend::SearchFilters.generate(query.filters, query.text, query.page, locale) %> -<%- end -%> - -<% if videos.size == 0 %>
<% else %>
<% end %> +<%= Invidious::Frontend::SearchFilters.generate(query.filters, query.text, query.page, locale) %> +
@@ -36,11 +31,21 @@
+<%- if videos.empty? -%> +
+
+ <%= translate(locale, "search_message_no_results") %>

+ <%= translate(locale, "search_message_change_filters_or_query") %>

+ <%= translate(locale, "search_message_use_another_instance", redirect_url) %> +
+
+<%- else -%>
- <% videos.each do |item| %> + <%- videos.each do |item| -%> <%= rendered "components/item" %> - <% end %> + <%- end -%>
+<%- end -%>
-- cgit v1.2.3