From 0e3820b634cd94a647af099805d3957cd5c8998c Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 14 Jun 2022 00:04:19 +0200 Subject: [PATCH 1/7] Add #to_http_params method to Query (Fixes #3148) --- spec/invidious/search/query_spec.cr | 42 +++++++++++++++++++++++++++++ src/invidious/routes/search.cr | 6 +++++ src/invidious/search/query.cr | 12 ++++++++- src/invidious/views/search.ecr | 10 ------- 4 files changed, 59 insertions(+), 11 deletions(-) diff --git a/spec/invidious/search/query_spec.cr b/spec/invidious/search/query_spec.cr index 4853e9e9..063b69f1 100644 --- a/spec/invidious/search/query_spec.cr +++ b/spec/invidious/search/query_spec.cr @@ -197,4 +197,46 @@ Spectator.describe Invidious::Search::Query do ) end end + + describe "#to_http_params" do + it "formats regular search" do + query = described_class.new( + HTTP::Params.parse("q=The+Simpsons+hiding+in+bush&duration=short"), + Invidious::Search::Query::Type::Regular, nil + ) + + params = query.to_http_params + + expect(params).to have_key("duration") + expect(params["duration"]?).to eq("short") + + expect(params).to have_key("q") + expect(params["q"]?).to eq("The Simpsons hiding in bush") + + # Check if there aren't other parameters + params.delete("duration") + params.delete("q") + expect(params).to be_empty + end + + it "formats channel search" do + query = described_class.new( + HTTP::Params.parse("q=channel:UC2DjFE7Xf11URZqWBigcVOQ%20multimeter"), + Invidious::Search::Query::Type::Regular, nil + ) + + params = query.to_http_params + + expect(params).to have_key("channel") + expect(params["channel"]?).to eq("UC2DjFE7Xf11URZqWBigcVOQ") + + expect(params).to have_key("q") + expect(params["q"]?).to eq("multimeter") + + # Check if there aren't other parameters + params.delete("channel") + params.delete("q") + expect(params).to be_empty + end + end end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 6f8bffea..2a9705cf 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -59,6 +59,12 @@ module Invidious::Routes::Search return error_template(500, ex) end + params = query.to_http_params + url_prev_page = "/search?#{params}&page=#{query.page - 1}" + url_next_page = "/search?#{params}&page=#{query.page + 1}" + + redirect_url = Invidious::Frontend::Misc.redirect_url(env) + env.set "search", query.text templated "search" end diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index 34b36b1d..24e79609 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -57,7 +57,7 @@ module Invidious::Search # Get the page number (also common to all search types) @page = params["page"]?.try &.to_i? || 1 - # Stop here is raw query in empty + # Stop here if raw query is empty # NOTE: maybe raise in the future? return if self.empty_raw_query? @@ -127,6 +127,16 @@ module Invidious::Search return items end + # Return the HTTP::Params corresponding to this Query (invidious format) + def to_http_params : HTTP::Params + params = @filters.to_iv_params + + params["q"] = @query + params["channel"] = @channel if !@channel.empty? + + return params + end + # TODO: clean code private def unnest_items(all_items) : Array(SearchItem) items = [] of SearchItem diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index 7110703e..254449a1 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -3,16 +3,6 @@ <% end %> -<%- - search_query_encoded = URI.encode_www_form(query.text, space_to_plus: true) - filter_params = query.filters.to_iv_params - - url_prev_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page - 1}" - url_next_page = "/search?q=#{search_query_encoded}&#{filter_params}&page=#{query.page + 1}" - - redirect_url = Invidious::Frontend::Misc.redirect_url(env) --%> - <%= Invidious::Frontend::SearchFilters.generate(query.filters, query.text, query.page, locale) %>
From 99bc230fe64512b3f87095bb8111b24e15aa4285 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 15 Jun 2022 21:14:38 +0200 Subject: [PATCH 2/7] Fix missing hash key: "availableCountries" (Closes #3047) --- src/invidious/channels/about.cr | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 565f2bca..1d7947a6 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -54,9 +54,6 @@ def get_about_info(ucid, locale) : AboutChannel banner = banners.try &.[-1]?.try &.["url"].as_s? description_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] - - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool - allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) else author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s @@ -74,13 +71,17 @@ def get_about_info(ucid, locale) : AboutChannel # end description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? - - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool - allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) end + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + + allowed_regions = initdata + .dig?("microformat", "microformatDataRenderer", "availableCountries") + .try &.as_a.map(&.as_s) || [] of String + description = !description_node.nil? ? description_node.as_s : "" description_html = HTML.escape(description) + if !description_node.nil? if description_node.as_h?.nil? description_node = text_to_parsed_content(description_node.as_s) From ce32873ef8450461ba55ec50aed93d427aa23084 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 15 Jun 2022 21:55:33 +0200 Subject: [PATCH 3/7] Remove item (video/channel/mix) thumbnail from keyboard navigation tree --- src/invidious/views/components/item.ecr | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index fb7ad1dc..4f3cf279 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -5,7 +5,7 @@ <% if !env.get("preferences").as(Preferences).thin_mode %>
- "/> + "/>
<% end %>

<%= HTML.escape(item.author) %><% if !item.author_verified.nil? && item.author_verified %> <% end %>

@@ -23,7 +23,7 @@ <% if !env.get("preferences").as(Preferences).thin_mode %>
- "/> + "/>

<%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>

<% end %> @@ -36,7 +36,7 @@ <% if !env.get("preferences").as(Preferences).thin_mode %>
- + <% if item.length_seconds != 0 %>

<%= recode_length_seconds(item.length_seconds) %>

<% end %> @@ -51,7 +51,7 @@ <% if !env.get("preferences").as(Preferences).thin_mode %>
- + <% if plid_form = env.get?("remove_playlist_items") %>
" method="post"> "> @@ -103,7 +103,7 @@ <% if !env.get("preferences").as(Preferences).thin_mode %>
- + <% if env.get? "show_watched" %> " method="post"> "> From 06af5a004e4c6cda28e5a4cc13ee47ed3cf9c155 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 15 Jun 2022 22:26:50 +0200 Subject: [PATCH 4/7] Remove useless link in item forms (buttons on thumbnail) --- src/invidious/views/components/item.ecr | 23 +++++-------------- src/invidious/views/feeds/history.ecr | 4 +--- .../views/user/subscription_manager.ecr | 4 +--- src/invidious/views/user/token_manager.ecr | 4 +--- 4 files changed, 9 insertions(+), 26 deletions(-) diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 4f3cf279..0e959ff2 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -52,15 +52,12 @@ <% if !env.get("preferences").as(Preferences).thin_mode %>
+ <% if plid_form = env.get?("remove_playlist_items") %> " method="post"> ">

- - - +

<% end %> @@ -108,24 +105,16 @@
" method="post"> ">

- - - +

<% elsif plid_form = env.get? "add_playlist_items" %>
" method="post"> ">

- - - +

<% end %> diff --git a/src/invidious/views/feeds/history.ecr b/src/invidious/views/feeds/history.ecr index 6c1243c5..471d21db 100644 --- a/src/invidious/views/feeds/history.ecr +++ b/src/invidious/views/feeds/history.ecr @@ -38,9 +38,7 @@
" method="post"> ">

- - - +

diff --git a/src/invidious/views/user/subscription_manager.ecr b/src/invidious/views/user/subscription_manager.ecr index c2a89ca2..c9801f09 100644 --- a/src/invidious/views/user/subscription_manager.ecr +++ b/src/invidious/views/user/subscription_manager.ecr @@ -39,9 +39,7 @@

" method="post"> "> - - "> - + ">

diff --git a/src/invidious/views/user/token_manager.ecr b/src/invidious/views/user/token_manager.ecr index 79f905a1..a73fa048 100644 --- a/src/invidious/views/user/token_manager.ecr +++ b/src/invidious/views/user/token_manager.ecr @@ -31,9 +31,7 @@

" method="post"> "> - - "> - + ">

From 8332ad0f1684cd5af68daeb4f7bc5cb04ff03669 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 20 Jun 2022 23:19:05 +0200 Subject: [PATCH 5/7] Fix syntax errors in shell scripts --- scripts/deploy-database.sh | 4 ++-- scripts/install-dependencies.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/deploy-database.sh b/scripts/deploy-database.sh index ed9464e6..fa24b8f0 100644 --- a/scripts/deploy-database.sh +++ b/scripts/deploy-database.sh @@ -6,7 +6,7 @@ interactive=true -if [ "$1" == "--no-interactive" ]; then +if [ "$1" = "--no-interactive" ]; then interactive=false fi @@ -21,7 +21,7 @@ sudo systemctl enable postgresql.service # Create databse and user # -if [ "$interactive" == "true" ]; then +if [ "$interactive" = "true" ]; then sudo -u postgres -- createuser -P kemal sudo -u postgres -- createdb -O kemal invidious else diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh index 27e0bf15..1e67bdaf 100644 --- a/scripts/install-dependencies.sh +++ b/scripts/install-dependencies.sh @@ -74,7 +74,7 @@ install_apt() { sudo apt-get install --yes --no-install-recommends \ libssl-dev libxml2-dev libyaml-dev libgmp-dev libevent-dev \ libpcre3-dev libreadline-dev libsqlite3-dev zlib1g-dev \ - crystal postgres git librsvg2-bin make + crystal postgresql-13 git librsvg2-bin make } install_yum() { From eb226e1dcf4ca88776aa42402e8d80fd5f14ae96 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 21 Jun 2022 01:01:25 +0200 Subject: [PATCH 6/7] Remove all backend code related to dislikes --- src/invidious/videos.cr | 41 ++++------------------------------- src/invidious/views/watch.ecr | 8 +++---- 2 files changed, 8 insertions(+), 41 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 1504e390..3204c98d 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -323,7 +323,7 @@ struct Video json.field "viewCount", self.views json.field "likeCount", self.likes - json.field "dislikeCount", self.dislikes + json.field "dislikeCount", 0_i64 json.field "paid", self.paid json.field "premium", self.premium @@ -354,7 +354,7 @@ struct Video json.field "lengthSeconds", self.length_seconds json.field "allowRatings", self.allow_ratings - json.field "rating", self.average_rating + json.field "rating", 0_i64 json.field "isListed", self.is_listed json.field "liveNow", self.live_now json.field "isUpcoming", self.is_upcoming @@ -556,11 +556,6 @@ struct Video info["dislikes"]?.try &.as_i64 || 0_i64 end - def average_rating : Float64 - # (likes / (likes + dislikes) * 4 + 1) - info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0 - end - def published : Time info .dig?("microformat", "playerMicroformatRenderer", "publishDate") @@ -813,14 +808,6 @@ struct Video return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s end - def wilson_score : Float64 - ci_lower_bound(likes, likes + dislikes).round(4) - end - - def engagement : Float64 - (((likes + dislikes) / views) * 100).round(4) - end - def reason : String? info["reason"]?.try &.as_s end @@ -1005,7 +992,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ params["relatedVideos"] = JSON::Any.new(related) - # Likes/dislikes + # Likes toplevel_buttons = video_primary_renderer .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") @@ -1023,30 +1010,10 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes end - - dislikes_button = toplevel_buttons.as_a - .find(&.dig("toggleButtonRenderer", "defaultIcon", "iconType").as_s.== "DISLIKE") - .try &.["toggleButtonRenderer"] - - if dislikes_button - dislikes_txt = (dislikes_button["defaultText"]? || dislikes_button["toggledText"]?) - .try &.dig?("accessibility", "accessibilityData", "label") - dislikes = dislikes_txt.as_s.gsub(/\D/, "").to_i64? if dislikes_txt - - LOGGER.trace("extract_video_info: Found \"dislikes\" button. Button text is \"#{dislikes_txt}\"") - LOGGER.debug("extract_video_info: Dislikes count is #{dislikes}") if dislikes - end - end - - if likes && likes != 0_i64 && (!dislikes || dislikes == 0_i64) - if rating = player_response.dig?("videoDetails", "averageRating").try { |x| x.as_i64? || x.as_f? } - dislikes = (likes * ((5 - rating)/(rating - 1))).round.to_i64 - LOGGER.debug("extract_video_info: Dislikes count (using fallback method) is #{dislikes}") - end end params["likes"] = JSON::Any.new(likes || 0_i64) - params["dislikes"] = JSON::Any.new(dislikes || 0_i64) + params["dislikes"] = JSON::Any.new(0_i64) # Description diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index d1fdcce2..50c63d21 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -173,7 +173,7 @@ we're going to need to do it here in order to allow for translations.

<%= number_with_separator(video.views) %>

<%= number_with_separator(video.likes) %>

-

+

<%= translate(locale, "Genre: ") %> <% if !video.genre_url %> <%= video.genre %> @@ -185,9 +185,9 @@ we're going to need to do it here in order to allow for translations.

<%= translate(locale, "License: ") %><%= video.license %>

<% end %>

<%= translate(locale, "Family friendly? ") %><%= translate_bool(locale, video.is_family_friendly) %>

-

<%= translate(locale, "Wilson score: ") %><%= video.wilson_score %>

-

-

<%= translate(locale, "Engagement: ") %><%= video.engagement %>%

+ + + <% if video.allowed_regions.size != REGIONS.size %>

<% if video.allowed_regions.size < REGIONS.size // 2 %> From f7b1dcc271bb14bf8962c9375c413c0cf01d880b Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 23 Jun 2022 21:32:02 +0200 Subject: [PATCH 7/7] Don't treat LIVE_STREAM_OFFLINE playability status as an error (fixes #3155) --- src/invidious/videos.cr | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 3204c98d..d9a7d846 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -895,13 +895,20 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_ player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) - if player_response.dig?("playabilityStatus", "status").try &.as_s != "OK" + playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s + + if playability_status != "OK" subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") reason = subreason.try &.[]?("simpleText").try &.as_s reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") reason ||= player_response.dig("playabilityStatus", "reason").as_s + params["reason"] = JSON::Any.new(reason) - return params + + # Stop here if video is not a scheduled livestream + if playability_status != "LIVE_STREAM_OFFLINE" + return params + end end params["shortDescription"] = player_response.dig?("videoDetails", "shortDescription") || JSON::Any.new(nil)