From 5e3c9cf2909608badae15ae9196cc5f2cab37b94 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 19 Jan 2022 18:34:35 +0100 Subject: [PATCH 01/15] Remove useless arguments from playlist-related functions --- src/invidious/playlists.cr | 36 ++++++++------------ src/invidious/routes/api/v1/authenticated.cr | 2 +- src/invidious/routes/api/v1/misc.cr | 10 +++--- src/invidious/routes/embed.cr | 8 ++--- src/invidious/routes/feeds.cr | 2 +- src/invidious/routes/playlists.cr | 10 +++--- 6 files changed, 30 insertions(+), 38 deletions(-) diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index a09e6cdbe..ecebba91b 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -90,7 +90,7 @@ struct Playlist property updated : Time property thumbnail : String? - def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil) + def to_json(offset, json : JSON::Builder, video_id : String? = nil) json.object do json.field "type", "playlist" json.field "title", self.title @@ -125,7 +125,7 @@ struct Playlist json.field "videos" do json.array do - videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id) + videos = get_playlist_videos(self, offset: offset, video_id: video_id) videos.each do |video| video.to_json(json) end @@ -134,13 +134,9 @@ struct Playlist end end - def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil) - if json - to_json(offset, locale, json, video_id: video_id) - else - JSON.build do |json| - to_json(offset, locale, json, video_id: video_id) - end + def to_json(offset, _json : Nil = nil, video_id : String? = nil) + JSON.build do |json| + to_json(offset, json, video_id: video_id) end end @@ -179,7 +175,7 @@ struct InvidiousPlaylist end end - def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil) + def to_json(offset, json : JSON::Builder, video_id : String? = nil) json.object do json.field "type", "invidiousPlaylist" json.field "title", self.title @@ -205,7 +201,7 @@ struct InvidiousPlaylist offset = self.index.index(index) || 0 end - videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id) + videos = get_playlist_videos(self, offset: offset, video_id: video_id) videos.each_with_index do |video, index| video.to_json(json, offset + index) end @@ -214,13 +210,9 @@ struct InvidiousPlaylist end end - def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil) - if json - to_json(offset, locale, json, video_id: video_id) - else - JSON.build do |json| - to_json(offset, locale, json, video_id: video_id) - end + def to_json(offset, _json : Nil = nil, video_id : String? = nil) + JSON.build do |json| + to_json(offset, json, video_id: video_id) end end @@ -320,7 +312,7 @@ def produce_playlist_continuation(id, index) return continuation end -def get_playlist(plid, locale, refresh = true, force_refresh = false) +def get_playlist(plid : String) if plid.starts_with? "IV" if playlist = Invidious::Database::Playlists.select(id: plid) return playlist @@ -328,11 +320,11 @@ def get_playlist(plid, locale, refresh = true, force_refresh = false) raise InfoException.new("Playlist does not exist.") end else - return fetch_playlist(plid, locale) + return fetch_playlist(plid) end end -def fetch_playlist(plid, locale) +def fetch_playlist(plid : String) if plid.starts_with? "UC" plid = "UU#{plid.lchop("UC")}" end @@ -402,7 +394,7 @@ def fetch_playlist(plid, locale) }) end -def get_playlist_videos(playlist, offset, locale = nil, video_id = nil) +def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, video_id = nil) # Show empy playlist if requested page is out of range # (e.g, when a new playlist has been created, offset will be negative) if offset >= playlist.video_count || offset < 0 diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index fda655ef1..19c0ee3fc 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -125,7 +125,7 @@ module Invidious::Routes::API::V1::Authenticated JSON.build do |json| json.array do playlists.each do |playlist| - playlist.to_json(0, locale, json) + playlist.to_json(0, json) end end end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index ac0576a09..d10f4fcc7 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -14,7 +14,7 @@ module Invidious::Routes::API::V1::Misc # APIv1 currently uses the same logic for both # user playlists and Invidious playlists. This means that we can't # reasonably split them yet. This should be addressed in APIv2 - def self.get_playlist(env) + def self.get_playlist(env : HTTP::Server::Context) locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" @@ -34,7 +34,7 @@ module Invidious::Routes::API::V1::Misc end begin - playlist = get_playlist(plid, locale) + playlist = get_playlist(plid) rescue ex : InfoException return error_json(404, ex) rescue ex @@ -49,7 +49,7 @@ module Invidious::Routes::API::V1::Misc # includes into the playlist a maximum of 20 videos, before the offset if offset > 0 lookback = offset < 50 ? offset : 50 - response = playlist.to_json(offset - lookback, locale) + response = playlist.to_json(offset - lookback) json_response = JSON.parse(response) else # Unless the continuation is really the offset 0, it becomes expensive. @@ -58,13 +58,13 @@ module Invidious::Routes::API::V1::Misc # it shouldn't happen often though lookback = 0 - response = playlist.to_json(offset, locale, video_id: video_id) + response = playlist.to_json(offset, video_id: video_id) json_response = JSON.parse(response) if json_response["videos"].as_a[0]["index"] != offset offset = json_response["videos"].as_a[0]["index"].as_i lookback = offset < 50 ? offset : 50 - response = playlist.to_json(offset - lookback, locale) + response = playlist.to_json(offset - lookback) json_response = JSON.parse(response) end end diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index ab722ae2a..0e9701f0f 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -6,9 +6,9 @@ module Invidious::Routes::Embed if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") begin - playlist = get_playlist(plid, locale: locale) + playlist = get_playlist(plid) offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(playlist, offset: offset, locale: locale) + videos = get_playlist_videos(playlist, offset: offset) rescue ex return error_template(500, ex) end @@ -60,9 +60,9 @@ module Invidious::Routes::Embed if plid begin - playlist = get_playlist(plid, locale: locale) + playlist = get_playlist(plid) offset = env.params.query["index"]?.try &.to_i? || 0 - videos = get_playlist_videos(playlist, offset: offset, locale: locale) + videos = get_playlist_videos(playlist, offset: offset) rescue ex return error_template(500, ex) end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index fd8c25ce9..c9271766d 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -265,7 +265,7 @@ module Invidious::Routes::Feeds if plid.starts_with? "IV" if playlist = Invidious::Database::Playlists.select(id: plid) - videos = get_playlist_videos(playlist, offset: 0, locale: locale) + videos = get_playlist_videos(playlist, offset: 0) return XML.build(indent: " ", encoding: "UTF-8") do |xml| xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index d437b79c5..7a502a058 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -66,7 +66,7 @@ module Invidious::Routes::Playlists user = user.as(User) playlist_id = env.params.query["list"] - playlist = get_playlist(playlist_id, locale) + playlist = get_playlist(playlist_id) subscribe_playlist(user, playlist) env.redirect "/playlist?list=#{playlist.id}" @@ -161,7 +161,7 @@ module Invidious::Routes::Playlists end begin - videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale) + videos = get_playlist_videos(playlist, offset: (page - 1) * 100) rescue ex videos = [] of PlaylistVideo end @@ -314,7 +314,7 @@ module Invidious::Routes::Playlists begin playlist_id = env.params.query["playlist_id"] - playlist = get_playlist(playlist_id, locale).as(InvidiousPlaylist) + playlist = get_playlist(playlist_id).as(InvidiousPlaylist) raise "Invalid user" if playlist.author != user.email rescue ex if redirect @@ -405,7 +405,7 @@ module Invidious::Routes::Playlists end begin - playlist = get_playlist(plid, locale) + playlist = get_playlist(plid) rescue ex return error_template(500, ex) end @@ -422,7 +422,7 @@ module Invidious::Routes::Playlists end begin - videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale) + videos = get_playlist_videos(playlist, offset: (page - 1) * 100) rescue ex return error_template(500, "Error encountered while retrieving playlist videos.
#{ex.message}") end From c7b74aa8b415819f44a4c32ee13ba26b9fd594c8 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 19 Jan 2022 18:47:54 +0100 Subject: [PATCH 02/15] Remove useless 'locale' argument from error template functions --- src/invidious/helpers/errors.cr | 70 ++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 26c386696..1a1366fea 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -6,8 +6,12 @@ class InfoException < Exception end +# ------------------- +# Issue template +# ------------------- + macro error_template(*args) - error_template_helper(env, locale, {{*args}}) + error_template_helper(env, {{*args}}) end def github_details(summary : String, content : String) @@ -22,11 +26,13 @@ def github_details(summary : String, content : String) return HTML.escape(details) end -def error_template_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception) +def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) if exception.is_a?(InfoException) - return error_template_helper(env, locale, status_code, exception.message || "") + return error_template_helper(env, status_code, exception.message || "") end + locale = env.get("preferences").as(Preferences).locale + env.response.content_type = "text/html" env.response.status_code = status_code @@ -77,71 +83,99 @@ def error_template_helper(env : HTTP::Server::Context, locale : String?, status_ return templated "error" end -def error_template_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String) +def error_template_helper(env : HTTP::Server::Context, status_code : Int32, message : String) env.response.content_type = "text/html" env.response.status_code = status_code + + locale = env.get("preferences").as(Preferences).locale + error_message = translate(locale, message) - next_steps = error_redirect_helper(env, locale) + next_steps = error_redirect_helper(env) + return templated "error" end +# ------------------- +# Atom feeds +# ------------------- + macro error_atom(*args) - error_atom_helper(env, locale, {{*args}}) + error_atom_helper(env, {{*args}}) end -def error_atom_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception) +def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) if exception.is_a?(InfoException) - return error_atom_helper(env, locale, status_code, exception.message || "") + return error_atom_helper(env, status_code, exception.message || "") end + env.response.content_type = "application/atom+xml" env.response.status_code = status_code + return "#{exception.inspect_with_backtrace}" end -def error_atom_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String) +def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, message : String) env.response.content_type = "application/atom+xml" env.response.status_code = status_code + return "#{message}" end +# ------------------- +# JSON +# ------------------- + macro error_json(*args) - error_json_helper(env, locale, {{*args}}) + error_json_helper(env, {{*args}}) end -def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil) +def error_json_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil) if exception.is_a?(InfoException) - return error_json_helper(env, locale, status_code, exception.message || "", additional_fields) + return error_json_helper(env, status_code, exception.message || "", additional_fields) end + env.response.content_type = "application/json" env.response.status_code = status_code + error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace} + if additional_fields error_message = error_message.merge(additional_fields) end + return error_message.to_json end -def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception) - return error_json_helper(env, locale, status_code, exception, nil) +def error_json_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) + return error_json_helper(env, status_code, exception, nil) end -def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil) +def error_json_helper(env : HTTP::Server::Context, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil) env.response.content_type = "application/json" env.response.status_code = status_code + error_message = {"error" => message} + if additional_fields error_message = error_message.merge(additional_fields) end + return error_message.to_json end -def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String) - error_json_helper(env, locale, status_code, message, nil) +def error_json_helper(env : HTTP::Server::Context, status_code : Int32, message : String) + error_json_helper(env, status_code, message, nil) end -def error_redirect_helper(env : HTTP::Server::Context, locale : String?) +# ------------------- +# Redirect +# ------------------- + +def error_redirect_helper(env : HTTP::Server::Context) request_path = env.request.path + locale = env.get("preferences").as(Preferences).locale + if request_path.starts_with?("/search") || request_path.starts_with?("/watch") || request_path.starts_with?("/channel") || request_path.starts_with?("/playlist?list=PL") next_steps_text = translate(locale, "next_steps_error_message") From 2d949834e976d433899341e46c72e7d8514ce50f Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 19 Jan 2022 22:15:43 +0100 Subject: [PATCH 03/15] Make 'additional_fields' optional in JSON error template functions This allows us to de-duplicate functions --- src/invidious/helpers/errors.cr | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 1a1366fea..3acbac840 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -129,7 +129,12 @@ macro error_json(*args) error_json_helper(env, {{*args}}) end -def error_json_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil) +def error_json_helper( + env : HTTP::Server::Context, + status_code : Int32, + exception : Exception, + additional_fields : Hash(String, Object) | Nil = nil +) if exception.is_a?(InfoException) return error_json_helper(env, status_code, exception.message || "", additional_fields) end @@ -146,11 +151,12 @@ def error_json_helper(env : HTTP::Server::Context, status_code : Int32, exceptio return error_message.to_json end -def error_json_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) - return error_json_helper(env, status_code, exception, nil) -end - -def error_json_helper(env : HTTP::Server::Context, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil) +def error_json_helper( + env : HTTP::Server::Context, + status_code : Int32, + message : String, + additional_fields : Hash(String, Object) | Nil = nil +) env.response.content_type = "application/json" env.response.status_code = status_code @@ -163,10 +169,6 @@ def error_json_helper(env : HTTP::Server::Context, status_code : Int32, message return error_message.to_json end -def error_json_helper(env : HTTP::Server::Context, status_code : Int32, message : String) - error_json_helper(env, status_code, message, nil) -end - # ------------------- # Redirect # ------------------- From fa99c9aa85ac9f0b4dc0d705b8e62a50cd4cc9a3 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 16:00:47 +0100 Subject: [PATCH 04/15] Use '.dig?()' in playlist parsing --- src/invidious/playlists.cr | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index ecebba91b..afbc86242 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -331,10 +331,10 @@ def fetch_playlist(plid : String) initial_data = YoutubeAPI.browse("VL" + plid, params: "") - playlist_sidebar_renderer = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]? + playlist_sidebar_renderer = initial_data.dig?("sidebar", "playlistSidebarRenderer", "items") raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer - playlist_info = playlist_sidebar_renderer[0]["playlistSidebarPrimaryInfoRenderer"]? + playlist_info = playlist_sidebar_renderer.dig?(0, "playlistSidebarPrimaryInfoRenderer") raise InfoException.new("Could not extract playlist info") if !playlist_info title = playlist_info.dig?("title", "runs", 0, "text").try &.as_s || "" @@ -347,12 +347,15 @@ def fetch_playlist(plid : String) description_html = desc_item.try &.["runs"]?.try &.as_a .try { |run| content_to_comment_html(run).try &.to_s } || "

" - thumbnail = playlist_info["thumbnailRenderer"]?.try &.["playlistVideoThumbnailRenderer"]? - .try &.["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s + thumbnail = playlist_info.dig?( + "thumbnailRenderer", "playlistVideoThumbnailRenderer", + "thumbnail", "thumbnails", 0, "url" + ).try &.as_s views = 0_i64 updated = Time.utc video_count = 0 + playlist_info["stats"]?.try &.as_a.each do |stat| text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s next if !text @@ -371,12 +374,15 @@ def fetch_playlist(plid : String) author_thumbnail = "" ucid = "" else - author_info = playlist_sidebar_renderer[1]["playlistSidebarSecondaryInfoRenderer"]?.try &.["videoOwner"]["videoOwnerRenderer"]? + author_info = playlist_sidebar_renderer[1].dig?( + "playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer" + ) + raise InfoException.new("Could not extract author info") if !author_info - author = author_info["title"]["runs"][0]["text"]?.try &.as_s || "" - author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || "" - ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || "" + author = author_info.dig?("title", "runs", 0, "text").try &.as_s || "" + author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url").try &.as_s || "" + ucid = author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || "" end return Playlist.new({ From 4cd7a3e83f904f763497f214c52d52237d99228b Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 16:03:04 +0100 Subject: [PATCH 05/15] Remove useless 'locale = env.get...' from many routes --- src/invidious/routes/api/v1/authenticated.cr | 13 +------------ src/invidious/routes/api/v1/misc.cr | 3 --- src/invidious/routes/api/v1/videos.cr | 6 ------ src/invidious/routes/embed.cr | 3 --- 4 files changed, 1 insertion(+), 24 deletions(-) diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 19c0ee3fc..5d2b4c1c2 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -115,8 +115,6 @@ module Invidious::Routes::API::V1::Authenticated end def self.list_playlists(env) - locale = env.get("preferences").as(Preferences).locale - env.response.content_type = "application/json" user = env.get("user").as(User) @@ -134,7 +132,6 @@ module Invidious::Routes::API::V1::Authenticated def self.create_playlist(env) env.response.content_type = "application/json" user = env.get("user").as(User) - locale = env.get("preferences").as(Preferences).locale title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150) if !title @@ -160,8 +157,6 @@ module Invidious::Routes::API::V1::Authenticated end def self.update_playlist_attribute(env) - locale = env.get("preferences").as(Preferences).locale - env.response.content_type = "application/json" user = env.get("user").as(User) @@ -197,8 +192,6 @@ module Invidious::Routes::API::V1::Authenticated end def self.delete_playlist(env) - locale = env.get("preferences").as(Preferences).locale - env.response.content_type = "application/json" user = env.get("user").as(User) @@ -219,8 +212,6 @@ module Invidious::Routes::API::V1::Authenticated end def self.insert_video_into_playlist(env) - locale = env.get("preferences").as(Preferences).locale - env.response.content_type = "application/json" user = env.get("user").as(User) @@ -274,8 +265,6 @@ module Invidious::Routes::API::V1::Authenticated end def self.delete_video_in_playlist(env) - locale = env.get("preferences").as(Preferences).locale - env.response.content_type = "application/json" user = env.get("user").as(User) @@ -389,8 +378,8 @@ module Invidious::Routes::API::V1::Authenticated end def self.unregister_token(env) - locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" + user = env.get("user").as(User) scopes = env.get("scopes").as(Array(String)) diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index d10f4fcc7..a1ce0cbc5 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -1,7 +1,6 @@ module Invidious::Routes::API::V1::Misc # Stats API endpoint for Invidious def self.stats(env) - locale = env.get("preferences").as(Preferences).locale env.response.content_type = "application/json" if !CONFIG.statistics_enabled @@ -15,8 +14,6 @@ module Invidious::Routes::API::V1::Misc # user playlists and Invidious playlists. This means that we can't # reasonably split them yet. This should be addressed in APIv2 def self.get_playlist(env : HTTP::Server::Context) - locale = env.get("preferences").as(Preferences).locale - env.response.content_type = "application/json" plid = env.params.url["plid"] diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 3a013ba08..be0f699be 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -20,8 +20,6 @@ module Invidious::Routes::API::V1::Videos end def self.captions(env) - locale = env.get("preferences").as(Preferences).locale - env.response.content_type = "application/json" id = env.params.url["id"] @@ -149,8 +147,6 @@ module Invidious::Routes::API::V1::Videos # thumbnails for individual scenes in a video. # See https://support.jwplayer.com/articles/how-to-add-preview-thumbnails def self.storyboards(env) - locale = env.get("preferences").as(Preferences).locale - env.response.content_type = "application/json" id = env.params.url["id"] @@ -223,8 +219,6 @@ module Invidious::Routes::API::V1::Videos end def self.annotations(env) - locale = env.get("preferences").as(Preferences).locale - env.response.content_type = "text/xml" id = env.params.url["id"] diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 0e9701f0f..207970b09 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -2,8 +2,6 @@ module Invidious::Routes::Embed def self.redirect(env) - locale = env.get("preferences").as(Preferences).locale - if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") begin playlist = get_playlist(plid) @@ -26,7 +24,6 @@ module Invidious::Routes::Embed end def self.show(env) - locale = env.get("preferences").as(Preferences).locale id = env.params.url["id"] plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") From 1c91110464e70e053282a73956754df97559d3d6 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 16:15:59 +0100 Subject: [PATCH 06/15] Fix some 'Lint/ShadowingOuterLocalVar' warnings reported by ameba --- src/invidious/routes/api/v1/authenticated.cr | 4 ++-- src/invidious/routes/api/v1/videos.cr | 6 ++--- src/invidious/routes/video_playback.cr | 24 ++++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 5d2b4c1c2..5da6143ba 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -138,7 +138,7 @@ module Invidious::Routes::API::V1::Authenticated return error_json(400, "Invalid title.") end - privacy = env.params.json["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String).downcase) } + privacy = env.params.json["privacy"]?.try { |p| PlaylistPrivacy.parse(p.as(String).downcase) } if !privacy return error_json(400, "Invalid privacy setting.") end @@ -175,7 +175,7 @@ module Invidious::Routes::API::V1::Authenticated end title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title - privacy = env.params.json["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String).downcase) } || playlist.privacy + privacy = env.params.json["privacy"]?.try { |p| PlaylistPrivacy.parse(p.as(String).downcase) } || playlist.privacy description = env.params.json["description"]?.try &.as(String).delete("\r") || playlist.description if title != playlist.title || diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index be0f699be..86eb26eea 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -71,9 +71,9 @@ module Invidious::Routes::API::V1::Videos env.response.content_type = "text/vtt; charset=UTF-8" if lang - caption = captions.select { |caption| caption.language_code == lang } + caption = captions.select(&.language_code.== lang) else - caption = captions.select { |caption| caption.name == label } + caption = captions.select(&.name.== label) end if caption.empty? @@ -179,7 +179,7 @@ module Invidious::Routes::API::V1::Videos env.response.content_type = "text/vtt" - storyboard = storyboards.select { |storyboard| width == "#{storyboard[:width]}" || height == "#{storyboard[:height]}" } + storyboard = storyboards.select { |sb| width == "#{sb[:width]}" || height == "#{sb[:height]}" } if storyboard.empty? haltf env, 404 diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 8a58b034f..f6340c579 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -75,8 +75,8 @@ module Invidious::Routes::VideoPlayback end begin - client.get(url, headers) do |response| - response.headers.each do |key, value| + client.get(url, headers) do |resp| + resp.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) env.response.headers[key] = value end @@ -84,7 +84,7 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" - if location = response.headers["Location"]? + if location = resp.headers["Location"]? location = URI.parse(location) location = "#{location.request_target}&host=#{location.host}" @@ -95,7 +95,7 @@ module Invidious::Routes::VideoPlayback return env.redirect location end - IO.copy(response.body_io, env.response) + IO.copy(resp.body_io, env.response) end rescue ex end @@ -132,15 +132,15 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{chunk_start}-#{chunk_end}" begin - client.get(url, headers) do |response| + client.get(url, headers) do |resp| if first_chunk - if !env.request.headers["Range"]? && response.status_code == 206 + if !env.request.headers["Range"]? && resp.status_code == 206 env.response.status_code = 200 else - env.response.status_code = response.status_code + env.response.status_code = resp.status_code end - response.headers.each do |key, value| + resp.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range" env.response.headers[key] = value end @@ -148,7 +148,7 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" - if location = response.headers["Location"]? + if location = resp.headers["Location"]? location = URI.parse(location) location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -161,8 +161,8 @@ module Invidious::Routes::VideoPlayback env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}" end - if !response.headers.includes_word?("Transfer-Encoding", "chunked") - content_length = response.headers["Content-Range"].split("/")[-1].to_i64 + if !resp.headers.includes_word?("Transfer-Encoding", "chunked") + content_length = resp.headers["Content-Range"].split("/")[-1].to_i64 if env.request.headers["Range"]? env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}" env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start @@ -172,7 +172,7 @@ module Invidious::Routes::VideoPlayback end end - proxy_file(response, env) + proxy_file(resp, env) end rescue ex if ex.message != "Error reading socket: Connection reset by peer" From 12b818a83ce119cc4b3943b01cf7d6353eaa664e Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 17:17:22 +0100 Subject: [PATCH 07/15] Fix more 'Lint/ShadowingOuterLocalVar' warnings reported by ameba --- src/invidious/channels/about.cr | 5 +++-- src/invidious/helpers/i18n.cr | 4 ++-- src/invidious/helpers/json_filter.cr | 4 ++-- src/invidious/helpers/static_file_handler.cr | 6 +++--- src/invidious/helpers/tokens.cr | 3 +++ src/invidious/helpers/utils.cr | 4 ++-- src/invidious/playlists.cr | 4 ++-- src/invidious/routes/api/manifest.cr | 2 +- src/invidious/routes/login.cr | 4 ++-- src/invidious/videos.cr | 4 ++-- 10 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 8cae7ae2a..0f3928f5d 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -96,7 +96,7 @@ def get_about_info(ucid, locale) : AboutChannel total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, node| acc + node["text"].as_s } + joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s } .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) # Normal Auto-generated channels @@ -136,7 +136,8 @@ def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedCha channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any - tab = tabs.find { |tab| tab.dig?("tabRenderer", "title").try(&.as_s?) == "Channels" } + tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels")) + return [] of AboutRelatedChannel if tab.nil? items = tab.dig?("tabRenderer", "content", "sectionListRenderer", "contents", 0, "itemSectionRenderer", "contents", 0, "gridRenderer", "items").try(&.as_a?) || [] of JSON::Any diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index e88e44911..3cf9ad1c4 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -94,8 +94,8 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin translation = "" match_length = 0 - raw_data.as_h.each do |key, value| - if md = text.try &.match(/#{key}/) + raw_data.as_h.each do |hash_key, value| + if md = text.try &.match(/#{hash_key}/) if md[0].size >= match_length translation = value.as_s match_length = md[0].size diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr index e4b57cea1..b8e8f96d0 100644 --- a/src/invidious/helpers/json_filter.cr +++ b/src/invidious/helpers/json_filter.cr @@ -98,9 +98,9 @@ module JSONFilter end end - group_name.split('/').each do |group_name| + group_name.split('/').each do |name| nest_stack.push({ - group_name: group_name, + group_name: name, closing_bracket_index: closing_bracket_index, }) end diff --git a/src/invidious/helpers/static_file_handler.cr b/src/invidious/helpers/static_file_handler.cr index 630c2fd26..6b3cd22e0 100644 --- a/src/invidious/helpers/static_file_handler.cr +++ b/src/invidious/helpers/static_file_handler.cr @@ -175,9 +175,9 @@ module Kemal if @cached_files.sum(&.[1][:data].bytesize) + (size = File.size(file_path)) < CACHE_LIMIT data = Bytes.new(size) - File.open(file_path) do |file| - file.read(data) - end + + File.open(file_path) { |f| f.read(data) } + filestat = File.info(file_path) @cached_files[file_path] = {data: data, filestat: filestat} diff --git a/src/invidious/helpers/tokens.cr b/src/invidious/helpers/tokens.cr index 8b076e39e..9b6646464 100644 --- a/src/invidious/helpers/tokens.cr +++ b/src/invidious/helpers/tokens.cr @@ -42,6 +42,9 @@ end def sign_token(key, hash) string_to_sign = [] of String + # TODO: figure out which "key" variable is used + # Ameba reports a warning for "Lint/ShadowingOuterLocalVar" on this + # variable, but its preferrable to not touch that (works fine atm). hash.each do |key, value| next if key == "signature" diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 09181c10c..3ab9a0fc3 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -292,8 +292,8 @@ def parse_range(range) end ranges = range.lchop("bytes=").split(',') - ranges.each do |range| - start_range, end_range = range.split('-') + ranges.each do |r| + start_range, end_range = r.split('-') start_range = start_range.to_i64? || 0_i64 end_range = end_range.to_i64? diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index afbc86242..177cee0f5 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -202,8 +202,8 @@ struct InvidiousPlaylist end videos = get_playlist_videos(self, offset: offset, video_id: video_id) - videos.each_with_index do |video, index| - video.to_json(json, offset + index) + videos.each_with_index do |video, idx| + video.to_json(json, offset + idx) end end end diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index b61830011..ca429df55 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -98,7 +98,7 @@ module Invidious::Routes::API::Manifest height = fmt["height"].as_i # Resolutions reported by YouTube player (may not accurately reflect source) - height = potential_heights.min_by { |i| (height - i).abs } + height = potential_heights.min_by { |x| (height - x).abs } next if unique_res && heights.includes? height heights << height diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 64da3e4ee..f4859e6fc 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -425,9 +425,9 @@ module Invidious::Routes::Login found_valid_captcha = false error_exception = Exception.new - tokens.each do |token| + tokens.each do |tok| begin - validate_request(token, answer, env.request, HMAC_KEY, locale) + validate_request(tok, answer, env.request, HMAC_KEY, locale) found_valid_captcha = true rescue ex error_exception = ex diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 499ed94d5..5cec11758 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -661,8 +661,8 @@ struct Video url = URI.parse(storyboards.shift) params = HTTP::Params.parse(url.query || "") - storyboards.each_with_index do |storyboard, i| - width, height, count, storyboard_width, storyboard_height, interval, _, sigh = storyboard.split("#") + storyboards.each_with_index do |sb, i| + width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#") params["sigh"] = sigh url.query = params.to_s From dee20f92a762048775699110a8cb320506d0084d Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 17:18:17 +0100 Subject: [PATCH 08/15] Avoid infinite loop in ChannelVideo's to_xml/to_json methods --- src/invidious/channels/channels.cr | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 155ec5597..46e34dd65 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -44,13 +44,9 @@ struct ChannelVideo end end - def to_json(locale, json : JSON::Builder | Nil = nil) - if json + def to_json(locale, _json : Nil = nil) + JSON.build do |json| to_json(locale, json) - else - JSON.build do |json| - to_json(locale, json) - end end end @@ -88,13 +84,9 @@ struct ChannelVideo end end - def to_xml(locale, xml : XML::Builder | Nil = nil) - if xml + def to_xml(locale, _xml : Nil = nil) + XML.build do |xml| to_xml(locale, xml) - else - XML.build do |xml| - to_xml(locale, xml) - end end end From e85bc3e0f6e11683fe71443c17ce5c0b991f3f0a Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 18:04:25 +0100 Subject: [PATCH 09/15] Remove script for locales propagation Not required anymore now that weblate takes care of it --- scripts/git/pre-commit | 6 -- scripts/propagate-new-locale-keys.cr | 95 ---------------------------- 2 files changed, 101 deletions(-) delete mode 100644 scripts/propagate-new-locale-keys.cr diff --git a/scripts/git/pre-commit b/scripts/git/pre-commit index e4a27750b..4460b670e 100644 --- a/scripts/git/pre-commit +++ b/scripts/git/pre-commit @@ -15,9 +15,3 @@ if [ ! -z "$changed_cr_files" ]; then git add $changed_cr_files fi - -# Locale equalizer -if [ ! -z $(git diff --name-only --cached -- locales/) ]; then - crystal run scripts/propagate-new-locale-keys.cr - git add locales > /dev/null -fi \ No newline at end of file diff --git a/scripts/propagate-new-locale-keys.cr b/scripts/propagate-new-locale-keys.cr deleted file mode 100644 index 570b408a7..000000000 --- a/scripts/propagate-new-locale-keys.cr +++ /dev/null @@ -1,95 +0,0 @@ -require "json" -require "../src/invidious/helpers/i18n.cr" - -def locale_to_array(locale_name) - arrayifed_locale_data = [] of Tuple(String, JSON::Any | String) - keys_only_array = [] of String - LOCALES[locale_name].each do |k, v| - if v.as_h? - arrayifed_locale_data << {k, JSON.parse(v.as_h.to_json)} - elsif v.as_s? - arrayifed_locale_data << {k, v.as_s} - end - - keys_only_array << k - end - - return arrayifed_locale_data, keys_only_array -end - -# Invidious currently has some unloaded localization files. We shouldn't need to propagate new keys onto those. -# We'll also remove the reference locale (english) from the list to process. -loaded_locales = LOCALES.keys.select! { |key| key != "en-US" } -english_locale, english_locale_keys = locale_to_array("en-US") - -# In order to automatically propagate locale keys we're going to be needing two arrays. -# One is an array containing each locale data encoded as tuples. The other would contain -# sets of only the keys of each locale files. -# -# The second array is to make sure that an key from the english reference file is present -# in whatever the current locale we're scanning is. -locale_list = [] of Array(Tuple(String, JSON::Any | String)) -locale_list_with_only_keys = [] of Array(String) - -# Populates the created arrays from above -loaded_locales.each do |name| - arrayifed_locale_data, keys_only_locale = locale_to_array(name) - - locale_list << arrayifed_locale_data - locale_list_with_only_keys << keys_only_locale -end - -# Propagate additions -locale_list_with_only_keys.dup.each_with_index do |keys_of_locale_in_processing, index_of_locale_in_processing| - insert_at = {} of Int32 => Tuple(String, JSON::Any | String) - - LOCALES["en-US"].each_with_index do |ref_locale_data, ref_locale_key_index| - ref_locale_key, ref_locale_value = ref_locale_data - - # Found an new key that isn't present in the current locale.. - if !keys_of_locale_in_processing.includes? ref_locale_key - # In terms of structure there's currently only two types; one for plural and the other for singular translations. - if ref_locale_value.as_h? - insert_at[ref_locale_key_index] = {ref_locale_key, JSON.parse({"([^.,0-9]|^)1([^.,0-9]|$)" => "", "" => ""}.to_json)} - else - insert_at[ref_locale_key_index] = {ref_locale_key, ""} - end - end - end - - insert_at.each do |location_to_insert, data| - locale_list_with_only_keys[index_of_locale_in_processing].insert(location_to_insert, data[0]) - locale_list[index_of_locale_in_processing].insert(location_to_insert, data) - end -end - -# Propagate removals -locale_list_with_only_keys.dup.each_with_index do |keys_of_locale_in_processing, index_of_locale_in_processing| - remove_at = [] of Int32 - - keys_of_locale_in_processing.each_with_index do |current_key, current_key_index| - if !english_locale_keys.includes? current_key - remove_at << current_key_index - end - end - - remove_at.each do |index_to_remove_at| - locale_list_with_only_keys[index_of_locale_in_processing].delete_at(index_to_remove_at) - locale_list[index_of_locale_in_processing].delete_at(index_to_remove_at) - end -end - -# Now we convert back to our original format. -final_locale_list = [] of String -locale_list.each do |locale| - intermediate_hash = {} of String => (JSON::Any | String) - locale.each { |k, v| intermediate_hash[k] = v } - final_locale_list << intermediate_hash.to_pretty_json(indent = " ") -end - -locale_map = Hash.zip(loaded_locales, final_locale_list) - -# Export -locale_map.each do |locale_name, locale_contents| - File.write("locales/#{locale_name}.json", "#{locale_contents}\n") -end From 46f7ca9ffae8f2d00bb0ef4ad7a2c2dbd344d047 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 18:05:16 +0100 Subject: [PATCH 10/15] Remove useless intermediary variable in youtube_api.cr This fixes an ameba warning --- src/invidious/yt_backend/youtube_api.cr | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 85239e728..426c076ac 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -416,10 +416,9 @@ module YoutubeAPI # Send the POST request if {{ !flag?(:disable_quic) }} && CONFIG.use_quic # Using QUIC client - response = YT_POOL.client(client_config.proxy_region, + body = YT_POOL.client(client_config.proxy_region, &.post(url, headers: headers, body: data.to_json) - ) - body = response.body + ).body else # Using HTTP client body = YT_POOL.client(client_config.proxy_region) do |client| From 971b6ec96f2a99d8df4d9535d80d71b51868e077 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 20:29:14 +0100 Subject: [PATCH 11/15] Fix 'Lint/UselessAssign' warnings reported by ameba --- src/invidious/comments.cr | 4 ---- src/invidious/playlists.cr | 1 - src/invidious/users.cr | 12 +++++------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index dda924402..65f4b135b 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -93,10 +93,6 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b end contents = body["contents"]? header = body["header"]? - if body["continuations"]? - # Removable? Doesn't seem like this is used. - more_replies_continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s - end else raise InfoException.new("Could not fetch comments") end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 177cee0f5..88888a65a 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -463,7 +463,6 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s index = i["navigationEndpoint"]["watchEndpoint"]["index"].as_i64 - thumbnail = i["thumbnail"]["thumbnails"][0]["url"].as_s title = i["title"].try { |t| t["simpleText"]? || t["runs"]?.try &.[0]["text"]? }.try &.as_s || "" author = i["shortBylineText"]?.try &.["runs"][0]["text"].as_s || "" ucid = i["shortBylineText"]?.try &.["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s || "" diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 490749943..213f56227 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -65,7 +65,6 @@ def fetch_user(sid, headers) feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) feed = XML.parse_html(feed.body) - channels = [] of String channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel| if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"] nil @@ -157,12 +156,11 @@ def generate_captcha(key) END_SVG - image = "" - convert = Process.run(%(rsvg-convert -w 400 -h 400 -b none -f png), shell: true, - input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe) do |proc| - image = proc.output.gets_to_end - image = Base64.strict_encode(image) - image = "data:image/png;base64,#{image}" + image = "data:image/png;base64," + image += Process.run(%(rsvg-convert -w 400 -h 400 -b none -f png), shell: true, + input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe + ) do |proc| + Base64.strict_encode(proc.output.gets_to_end) end answer = "#{hour}:#{minute.to_s.rjust(2, '0')}:#{second.to_s.rjust(2, '0')}" From 84cc7322810e024bd0b5c0c05d752cc3476ad717 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 21:44:18 +0100 Subject: [PATCH 12/15] search functions: Don't return result count This is useless, as the items count can be directly acessed using the '.size' method, so use that instead when needed. --- src/invidious/routes/api/v1/channels.cr | 2 +- src/invidious/routes/api/v1/search.cr | 2 +- src/invidious/routes/playlists.cr | 4 +--- src/invidious/routes/search.cr | 2 +- src/invidious/search.cr | 21 +++++++++------------ src/invidious/views/add_playlist_items.ecr | 2 +- src/invidious/views/search.ecr | 8 ++++---- 7 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 322ac42e8..3e55b4120 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -254,7 +254,7 @@ module Invidious::Routes::API::V1::Channels page = env.params.query["page"]?.try &.to_i? page ||= 1 - count, search_results = channel_search(query, page, ucid) + search_results = channel_search(query, page, ucid) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index a3b6c795f..0b0853b14 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -32,7 +32,7 @@ module Invidious::Routes::API::V1::Search return error_json(400, ex) end - count, search_results = search(query, search_params, region).as(Tuple) + search_results = search(query, search_params, region) JSON.build do |json| json.array do search_results.each do |item| diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 7a502a058..9c73874e0 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -247,15 +247,13 @@ module Invidious::Routes::Playlists query = env.params.query["q"]? if query begin - search_query, count, items, operators = process_search_query(query, page, user, region: nil) + search_query, items, operators = process_search_query(query, page, user, region: nil) videos = items.select(SearchVideo).map(&.as(SearchVideo)) rescue ex videos = [] of SearchVideo - count = 0 end else videos = [] of SearchVideo - count = 0 end env.set "add_playlist_items", plid diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 5e606adf7..3f4c7e5ec 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -54,7 +54,7 @@ module Invidious::Routes::Search user = env.get? "user" begin - search_query, count, videos, operators = process_search_query(query, page, user, region: region) + search_query, videos, operators = process_search_query(query, page, user, region: region) rescue ex : ChannelSearchException return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 0f6dc6eba..45d290597 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -5,7 +5,7 @@ class ChannelSearchException < InfoException end end -def channel_search(query, page, channel) +def channel_search(query, page, channel) : Array(SearchItem) response = YT_POOL.client &.get("/channel/#{channel}") if response.status_code == 404 @@ -24,7 +24,7 @@ def channel_search(query, page, channel) continuation_items = response_json["onResponseReceivedActions"]? .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - return 0, [] of SearchItem if !continuation_items + return [] of SearchItem if !continuation_items items = [] of SearchItem continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each { |item| @@ -32,17 +32,16 @@ def channel_search(query, page, channel) .try { |t| items << t } } - return items.size, items + return items end -def search(query, search_params = produce_search_params(content_type: "all"), region = nil) - return 0, [] of SearchItem if query.empty? +def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem) + return [] of SearchItem if query.empty? client_config = YoutubeAPI::ClientConfig.new(region: region) initial_data = YoutubeAPI.search(query, search_params, client_config: client_config) - items = extract_items(initial_data) - return items.size, items + return extract_items(initial_data) end def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "", @@ -217,7 +216,7 @@ def process_search_query(query, page, user, region) search_query = (query.split(" ") - operators).join(" ") if channel - count, items = channel_search(search_query, page, channel) + items = channel_search(search_query, page, channel) elsif subscriptions if view_name items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM ( @@ -227,16 +226,14 @@ def process_search_query(query, page, user, region) as document FROM #{view_name} ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo) - count = items.size else items = [] of ChannelVideo - count = 0 end else search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, duration: duration, features: features) - count, items = search(search_query, search_params, region).as(Tuple) + items = search(search_query, search_params, region) end # Light processing to flatten search results out of Categories. @@ -254,5 +251,5 @@ def process_search_query(query, page, user, region) end end - {search_query, items_without_category.size, items_without_category, operators} + {search_query, items_without_category, operators} end diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr index c62861b06..ad50909aa 100644 --- a/src/invidious/views/add_playlist_items.ecr +++ b/src/invidious/views/add_playlist_items.ecr @@ -48,7 +48,7 @@
- <% if count >= 20 %> + <% if videos.size >= 20 %> <%= translate(locale, "Next page") %> diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index db374548e..45bbdefcb 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -5,7 +5,7 @@ <% search_query_encoded = env.get?("search").try { |x| URI.encode_www_form(x.as(String), space_to_plus: true) } %> -<% if count == 0 %> +<% if videos.size == 0 %>

"><%= translate(locale, "Broken? Try another Invidious Instance!") %>

@@ -98,7 +98,7 @@ <% end %> -<% if count == 0 %> +<% if videos.size == 0 %>
<% else %>
@@ -114,7 +114,7 @@
- <% if count >= 20 %> + <% if videos.size >= 20 %> <%= translate(locale, "Next page") %> @@ -138,7 +138,7 @@
- <% if count >= 20 %> + <% if videos.size >= 20 %> <%= translate(locale, "Next page") %> From 63e1165936339264981975614e189dbcad9f7f7e Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 22:22:48 +0100 Subject: [PATCH 13/15] videos.cr: use '.dig?()' where possible --- src/invidious/videos.cr | 45 +++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 5cec11758..bdd7381c6 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -497,7 +497,7 @@ struct Video end def length_seconds : Int32 - info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["lengthSeconds"]?.try &.as_s.to_i || + info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i || info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0 end @@ -519,7 +519,9 @@ struct Video end def published : Time - info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc + info + .dig?("microformat", "playerMicroformatRenderer", "publishDate") + .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc end def published=(other : Time) @@ -545,8 +547,9 @@ struct Video end def premiere_timestamp : Time? - info["microformat"]?.try &.["playerMicroformatRenderer"]? - .try &.["liveBroadcastDetails"]?.try &.["startTimestamp"]?.try { |t| Time.parse_rfc3339(t.as_s) } + info + .dig?("microformat", "playerMicroformatRenderer", "liveBroadcastDetails", "startTimestamp") + .try { |t| Time.parse_rfc3339(t.as_s) } end def keywords @@ -558,8 +561,9 @@ struct Video end def allowed_regions - info["microformat"]?.try &.["playerMicroformatRenderer"]? - .try &.["availableCountries"]?.try &.as_a.map &.as_s || [] of String + info + .dig("microformat", "playerMicroformatRenderer", "availableCountries") + .try &.as_a.map &.as_s || [] of String end def author_thumbnail : String @@ -621,18 +625,11 @@ struct Video end def storyboards - storyboards = info["storyboards"]? - .try &.as_h - .try &.["playerStoryboardSpecRenderer"]? - .try &.["spec"]? - .try &.as_s.split("|") + storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec") + .try &.as_s.split("|") if !storyboards - if storyboard = info["storyboards"]? - .try &.as_h - .try &.["playerLiveStoryboardSpecRenderer"]? - .try &.["spec"]? - .try &.as_s + if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s return [{ url: storyboard.split("#")[0], width: 106, @@ -690,9 +687,8 @@ struct Video end def paid - reason = info["playabilityStatus"]?.try &.["reason"]? - paid = reason == "This video requires payment to watch." ? true : false - paid + reason = info.dig?("playabilityStatus", "reason") || "" + return reason.includes? "requires payment" end def premium @@ -716,8 +712,9 @@ struct Video end def description - description = info["microformat"]?.try &.["playerMicroformatRenderer"]? - .try &.["description"]?.try &.["simpleText"]?.try &.as_s || "" + description = info! + .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText") + .try &.as_s || "" end # TODO @@ -738,11 +735,11 @@ struct Video end def hls_manifest_url : String? - info["streamingData"]?.try &.["hlsManifestUrl"]?.try &.as_s + info.dig?("streamingData", "hlsManifestUrl").try &.as_s end def dash_manifest_url - info["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s + info.dig?("streamingData", "dashManifestUrl").try &.as_s end def genre : String @@ -758,7 +755,7 @@ struct Video end def is_family_friendly : Bool - info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false + info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false end def is_vr : Bool? From 6f4665588f0bb096576221d20fbfc2dd758d7b96 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 20 Jan 2022 22:23:21 +0100 Subject: [PATCH 14/15] search.cr: use do/end rather than inline {} block --- src/invidious/search.cr | 7 +++---- src/invidious/videos.cr | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 45d290597..d8971e791 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -27,10 +27,9 @@ def channel_search(query, page, channel) : Array(SearchItem) return [] of SearchItem if !continuation_items items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each { |item| - extract_item(item["itemSectionRenderer"]["contents"].as_a[0]) - .try { |t| items << t } - } + continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| + extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } + end return items end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index bdd7381c6..d77d56d2a 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -687,7 +687,7 @@ struct Video end def paid - reason = info.dig?("playabilityStatus", "reason") || "" + reason = info.dig?("playabilityStatus", "reason").try &.as_s || "" return reason.includes? "requires payment" end @@ -712,7 +712,7 @@ struct Video end def description - description = info! + description = info .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText") .try &.as_s || "" end From 519c227c4f4df81522e38b34e6a5ddfdb73f0def Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 29 Jan 2022 13:43:26 +0100 Subject: [PATCH 15/15] Use short syntax for 'File.open' block --- src/invidious/helpers/static_file_handler.cr | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/invidious/helpers/static_file_handler.cr b/src/invidious/helpers/static_file_handler.cr index 6b3cd22e0..6ef2d74c5 100644 --- a/src/invidious/helpers/static_file_handler.cr +++ b/src/invidious/helpers/static_file_handler.cr @@ -175,8 +175,7 @@ module Kemal if @cached_files.sum(&.[1][:data].bytesize) + (size = File.size(file_path)) < CACHE_LIMIT data = Bytes.new(size) - - File.open(file_path) { |f| f.read(data) } + File.open(file_path, &.read(data)) filestat = File.info(file_path)