From 1aa154b9787eddcdee960d06aed4c1c91f17c1c3 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 16 Nov 2024 22:33:28 +0100 Subject: [PATCH] separate invidious_companion logic + better config.yaml config --- src/invidious/config.cr | 29 ++++++- src/invidious/routes/api/manifest.cr | 8 +- src/invidious/routes/embed.cr | 7 ++ src/invidious/routes/video_playback.cr | 8 +- src/invidious/routes/watch.cr | 6 +- src/invidious/videos.cr | 6 +- src/invidious/videos/parser.cr | 19 ++--- src/invidious/views/components/player.ecr | 6 +- src/invidious/yt_backend/youtube_api.cr | 95 ++++++++++++++--------- 9 files changed, 119 insertions(+), 65 deletions(-) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 7cd4bbdc..29f4b60c 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -67,6 +67,28 @@ end class Config include YAML::Serializable + module URIArrayConverter + def self.to_yaml(values : Array(URI), yaml : YAML::Nodes::Builder) + yaml.sequence do + values.each { |v| yaml.scalar v.to_s } + end + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(URI) + if node.is_a?(YAML::Nodes::Sequence) + node.map do |child| + unless child.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{child.class}" + end + + URI.parse(child.value) + end + else + node.raise "Expected sequence, not #{node.class}" + end + end + end + # Number of threads to use for crawling videos from channels (for updating subscriptions) property channel_threads : Int32 = 1 # Time interval between two executions of the job that crawls channel videos (subscriptions update). @@ -152,10 +174,11 @@ class Config property po_token : String? = nil # Invidious companion - property invidious_companion : Array(String)? = nil + @[YAML::Field(converter: Config::URIArrayConverter)] + property invidious_companion : Array(URI) = [] of URI # Invidious companion API key - property invidious_companion_key : String? = nil + property invidious_companion_key : String = "" # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] @@ -228,7 +251,7 @@ class Config end {% end %} - if CONFIG.invidious_companion + if !CONFIG.invidious_companion.empty? # invidious_companion and signature_server can't work together if CONFIG.signature_server puts "Config: You can not run inv_sig_helper and invidious_companion at the same time." diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index 43d0eb2f..2892d1b4 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -1,6 +1,10 @@ module Invidious::Routes::API::Manifest # /api/manifest/dash/id/:id def self.get_dash_video_id(env) + if !CONFIG.invidious_companion.empty? + return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.") + end + env.response.headers.add("Access-Control-Allow-Origin", "*") env.response.content_type = "application/dash+xml" @@ -20,10 +24,6 @@ module Invidious::Routes::API::Manifest haltf env, status_code: 403 end - if local && CONFIG.invidious_companion - return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" - end - if dashmpd = video.dash_manifest_url response = YT_POOL.client &.get(URI.parse(dashmpd).request_target) diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 266f7ba4..23f2bbe1 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -201,6 +201,13 @@ module Invidious::Routes::Embed return env.redirect url end + if (!CONFIG.invidious_companion.empty? && (preferences.local || preferences.quality == "dash")) + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) + .gsub("connect-src", "connect-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) + end + rendered "embed" end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 5fda7084..17b1022a 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -253,6 +253,10 @@ module Invidious::Routes::VideoPlayback # YouTube /videoplayback links expire after 6 hours, # so we have a mechanism here to redirect to the latest version def self.latest_version(env) + if !CONFIG.invidious_companion.empty? && CONFIG.disabled?("downloads") + return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.") + end + id = env.params.query["id"]? itag = env.params.query["itag"]?.try &.to_i? @@ -294,8 +298,8 @@ module Invidious::Routes::VideoPlayback end if local - if (CONFIG.invidious_companion) - return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" + if (!CONFIG.invidious_companion.empty?) + return env.redirect "#{video.invidious_companion.not_nil!["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" end url = URI.parse(url).request_target.not_nil! url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 2cf8a725..69780d78 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -190,11 +190,11 @@ module Invidious::Routes::Watch captions: video.captions ) - if (CONFIG.invidious_companion && (preferences.local || preferences.quality == "dash")) + if (!CONFIG.invidious_companion.empty? && (preferences.local || preferences.quality == "dash")) env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src " + video.invidious_companion["baseUrl"].as_s) - .gsub("connect-src", "connect-src " + video.invidious_companion["baseUrl"].as_s) + .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) + .gsub("connect-src", "connect-src " + video.invidious_companion.not_nil!["baseUrl"].as_s) end templated "watch" diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 131549d0..7670e9a9 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -15,7 +15,7 @@ struct Video # NOTE: don't forget to bump this number if any change is made to # the `params` structure in videos/parser.cr!!! # - SCHEMA_VERSION = 2 + SCHEMA_VERSION = 3 property id : String @@ -192,8 +192,8 @@ struct Video } end - def invidious_companion : Hash(String, JSON::Any) - info["invidiousCompanion"].try &.as_h + def invidious_companion : Hash(String, JSON::Any)? + info["invidiousCompanion"]?.try &.as_h end # Macros defining getters/setters for various types of data diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 28f3ea41..24729beb 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -100,7 +100,7 @@ def extract_video_info(video_id : String) params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason - if CONFIG.invidious_companion.nil? + if !CONFIG.invidious_companion.empty? new_player_response = nil # Don't use Android test suite client if po_token is passed because po_token doesn't @@ -126,7 +126,7 @@ def extract_video_info(video_id : String) end end - {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| + {"captions", "playabilityStatus", "playerConfig", "storyboards", "invidiousCompanion"}.each do |f| params[f] = player_response[f] if player_response[f]? end @@ -141,10 +141,6 @@ def extract_video_info(video_id : String) params["streamingData"] = streaming_data end - if CONFIG.invidious_companion - params["invidiousCompanion"] = player_response["invidiousCompanion"] - end - # Data structure version, for cache control params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) @@ -453,12 +449,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Music section "music" => JSON.parse(music_list.to_json), # Author infos - "author" => JSON::Any.new(author || ""), - "ucid" => JSON::Any.new(ucid || ""), - "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), - "authorVerified" => JSON::Any.new(author_verified || false), - "subCountText" => JSON::Any.new(subs_text || "-"), - "invidiousCompanion" => JSON::Any.new(subs_text), + "author" => JSON::Any.new(author || ""), + "ucid" => JSON::Any.new(ucid || ""), + "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), + "authorVerified" => JSON::Any.new(author_verified || false), + "subCountText" => JSON::Any.new(subs_text || "-"), } return params diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 017b3462..59f43239 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -22,7 +22,7 @@ audio_streams.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local) + src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty? && params.local) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -37,7 +37,7 @@ <% else %> <% if params.quality == "dash" src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1" - src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion) + src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?) %> <% end %> @@ -48,7 +48,7 @@ fmt_stream.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local - src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local) + src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty? && params.local) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 68abcf48..6a0e50a1 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -500,7 +500,11 @@ module YoutubeAPI data["params"] = params end - return self._post_json("/youtubei/v1/player", data, client_config) + if !CONFIG.invidious_companion.empty? + return self._post_invidious_companion("/youtubei/v1/player", data) + else + return self._post_json("/youtubei/v1/player", data, client_config) + end end #################################################################### @@ -615,19 +619,12 @@ module YoutubeAPI headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", + "Accept-Encoding" => "gzip, deflate", "x-goog-api-format-version" => "2", "x-youtube-client-name" => client_config.name_proto, "x-youtube-client-version" => client_config.version, } - if CONFIG.invidious_companion && endpoint == "/youtubei/v1/player" - headers["Authorization"] = "Bearer " + CONFIG.hmac_key - end - - if !CONFIG.invidious_companion - headers["Accept-Encoding"] = "gzip, deflate" - end - if user_agent = client_config.user_agent headers["User-Agent"] = user_agent end @@ -641,35 +638,16 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") LOGGER.trace("YoutubeAPI: POST data: #{data}") - invidious_companion_urls = CONFIG.invidious_companion - # Send the POST request - if invidious_companion_urls && endpoint == "/youtubei/v1/player" - begin - invidious_companion_response = make_client(URI.parse(invidious_companion_urls.sample), - &.post(endpoint, headers: headers, body: data.to_json)) - body = invidious_companion_response.body - if (invidious_companion_response.status_code != 200) - raise Exception.new("status code: " + invidious_companion_response.status_code.to_s + " and body: " + body) + body = YT_POOL.client() do |client| + client.post(url, headers: headers, body: data.to_json) do |response| + if response.status_code != 200 + raise InfoException.new("Error: non 200 status code. Youtube API returned \ + status code #{response.status_code}. See \ + https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.") end - rescue ex - raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) + self._decompress(response.body_io, response.headers["Content-Encoding"]?) end - else - body = YT_POOL.client() do |client| - client.post(url, headers: headers, body: data.to_json) do |response| - if response.status_code != 200 - raise InfoException.new("Error: non 200 status code. Youtube API returned \ - status code #{response.status_code}. See \ - https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.") - end - self._decompress(response.body_io, response.headers["Content-Encoding"]?) - end - end - end - - if body.nil? && CONFIG.invidious_companion - raise InfoException.new("Error while communicating with Invidious companion: no response data.") end # Convert result to Hash @@ -692,6 +670,53 @@ module YoutubeAPI return initial_data end + #################################################################### + # _post_invidious_companion(endpoint, data) + # + # Internal function that does the actual request to Invidious companion + # and handles errors. + # + # The requested data is an endpoint (URL without the domain part) + # and the data as a Hash object. + # + def _post_invidious_companion( + endpoint : String, + data : Hash + ) : Hash(String, JSON::Any) + headers = HTTP::Headers{ + "Content-Type" => "application/json; charset=UTF-8", + "Accept-Encoding" => "gzip", + "Authorization" => "Bearer " + CONFIG.invidious_companion_key, + } + + # Logging + LOGGER.debug("Invidious companion: Using endpoint: \"#{endpoint}\"") + LOGGER.trace("Invidious companion: POST data: #{data}") + + # Send the POST request + + begin + response = make_client(CONFIG.invidious_companion.sample, + &.post(endpoint, headers: headers, body: data.to_json)) + body = self._decompress(response.body_io, response.headers["Content-Encoding"]?) + if (response.status_code != 200) + raise Exception.new("Error while communicating with Invidious companion: \ + status code: " + response.status_code.to_s + " and body: " + body) + end + rescue ex + raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) + end + + if body.nil? + raise InfoException.new("Error while communicating with Invidious companion: no response data.") + end + + # Convert result to Hash + initial_data = JSON.parse(body).as_h + + return initial_data + end + #################################################################### # _decompress(body_io, headers) #