Mirrored_Repos/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2025-04-25 08:06:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Connection pool: ensure response is fully read

Browse Source 
The streaming API of HTTP::Client has an internal buffer
that will continue to persist onto the next request unless
the response is fully read.

This commit privatizes the #client method of Pool and instead
expose various HTTP request methods that will call and yield
the underlying request and response.

This way, we can ensure that the resposne is fully read before
the client is passed back into the pool for another request.
This commit is contained in:
syeopite 2024-11-14 16:20:23 -08:00
parent 4ce0f775fb
commit e9b2db4d80
No known key found for this signature in database
GPG Key ID: A73C186DA3955A1A
13 changed files with 60 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/invidious/channels/channels.cr
View File

@ -166,7 +166,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
}
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
rss = YT_POOL.get("/feeds/videos.xml?channel_id=#{ucid}").body
LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
rss = XML.parse(rss)

29
src/invidious/connection/pool.cr
View File

@ -33,8 +33,33 @@ module Invidious::ConnectionPool
# Returns the underlying `DB::Pool` object
abstract def pool : DB::Pool(PoolClient)
# Checks out a client from the pool
def client(&)
{% for method in %w[get post put patch delete head options] %}
def {{method.id}}(*args, **kwargs)
self.client do | client |
client.{{method.id}}(*args, **kwargs) do | response |
result = yield response
return result
ensure
response.body_io?.try &. skip_to_end
end
end
end
def {{method.id}}(*args, **kwargs)
self.client do | client |
return response = client.{{method.id}}(*args, **kwargs)
ensure
if response
response.body_io?.try &. skip_to_end
end
end
end
{% end %}
# Checks out a client in the pool
private def client(&)
pool.checkout do |http_client|
# Proxy needs to be reinstated every time we get a client from the pool
http_client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy

2
src/invidious/mixes.cr
View File

@ -26,7 +26,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
end
video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"
response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
response = YT_POOL.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
initial_data = extract_initial_data(response.body)
if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?

6
src/invidious/routes/api/manifest.cr
View File

@ -26,7 +26,7 @@ module Invidious::Routes::API::Manifest
end
if dashmpd = video.dash_manifest_url
response = YT_POOL.client &.get(URI.parse(dashmpd).request_target)
response = YT_POOL.get(URI.parse(dashmpd).request_target)
if response.status_code != 200
haltf env, status_code: response.status_code
@ -167,7 +167,7 @@ module Invidious::Routes::API::Manifest
# /api/manifest/hls_playlist/*
def self.get_hls_playlist(env)
response = YT_POOL.client &.get(env.request.path)
response = YT_POOL.get(env.request.path)
if response.status_code != 200
haltf env, status_code: response.status_code
@ -223,7 +223,7 @@ module Invidious::Routes::API::Manifest
# /api/manifest/hls_variant/*
def self.get_hls_variant(env)
response = YT_POOL.client &.get(env.request.path)
response = YT_POOL.get(env.request.path)
if response.status_code != 200
haltf env, status_code: response.status_code

6
src/invidious/routes/api/v1/videos.cr
View File

@ -106,7 +106,7 @@ module Invidious::Routes::API::V1::Videos
# Auto-generated captions often have cues that aren't aligned properly with the video,
# as well as some other markup that makes it cumbersome, so we try to fix that here
if caption.name.includes? "auto-generated"
caption_xml = YT_POOL.client &.get(url).body
caption_xml = YT_POOL.get(url).body
settings_field = {
"Kind" => "captions",
@ -147,7 +147,7 @@ module Invidious::Routes::API::V1::Videos
query_params = uri.query_params
query_params["fmt"] = "vtt"
uri.query_params = query_params
webvtt = YT_POOL.client &.get(uri.request_target).body
webvtt = YT_POOL.get(uri.request_target).body
if webvtt.starts_with?("<?xml")
webvtt = caption.timedtext_to_vtt(webvtt)
@ -300,7 +300,7 @@ module Invidious::Routes::API::V1::Videos
cache_annotation(id, annotations)
end
else # "youtube"
response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
response = YT_POOL.get("/annotations_invideo?video_id=#{id}")
if response.status_code != 200
haltf env, response.status_code

2
src/invidious/routes/channels.cr
View File

@ -392,7 +392,7 @@ module Invidious::Routes::Channels
value = env.request.resource.split("/")[2]
body = ""
{"channel", "user", "c"}.each do |type|
response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1")
response = YT_POOL.get("/#{type}/#{value}/live?disable_polymer=1")
if response.status_code == 200
body = response.body
end

2
src/invidious/routes/embed.cr
View File

@ -92,7 +92,7 @@ module Invidious::Routes::Embed
return env.redirect url
when "live_stream"
response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
response = YT_POOL.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
video_id = response.body.match(/"video_id":"(?<video_id>[a-zA-Z0-9_-]{11})"/).try &.["video_id"]
env.params.query.delete_all("channel")

6
src/invidious/routes/errors.cr
View File

@ -9,10 +9,10 @@ module Invidious::Routes::ErrorRoutes
item = md["id"]
# Check if item is branding URL e.g. https://youtube.com/gaming
response = YT_POOL.client &.get("/#{item}")
response = YT_POOL.get("/#{item}")
if response.status_code == 301
response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target)
response = YT_POOL.get(URI.parse(response.headers["Location"]).request_target)
end
if response.body.empty?
@ -40,7 +40,7 @@ module Invidious::Routes::ErrorRoutes
end
# Check if item is video ID
if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404
if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.head("/watch?v=#{item}").status_code != 404
env.response.headers["Location"] = url
haltf env, status_code: 302
end

5
src/invidious/routes/feeds.cr
View File

@ -160,8 +160,9 @@ module Invidious::Routes::Feeds
"default" => "http://www.w3.org/2005/Atom",
}
response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
response = YT_POOL.get("/feeds/videos.xml?channel_id=#{ucid}")
return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
rss = XML.parse(response.body)
videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
@ -304,7 +305,7 @@ module Invidious::Routes::Feeds
end
end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
response = YT_POOL.get("/feeds/videos.xml?playlist_id=#{plid}")
return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
document = XML.parse(response.body)

12
src/invidious/routes/images.cr
View File

@ -12,7 +12,7 @@ module Invidious::Routes::Images
end
begin
GGPHT_POOL.client &.get(url, headers) do |resp|
GGPHT_POOL.get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
@ -42,7 +42,7 @@ module Invidious::Routes::Images
end
begin
Invidious::ConnectionPool.get_ytimg_pool(authority).client &.get(url, headers) do |resp|
Invidious::ConnectionPool.get_ytimg_pool(authority).get(url, headers) do |resp|
env.response.headers["Connection"] = "close"
return self.proxy_image(env, resp)
end
@ -65,7 +65,7 @@ module Invidious::Routes::Images
end
begin
Invidious::ConnectionPool.get_ytimg_pool("i9").client &.get(url, headers) do |resp|
Invidious::ConnectionPool.get_ytimg_pool("i9").get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
@ -81,7 +81,7 @@ module Invidious::Routes::Images
end
begin
YT_POOL.client &.get(env.request.resource, headers) do |response|
YT_POOL.get(env.request.resource, headers) do |response|
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
@ -111,7 +111,7 @@ module Invidious::Routes::Images
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
if Invidious::ConnectionPool.get_ytimg_pool("i").client &.head(thumbnail_resource_path, headers).status_code == 200
if Invidious::ConnectionPool.get_ytimg_pool("i").head(thumbnail_resource_path, headers).status_code == 200
name = thumb[:url] + ".jpg"
break
end
@ -127,7 +127,7 @@ module Invidious::Routes::Images
end
begin
Invidious::ConnectionPool.get_ytimg_pool("i").client &.get(url, headers) do |resp|
Invidious::ConnectionPool.get_ytimg_pool("i").get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex

2
src/invidious/routes/playlists.cr
View File

@ -464,7 +464,7 @@ module Invidious::Routes::Playlists
# Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos
def self.watch_videos(env)
response = YT_POOL.client &.get(env.request.resource)
response = YT_POOL.get(env.request.resource)
if url = response.headers["Location"]?
url = URI.parse(url).request_target
return env.redirect url

6
src/invidious/search/processors.cr
View File

@ -16,11 +16,11 @@ module Invidious::Search
# Search a youtube channel
# TODO: clean code, and rely more on YoutubeAPI
def channel(query : Query) : Array(SearchItem)
response = YT_POOL.client &.get("/channel/#{query.channel}")
response = YT_POOL.get("/channel/#{query.channel}")
if response.status_code == 404
response = YT_POOL.client &.get("/user/#{query.channel}")
response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
response = YT_POOL.get("/user/#{query.channel}")
response = YT_POOL.get("/c/#{query.channel}") if response.status_code == 404
initial_data = extract_initial_data(response.body)
ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
raise ChannelSearchException.new(query.channel) if !ucid

16
src/invidious/yt_backend/youtube_api.cr
View File

@ -639,15 +639,13 @@ module YoutubeAPI
LOGGER.trace("YoutubeAPI: POST data: #{data}")
# Send the POST request
body = YT_POOL.client() do |client|
client.post(url, headers: headers, body: data.to_json) do |response|
if response.status_code != 200
raise InfoException.new("Error: non 200 status code. Youtube API returned \
status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \
https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.")
end
self._decompress(response.body_io, response.headers["Content-Encoding"]?)
body = YT_POOL.post(url, headers: headers, body: data.to_json) do |response|
if response.status_code != 200
raise InfoException.new("Error: non 200 status code. Youtube API returned \
status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \
https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.")
end
self._decompress(response.body_io, response.headers["Content-Encoding"]?)
end
# Convert result to Hash
@ -695,7 +693,7 @@ module YoutubeAPI
# Send the POST request
begin
response = COMPANION_POOL.client &.post(endpoint, headers: headers, body: data.to_json)
response = COMPANION_POOL.post(endpoint, headers: headers, body: data.to_json)
body = response.body
if (response.status_code != 200)
raise Exception.new(