Mirrored_Repos/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2024-11-22 05:27:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove lsquic from codebase

Browse Source
This commit is contained in:
syeopite 2023-07-26 07:19:12 -07:00
parent d956b1826e
commit c5fe96e936
No known key found for this signature in database
GPG Key ID: A73C186DA3955A1A
8 changed files with 32 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
config/config.example.yml
View File

@ -140,27 +140,6 @@ https_only: false
## ##
#pool_size: 100 #pool_size: 100
##
## Enable/Disable the use of QUIC (HTTP/3) when connecting
## to the youtube API and websites ('youtube.com', 'ytimg.com').
## QUIC's main advantages are its lower latency and lower bandwidth
## use, compared to its predecessors. However, the current version
## of QUIC used in invidious is still based on the IETF draft 31,
## meaning that the underlying library may still not be fully
## optimized. You can read more about QUIC at the link below:
## https://datatracker.ietf.org/doc/html/draft-ietf-quic-transport-31
##
## Note: you should try both options and see what is the best for your
## instance. In general QUIC is recommended for public instances. Your
## mileage may vary.
##
## Note 2: Using QUIC prevents some captcha challenges from appearing.
## See: https://github.com/iv-org/invidious/issues/957#issuecomment-576424042
##
## Accepted values: true, false
## Default: false
##
#use_quic: false
## ##
## Additional cookies to be sent when requesting the youtube API. ## Additional cookies to be sent when requesting the youtube API.

4
shard.lock
View File

@ -24,10 +24,6 @@ shards:
git: https://github.com/jeromegn/kilt.git git: https://github.com/jeromegn/kilt.git
version: 0.6.1 version: 0.6.1
lsquic:
git: https://github.com/iv-org/lsquic.cr.git
version: 2.18.1-2
pg: pg:
git: https://github.com/will/crystal-pg.git git: https://github.com/will/crystal-pg.git
version: 0.24.0 version: 0.24.0

3
shard.yml
View File

@ -25,9 +25,6 @@ dependencies:
protodec: protodec:
github: iv-org/protodec github: iv-org/protodec
version: ~> 0.1.5 version: ~> 0.1.5
lsquic:
github: iv-org/lsquic.cr
version: ~> 2.18.1-2
athena-negotiation: athena-negotiation:
github: athena-framework/negotiation github: athena-framework/negotiation
version: ~> 0.1.1 version: ~> 0.1.1

2
src/invidious.cr
View File

@ -90,7 +90,7 @@ SOFTWARE = {
"branch" => "#{CURRENT_BRANCH}", "branch" => "#{CURRENT_BRANCH}",
} }
YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size, use_quic: CONFIG.use_quic) YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
# CLI # CLI
Kemal.config.extra_options do |parser| Kemal.config.extra_options do |parser|

2
src/invidious/config.cr
View File

@ -126,8 +126,6 @@ class Config
property host_binding : String = "0.0.0.0" property host_binding : String = "0.0.0.0"
# Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property pool_size : Int32 = 100 property pool_size : Int32 = 100
# Use quic transport for youtube api
property use_quic : Bool = false
# Saved cookies in "name1=value1; name2=value2..." format # Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)] @[YAML::Field(converter: Preferences::StringToCookies)]

142
src/invidious/routes/images.cr
View File

@ -3,17 +3,7 @@ module Invidious::Routes::Images
def self.ggpht(env) def self.ggpht(env)
url = env.request.path.lchop("/ggpht") url = env.request.path.lchop("/ggpht")
headers = ( headers = HTTP::Headers.new
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
HTTP::Headers{":authority" => "yt3.ggpht.com"}
else
HTTP::Headers.new
end
{% else %}
HTTP::Headers.new
{% end %}
)
REQUEST_HEADERS_WHITELIST.each do |header| REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]? if env.request.headers[header]?
@ -42,22 +32,9 @@ module Invidious::Routes::Images
} }
begin begin
{% unless flag?(:disable_quic) %} HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
if CONFIG.use_quic return request_proc.call(resp)
YT_POOL.client &.get(url, headers) do |resp| end
return request_proc.call(resp)
end
else
HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
return request_proc.call(resp)
end
end
{% else %}
# This can likely be optimized into a (small) pool sometime in the future.
HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
return request_proc.call(resp)
end
{% end %}
rescue ex rescue ex
end end
end end
@ -78,10 +55,6 @@ module Invidious::Routes::Images
headers = HTTP::Headers.new headers = HTTP::Headers.new
{% unless flag?(:disable_quic) %}
headers[":authority"] = "#{authority}.ytimg.com"
{% end %}
REQUEST_HEADERS_WHITELIST.each do |header| REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]? if env.request.headers[header]?
headers[header] = env.request.headers[header] headers[header] = env.request.headers[header]
@ -107,22 +80,9 @@ module Invidious::Routes::Images
} }
begin begin
{% unless flag?(:disable_quic) %} HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
if CONFIG.use_quic return request_proc.call(resp)
YT_POOL.client &.get(url, headers) do |resp| end
return request_proc.call(resp)
end
else
HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
end
{% else %}
# This can likely be optimized into a (small) pool sometime in the future.
HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
{% end %}
rescue ex rescue ex
end end
end end
@ -133,17 +93,7 @@ module Invidious::Routes::Images
name = env.params.url["name"] name = env.params.url["name"]
url = env.request.resource url = env.request.resource
headers = ( headers = HTTP::Headers.new
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
HTTP::Headers{":authority" => "i9.ytimg.com"}
else
HTTP::Headers.new
end
{% else %}
HTTP::Headers.new
{% end %}
)
REQUEST_HEADERS_WHITELIST.each do |header| REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]? if env.request.headers[header]?
@ -169,22 +119,9 @@ module Invidious::Routes::Images
} }
begin begin
{% unless flag?(:disable_quic) %} HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
if CONFIG.use_quic return request_proc.call(resp)
YT_POOL.client &.get(url, headers) do |resp| end
return request_proc.call(resp)
end
else
HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
end
{% else %}
# This can likely be optimized into a (small) pool sometime in the future.
HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
{% end %}
rescue ex rescue ex
end end
end end
@ -223,41 +160,16 @@ module Invidious::Routes::Images
id = env.params.url["id"] id = env.params.url["id"]
name = env.params.url["name"] name = env.params.url["name"]
headers = ( headers = HTTP::Headers.new
{% unless flag?(:disable_quic) %}
if CONFIG.use_quic
HTTP::Headers{":authority" => "i.ytimg.com"}
else
HTTP::Headers.new
end
{% else %}
HTTP::Headers.new
{% end %}
)
if name == "maxres.jpg" if name == "maxres.jpg"
build_thumbnails(id).each do |thumb| build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
# Logic here is short enough that manually typing them out should be fine. # This can likely be optimized into a (small) pool sometime in the future.
{% unless flag?(:disable_quic) %} if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
if CONFIG.use_quic name = thumb[:url] + ".jpg"
if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200 break
name = thumb[:url] + ".jpg" end
break
end
else
if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
name = thumb[:url] + ".jpg"
break
end
end
{% else %}
# This can likely be optimized into a (small) pool sometime in the future.
if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
name = thumb[:url] + ".jpg"
break
end
{% end %}
end end
end end
@ -287,22 +199,10 @@ module Invidious::Routes::Images
} }
begin begin
{% unless flag?(:disable_quic) %} # This can likely be optimized into a (small) pool sometime in the future.
if CONFIG.use_quic HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
YT_POOL.client &.get(url, headers) do |resp| return request_proc.call(resp)
return request_proc.call(resp) end
end
else
HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
end
{% else %}
# This can likely be optimized into a (small) pool sometime in the future.
HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
end
{% end %}
rescue ex rescue ex
end end
end end

37
src/invidious/yt_backend/connection_pool.cr
View File

@ -1,11 +1,3 @@
{% unless flag?(:disable_quic) %}
require "lsquic"
alias HTTPClientType = QUIC::Client | HTTP::Client
{% else %}
alias HTTPClientType = HTTP::Client
{% end %}
def add_yt_headers(request) def add_yt_headers(request)
if request.headers["User-Agent"] == "Crystal" if request.headers["User-Agent"] == "Crystal"
request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
@ -26,11 +18,11 @@ struct YoutubeConnectionPool
property! url : URI property! url : URI
property! capacity : Int32 property! capacity : Int32
property! timeout : Float64 property! timeout : Float64
property pool : DB::Pool(HTTPClientType) property pool : DB::Pool(HTTP::Client)
def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) def initialize(url : URI, @capacity = 5, @timeout = 5.0)
@url = url @url = url
@pool = build_pool(use_quic) @pool = build_pool()
end end
def client(region = nil, &block) def client(region = nil, &block)
@ -43,11 +35,7 @@ struct YoutubeConnectionPool
response = yield conn response = yield conn
rescue ex rescue ex
conn.close conn.close
{% unless flag?(:disable_quic) %} conn = HTTP::Client.new(url)
conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url)
{% else %}
conn = HTTP::Client.new(url)
{% end %}
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
@ -61,19 +49,9 @@ struct YoutubeConnectionPool
response response
end end
private def build_pool(use_quic) private def build_pool
DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
conn = nil # Declare conn = HTTP::Client.new(url)
{% unless flag?(:disable_quic) %}
if use_quic
conn = QUIC::Client.new(url)
else
conn = HTTP::Client.new(url)
end
{% else %}
conn = HTTP::Client.new(url)
{% end %}
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@ -83,7 +61,6 @@ struct YoutubeConnectionPool
end end
def make_client(url : URI, region = nil) def make_client(url : URI, region = nil)
# TODO: Migrate any applicable endpoints to QUIC
client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"

14
src/invidious/yt_backend/youtube_api.cr
View File

@ -595,17 +595,9 @@ module YoutubeAPI
LOGGER.trace("YoutubeAPI: POST data: #{data}") LOGGER.trace("YoutubeAPI: POST data: #{data}")
# Send the POST request # Send the POST request
if {{ !flag?(:disable_quic) }} && CONFIG.use_quic body = YT_POOL.client(client_config.proxy_region) do |client|
# Using QUIC client client.post(url, headers: headers, body: data.to_json) do |response|
body = YT_POOL.client(client_config.proxy_region, self._decompress(response.body_io, response.headers["Content-Encoding"]?)
&.post(url, headers: headers, body: data.to_json)
).body
else
# Using HTTP client
body = YT_POOL.client(client_config.proxy_region) do |client|
client.post(url, headers: headers, body: data.to_json) do |response|
self._decompress(response.body_io, response.headers["Content-Encoding"]?)
end
end end
end end