mirror of
https://github.com/iv-org/invidious.git
synced 2024-11-10 10:18:24 +00:00
Add geo-bypass for video info
This commit is contained in:
Omar Roth
parent
20e815e757
commit
fada57a307
143
src/invidious/helpers/proxy.cr
Normal file
143
src/invidious/helpers/proxy.cr
Normal file
@ -0,0 +1,143 @@
|
||||
# See https://github.com/crystal-lang/crystal/issues/2963
|
||||
class HTTPProxy
|
||||
getter proxy_host : String
|
||||
getter proxy_port : Int32
|
||||
getter options : Hash(Symbol, String)
|
||||
getter tls : OpenSSL::SSL::Context::Client?
|
||||
|
||||
def initialize(@proxy_host, @proxy_port = 80, @options = {} of Symbol => String)
|
||||
end
|
||||
|
||||
def open(host, port, tls = nil, connection_options = {} of Symbol => Float64 | Nil)
|
||||
dns_timeout = connection_options.fetch(:dns_timeout, nil)
|
||||
connect_timeout = connection_options.fetch(:connect_timeout, nil)
|
||||
read_timeout = connection_options.fetch(:read_timeout, nil)
|
||||
|
||||
socket = TCPSocket.new @proxy_host, @proxy_port, dns_timeout, connect_timeout
|
||||
socket.read_timeout = read_timeout if read_timeout
|
||||
socket.sync = true
|
||||
|
||||
socket << "CONNECT #{host}:#{port} HTTP/1.1\r\n"
|
||||
|
||||
if options[:user]?
|
||||
credentials = Base64.strict_encode("#{options[:user]}:#{options[:password]}")
|
||||
credentials = "#{credentials}\n".gsub(/\s/, "")
|
||||
socket << "Proxy-Authorization: Basic #{credentials}\r\n"
|
||||
end
|
||||
|
||||
socket << "\r\n"
|
||||
|
||||
resp = parse_response(socket)
|
||||
|
||||
if resp[:code]? == 200
|
||||
{% if !flag?(:without_openssl) %}
|
||||
if tls
|
||||
tls_socket = OpenSSL::SSL::Socket::Client.new(socket, context: tls, sync_close: true, hostname: host)
|
||||
socket = tls_socket
|
||||
end
|
||||
{% end %}
|
||||
|
||||
return socket
|
||||
else
|
||||
socket.close
|
||||
raise IO::Error.new(resp.inspect)
|
||||
end
|
||||
end
|
||||
|
||||
private def parse_response(socket)
|
||||
resp = {} of Symbol => Int32 | String | Hash(String, String)
|
||||
|
||||
begin
|
||||
version, code, reason = socket.gets.as(String).chomp.split(/ /, 3)
|
||||
|
||||
headers = {} of String => String
|
||||
|
||||
while (line = socket.gets.as(String)) && (line.chomp != "")
|
||||
name, value = line.split(/:/, 2)
|
||||
headers[name.strip] = value.strip
|
||||
end
|
||||
|
||||
resp[:version] = version
|
||||
resp[:code] = code.to_i
|
||||
resp[:reason] = reason
|
||||
resp[:headers] = headers
|
||||
rescue
|
||||
end
|
||||
|
||||
return resp
|
||||
end
|
||||
end
|
||||
|
||||
class HTTPClient < HTTP::Client
|
||||
def set_proxy(proxy : HTTPProxy)
|
||||
begin
|
||||
@socket = proxy.open(host: @host, port: @port, tls: @tls, connection_options: proxy_connection_options)
|
||||
rescue IO::Error
|
||||
@socket = nil
|
||||
end
|
||||
end
|
||||
|
||||
def proxy_connection_options
|
||||
opts = {} of Symbol => Float64 | Nil
|
||||
|
||||
opts[:dns_timeout] = @dns_timeout
|
||||
opts[:connect_timeout] = @connect_timeout
|
||||
opts[:read_timeout] = @read_timeout
|
||||
|
||||
return opts
|
||||
end
|
||||
end
|
||||
|
||||
def get_proxies(country_code = "US")
|
||||
client = HTTP::Client.new(URI.parse("http://spys.one"))
|
||||
client.read_timeout = 10.seconds
|
||||
client.connect_timeout = 10.seconds
|
||||
|
||||
headers = HTTP::Headers.new
|
||||
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
|
||||
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
|
||||
headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9"
|
||||
headers["Host"] = "spys.one"
|
||||
headers["Content-Type"] = "application/x-www-form-urlencoded"
|
||||
body = {
|
||||
"xpp" => "2",
|
||||
"xf1" => "0",
|
||||
"xf2" => "2",
|
||||
"xf4" => "1",
|
||||
"xf5" => "1",
|
||||
}
|
||||
response = client.post("/free-proxy-list/#{country_code}/", headers, form: body)
|
||||
response = XML.parse_html(response.body)
|
||||
|
||||
proxies = [] of {ip: String, port: Int32, score: Float64}
|
||||
response = response.xpath_nodes(%q(//table))[1]
|
||||
response.xpath_nodes(%q(.//tr)).each do |node|
|
||||
if !node["onmouseover"]?
|
||||
next
|
||||
end
|
||||
|
||||
ip = node.xpath_node(%q(.//td[1]/font[2])).to_s.match(/<font class="spy14">(?<address>[^<]+)</).not_nil!["address"]
|
||||
port = 3128
|
||||
|
||||
latency = node.xpath_node(%q(.//td[6])).not_nil!.content.to_f
|
||||
speed = node.xpath_node(%q(.//td[7]/font/table)).not_nil!["width"].to_f
|
||||
uptime = node.xpath_node(%q(.//td[8]/font/acronym)).not_nil!
|
||||
|
||||
# Skip proxies that are down
|
||||
if uptime["title"].ends_with? "?"
|
||||
next
|
||||
end
|
||||
|
||||
if md = uptime.content.match(/^\d+/)
|
||||
uptime = md[0].to_f
|
||||
else
|
||||
next
|
||||
end
|
||||
|
||||
score = (uptime*4 + speed*2 + latency)/7
|
||||
|
||||
proxies << {ip: ip, port: port, score: score}
|
||||
end
|
||||
|
||||
return proxies
|
||||
end
|
||||
@ -108,6 +108,9 @@ CAPTION_LANGUAGES = {
|
||||
"Zulu",
|
||||
}
|
||||
|
||||
REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"}
|
||||
BYPASS_REGIONS = {"CA", "DE", "FR", "JP", "RU", "UK"}
|
||||
|
||||
VIDEO_THUMBNAILS = {
|
||||
{name: "default", url: "default", height: 90, width: 120},
|
||||
{name: "medium", url: "mqdefault", height: 180, width: 320},
|
||||
@ -409,6 +412,49 @@ def fetch_video(id)
|
||||
html = html_channel.receive
|
||||
info = info_channel.receive
|
||||
|
||||
if info["reason"]? && info["reason"].includes? "your country"
|
||||
bypass_channel = Channel({HTTP::Params | Nil, XML::Node | Nil}).new
|
||||
|
||||
BYPASS_REGIONS.each do |country_code|
|
||||
spawn do
|
||||
begin
|
||||
proxies = get_proxies(country_code)
|
||||
|
||||
# Try not to overload single proxy
|
||||
proxy = proxies[0, 5].sample(1)[0]
|
||||
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
|
||||
|
||||
client = HTTPClient.new(URI.parse("https://www.youtube.com"))
|
||||
client.read_timeout = 10.seconds
|
||||
client.connect_timeout = 10.seconds
|
||||
client.set_proxy(proxy)
|
||||
|
||||
proxy_info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1")
|
||||
proxy_info = HTTP::Params.parse(proxy_info.body)
|
||||
|
||||
if !proxy_info["reason"]?
|
||||
proxy_html = client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1")
|
||||
proxy_html = XML.parse_html(proxy_html.body)
|
||||
|
||||
bypass_channel.send({proxy_info, proxy_html})
|
||||
else
|
||||
bypass_channel.send({nil, nil})
|
||||
end
|
||||
rescue ex
|
||||
bypass_channel.send({nil, nil})
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
BYPASS_REGIONS.size.times do
|
||||
response = bypass_channel.receive
|
||||
if response[0] || response[1]
|
||||
info = response[0].not_nil!
|
||||
html = response[1].not_nil!
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if info["reason"]?
|
||||
raise info["reason"]
|
||||
end
|
||||
|
||||
@ -199,6 +199,15 @@ get_youtube_comments();
|
||||
<p id="Wilson">Wilson Score : <%= video.wilson_score.round(4) %></p>
|
||||
<p id="Rating">Rating : <%= rating.round(4) %> / 5</p>
|
||||
<p id="Engagement">Engagement : <%= engagement.round(2) %>%</p>
|
||||
<% if video.allowed_regions.size != REGIONS.size %>
|
||||
<p id="AllowedRegions">
|
||||
<% if video.allowed_regions.size < REGIONS.size / 2 %>
|
||||
Whitelisted regions: <%= video.allowed_regions.join(", ") %>
|
||||
<% else %>
|
||||
Blacklisted regions: <%= (REGIONS.to_a - video.allowed_regions).join(", ") %>
|
||||
<% end %>
|
||||
</p>
|
||||
<% end %>
|
||||
<% if ad_slots %>
|
||||
<p id="Slots">Ad Slots : <%= ad_slots %></p>
|
||||
<% end %>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user