diff --git a/config/config.example.yml b/config/config.example.yml index a9c07a99..56c51629 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -247,15 +247,22 @@ https_only: false #force_resolve: ## -## Configuration for using a HTTP proxy -## If unset, then no HTTP proxy will be used. -## Proxy type supported: HTTP, HTTPS +## Configuration for using an outbound proxy. +## If unset, then no proxy will be used. +## +## The 'type' field selects the proxy protocol: +## - "http" : HTTP CONNECT proxy (default when 'type' is omitted) +## - "socks5" : SOCKS5 proxy (target hostnames are resolved by the proxy) +## - "socks5h" : alias for "socks5" +## +## 'user' and 'password' are optional (leave empty for an unauthenticated proxy). ## ## This is not used for loading the video streams from YouTube servers (circumvent YouTube restrictions) ## Please instead configure the proxy in Invidious companion: ## https://github.com/iv-org/invidious-companion/blob/master/config/config.example.toml ## #http_proxy: +# type: http # user: # password: # host: diff --git a/spec/invidious/socks_proxy_spec.cr b/spec/invidious/socks_proxy_spec.cr new file mode 100644 index 00000000..1c29406d --- /dev/null +++ b/spec/invidious/socks_proxy_spec.cr @@ -0,0 +1,243 @@ +require "http/client" +require "socket" +require "openssl" +require "spectator" +require "../../src/invidious/yt_backend/socks_proxy" + +# A minimal in-process SOCKS5 server used to assert exactly what bytes our +# client emits (auth negotiation, CONNECT address type, target host/port) and +# that the returned IO is a usable tunnel. It handles a single connection. +class MockSocksServer + record Captured, + methods : Array(UInt8), + username : String?, + password : String?, + atyp : UInt8, + address : Bytes, + port : UInt16 + + # Every server registers itself so specs can close all of them in after_each, + # even when an assertion fails before an explicit close. + @@instances = [] of MockSocksServer + + def self.close_all + @@instances.each(&.close) + @@instances.clear + end + + getter port : Int32 + + def initialize(@require_auth : Bool = false, + @valid_user : String? = nil, + @valid_pass : String? = nil, + @echo : Bool = false, + @http_reply : String? = nil) + @server = TCPServer.new("127.0.0.1", 0) + @port = @server.local_address.port + @captured = Channel(Captured | Exception).new(1) + @@instances << self + spawn run + end + + # Blocks until the handshake completed, returning what the server observed. + def wait : Captured + result = @captured.receive + raise result if result.is_a?(Exception) + result + end + + def close + @server.close + end + + private def run + socket = @server.accept + begin + captured = handshake(socket) + @captured.send(captured) + + if reply = @http_reply + # Drain the request headers, then send a canned HTTP response. + while (line = socket.gets) && line != "" + end + socket << reply + socket.flush + elsif @echo + if line = socket.gets + socket << "PONG:#{line}\n" + socket.flush + end + end + rescue ex + @captured.send(ex) + ensure + socket.close + end + end + + private def handshake(io : IO) : Captured + raise "unexpected version" unless io.read_byte == 0x05_u8 + nmethods = io.read_byte.not_nil! + method_bytes = Bytes.new(nmethods) + io.read_fully(method_bytes) + methods = method_bytes.to_a + + username = nil + password = nil + + if @require_auth + unless methods.includes?(0x02_u8) + io.write(Bytes[0x05_u8, 0xFF_u8]); io.flush + raise "no acceptable auth methods offered" + end + + io.write(Bytes[0x05_u8, 0x02_u8]); io.flush + + raise "unexpected auth version" unless io.read_byte == 0x01_u8 + ulen = io.read_byte.not_nil! + ubuf = Bytes.new(ulen); io.read_fully(ubuf); username = String.new(ubuf) + plen = io.read_byte.not_nil! + pbuf = Bytes.new(plen); io.read_fully(pbuf); password = String.new(pbuf) + + ok = username == @valid_user && password == @valid_pass + io.write(Bytes[0x01_u8, ok ? 0x00_u8 : 0x01_u8]); io.flush + raise "authentication rejected" unless ok + else + io.write(Bytes[0x05_u8, 0x00_u8]); io.flush + end + + raise "unexpected request version" unless io.read_byte == 0x05_u8 + raise "expected CONNECT command" unless io.read_byte == 0x01_u8 + io.read_byte # RSV + atyp = io.read_byte.not_nil! + + address = + case atyp + when 0x01_u8 + buf = Bytes.new(4); io.read_fully(buf); buf + when 0x04_u8 + buf = Bytes.new(16); io.read_fully(buf); buf + when 0x03_u8 + dlen = io.read_byte.not_nil! + buf = Bytes.new(dlen); io.read_fully(buf); buf + else + raise "unknown address type" + end + + port_bytes = Bytes.new(2); io.read_fully(port_bytes) + port = IO::ByteFormat::BigEndian.decode(UInt16, port_bytes) + + # Reply: success, BND.ADDR/PORT = 0.0.0.0:0 + io.write(Bytes[0x05_u8, 0x00_u8, 0x00_u8, 0x01_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8]) + io.flush + + Captured.new(methods, username, password, atyp, address, port) + end +end + +Spectator.describe SOCKS5::ProxyClient do + after_each { MockSocksServer.close_all } + + it "sends a hostname target as a domain-type address (ATYP 0x03) for proxy-side resolution" do + server = MockSocksServer.new + client = SOCKS5::ProxyClient.new("127.0.0.1", server.port) + + io = client.open("www.youtube.com", 443) + captured = server.wait + + expect(captured.methods).to eq([0x00_u8]) # only no-auth offered + expect(captured.atyp).to eq(0x03_u8) + expect(String.new(captured.address)).to eq("www.youtube.com") + expect(captured.port).to eq(443_u16) + + io.close + end + + it "encodes an IPv4 literal target as ATYP 0x01" do + server = MockSocksServer.new + client = SOCKS5::ProxyClient.new("127.0.0.1", server.port) + + io = client.open("142.250.72.174", 80) + captured = server.wait + + expect(captured.atyp).to eq(0x01_u8) + expect(captured.address.to_a).to eq([142_u8, 250_u8, 72_u8, 174_u8]) + expect(captured.port).to eq(80_u16) + + io.close + end + + it "encodes an IPv6 literal target as ATYP 0x04 (with :: zero-compression)" do + server = MockSocksServer.new + client = SOCKS5::ProxyClient.new("127.0.0.1", server.port) + + io = client.open("2607:f8b0::200e", 443) + captured = server.wait + + expected = Bytes[0x26, 0x07, 0xf8, 0xb0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x20, 0x0e] + expect(captured.atyp).to eq(0x04_u8) + expect(captured.address.to_a).to eq(expected.to_a) + expect(captured.port).to eq(443_u16) + + io.close + end + + it "offers username/password auth and authenticates (RFC 1929)" do + server = MockSocksServer.new(require_auth: true, valid_user: "alice", valid_pass: "s3cret") + client = SOCKS5::ProxyClient.new("127.0.0.1", server.port, username: "alice", password: "s3cret") + + io = client.open("example.com", 80) + captured = server.wait + + expect(captured.methods.includes?(0x02_u8)).to be_true + expect(captured.username).to eq("alice") + expect(captured.password).to eq("s3cret") + + io.close + end + + it "raises when the proxy rejects the supplied credentials" do + server = MockSocksServer.new(require_auth: true, valid_user: "alice", valid_pass: "s3cret") + client = SOCKS5::ProxyClient.new("127.0.0.1", server.port, username: "alice", password: "wrong") + + expect { client.open("example.com", 80) }.to raise_error(SOCKS5::Error, /authentication failed/) + end + + it "raises when the proxy requires auth but no credentials are configured" do + server = MockSocksServer.new(require_auth: true, valid_user: "alice", valid_pass: "s3cret") + client = SOCKS5::ProxyClient.new("127.0.0.1", server.port) + + expect { client.open("example.com", 80) }.to raise_error(SOCKS5::Error) + end + + it "returns a usable tunnel IO after the handshake" do + server = MockSocksServer.new(echo: true) + client = SOCKS5::ProxyClient.new("127.0.0.1", server.port) + + io = client.open("example.com", 80) + server.wait + + io << "hi\n" + io.flush + expect(io.gets).to eq("PONG:hi") + + io.close + end + + it "drives a real HTTP::Client request through the SOCKS tunnel via #socks_proxy=" do + server = MockSocksServer.new(http_reply: "HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n") + + client = HTTP::Client.new("example.com", 80) + client.socks_proxy = SOCKS5::ProxyClient.new("127.0.0.1", server.port) + + response = client.get("/") + captured = server.wait + + expect(captured.atyp).to eq(0x03_u8) + expect(String.new(captured.address)).to eq("example.com") + expect(captured.port).to eq(80_u16) + expect(response.status_code).to eq(204) + + client.close + end +end diff --git a/src/invidious/config.cr b/src/invidious/config.cr index bde67aff..58b77cba 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -65,11 +65,17 @@ struct ConfigPreferences end end +# Outbound proxy configuration. The name is kept for config backwards +# compatibility; `type` selects the protocol (HTTP CONNECT or SOCKS5). struct HTTPProxyConfig include YAML::Serializable - property user : String - property password : String + # Proxy protocol: "http" (HTTP CONNECT, the default), "socks5", or "socks5h". + # SOCKS5 resolves target hostnames on the proxy side (SOCKS5h semantics). + property type : String = "http" + # Credentials are optional: omit both for an unauthenticated proxy. + property user : String? = nil + property password : String? = nil property host : String property port : Int32 end diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 7a0f1959..3fc450b5 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -16,7 +16,7 @@ struct YoutubeConnectionPool def client(&) conn = pool.checkout # Proxy needs to be reinstated every time we get a client from the pool - conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + configure_proxy(conn) if CONFIG.http_proxy begin response = yield conn @@ -121,7 +121,7 @@ end def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true) client = HTTP::Client.new(url) - client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy + configure_proxy(client) if CONFIG.http_proxy && use_http_proxy # Force the usage of a specific configured IP Family if force_resolve @@ -145,6 +145,26 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_ end end +# Attaches the configured outbound proxy (HTTP CONNECT or SOCKS5) to a client. +# Only called when an outbound proxy is configured. +def configure_proxy(client : HTTP::Client) : Nil + config_proxy = CONFIG.http_proxy.not_nil! + + case config_proxy.type.downcase + when "http" + client.proxy = make_configured_http_proxy_client + when "socks5", "socks5h" + client.socks_proxy = SOCKS5::ProxyClient.new( + config_proxy.host, + config_proxy.port, + username: config_proxy.user, + password: config_proxy.password, + ) + else + raise %(Invalid http_proxy.type #{config_proxy.type.inspect} (expected "http", "socks5", or "socks5h")) + end +end + def make_configured_http_proxy_client # This method is only called when configuration for an HTTP proxy are set config_proxy = CONFIG.http_proxy.not_nil! diff --git a/src/invidious/yt_backend/socks_proxy.cr b/src/invidious/yt_backend/socks_proxy.cr new file mode 100644 index 00000000..0848b15b --- /dev/null +++ b/src/invidious/yt_backend/socks_proxy.cr @@ -0,0 +1,226 @@ +# Minimal SOCKS5 (RFC 1928) client proxy support. +# +# Invidious already tunnels outbound requests through an HTTP CONNECT proxy via +# the `http_proxy` shard, which works by giving `HTTP::Client` a socket factory +# whose `#open(host, port, tls, ...)` returns a connected `IO`. This provides +# the same contract for SOCKS5 so it can be wired in the exact same way (see +# `configure_proxy` in `connection_pool.cr`). +# +# Supported: TCP CONNECT, IPv4/IPv6/hostname targets, optional username/password +# authentication (RFC 1929). Hostnames are sent as domain-type addresses so the +# proxy performs DNS resolution (SOCKS5h semantics) — this is what Invidious +# wants for region/geo handling. BIND and UDP ASSOCIATE are not implemented. +module SOCKS5 + VERSION = 0x05_u8 + + # A SOCKS-level failure (bad handshake, rejected auth, refused CONNECT, ...). + # Subclasses IO::Error so callers that rescue transport failures — including + # Invidious's connection pool — treat it like any other connection error. + class Error < IO::Error + end + + class ProxyClient + getter host : String + getter port : Int32 + getter username : String? + getter password : String? + + def initialize(@host : String, @port : Int32, *, + username : String? = nil, password : String? = nil) + @username = username.presence + @password = password.presence + end + + # Opens a TCP connection to the SOCKS server, negotiates the tunnel to + # `host`:`port`, and returns the resulting `IO` (TLS-wrapped when `tls` is + # set). Mirrors `HTTP::Proxy::Client#open`. + def open(host : String, port : Int32, tls = nil, *, + dns_timeout = nil, connect_timeout = nil, + read_timeout = nil, write_timeout = nil) : IO + socket = TCPSocket.new(@host, @port, dns_timeout, connect_timeout) + socket.read_timeout = read_timeout if read_timeout + socket.write_timeout = write_timeout if write_timeout + socket.sync = false + + begin + negotiate(socket) + request_connect(socket, host, port) + rescue ex + socket.close + raise ex + end + + {% if !flag?(:without_openssl) %} + if tls + socket = OpenSSL::SSL::Socket::Client.new(socket, context: tls, sync_close: true, hostname: host) + end + {% end %} + + socket + end + + # Method-selection handshake, followed by username/password auth if the + # server selects it. + private def negotiate(socket : IO) : Nil + methods = @username ? Bytes[0x00_u8, 0x02_u8] : Bytes[0x00_u8] + + socket.write Bytes[VERSION, methods.size.to_u8] + socket.write methods + socket.flush + + reply = uninitialized UInt8[2] + socket.read_fully(reply.to_slice) + raise Error.new("Unexpected SOCKS version in method reply") unless reply[0] == VERSION + + case reply[1] + when 0x00_u8 then return # no authentication + when 0x02_u8 then authenticate(socket) + when 0xFF_u8 then raise Error.new("SOCKS proxy rejected all offered auth methods (credentials required?)") + else raise Error.new("SOCKS proxy selected unsupported auth method 0x#{reply[1].to_s(16)}") + end + end + + private def authenticate(socket : IO) : Nil + user = @username + raise Error.new("SOCKS proxy requested username/password auth but none is configured") unless user + pass = @password || "" + + raise Error.new("SOCKS username exceeds 255 bytes") if user.bytesize > 255 + raise Error.new("SOCKS password exceeds 255 bytes") if pass.bytesize > 255 + + io = IO::Memory.new + io.write_byte 0x01_u8 # auth sub-negotiation version + io.write_byte user.bytesize.to_u8 + io << user + io.write_byte pass.bytesize.to_u8 + io << pass + socket.write io.to_slice + socket.flush + + reply = uninitialized UInt8[2] + socket.read_fully(reply.to_slice) + raise Error.new("Unexpected auth sub-negotiation version 0x#{reply[0].to_s(16)}") unless reply[0] == 0x01_u8 + raise Error.new("SOCKS authentication failed") unless reply[1] == 0x00_u8 + end + + private def request_connect(socket : IO, host : String, port : Int32) : Nil + io = IO::Memory.new + io.write_byte VERSION + io.write_byte 0x01_u8 # CMD = CONNECT + io.write_byte 0x00_u8 # RSV + write_address(io, host) + io.write_bytes(port.to_u16, IO::ByteFormat::BigEndian) + socket.write io.to_slice + socket.flush + + # Reply: VER REP RSV ATYP BND.ADDR BND.PORT + header = uninitialized UInt8[4] + socket.read_fully(header.to_slice) + raise Error.new("Unexpected SOCKS version in connect reply") unless header[0] == VERSION + raise Error.new(reply_message(header[1])) unless header[1] == 0x00_u8 + + # BND.ADDR length depends on ATYP; drain it plus the 2-byte BND.PORT. + # Invidious does not use the server-bound address. + bnd_len = + case header[3] + when 0x01_u8 then 4 # IPv4 + when 0x04_u8 then 16 # IPv6 + when 0x03_u8 # domain: 1 length byte + N + len = uninitialized UInt8[1] + socket.read_fully(len.to_slice) + len[0].to_i + else + raise Error.new("Unknown address type 0x#{header[3].to_s(16)} in SOCKS reply") + end + socket.skip(bnd_len + 2) + end + + private def write_address(io : IO, host : String) : Nil + if addr = parse_ip(host) + case addr.family + when .inet? + io.write_byte 0x01_u8 + addr.address.split('.').each { |octet| io.write_byte octet.to_u8 } + return + when .inet6? + io.write_byte 0x04_u8 + io.write ipv6_bytes(addr.address) + return + end + end + + # Hostname: let the proxy resolve it (SOCKS5h). + raise Error.new("Hostname exceeds 255 bytes: #{host}") if host.bytesize > 255 + io.write_byte 0x03_u8 + io.write_byte host.bytesize.to_u8 + io << host + end + + # Returns the parsed address only for IP literals; hostnames return nil and + # are sent as domain-type addresses. `.valid?` gates construction so we do + # not raise (and catch) an exception on every hostname request. + private def parse_ip(host : String) : Socket::IPAddress? + Socket::IPAddress.new(host, 0) if Socket::IPAddress.valid?(host) + end + + # Converts an IPv6 address string (possibly using "::" zero-compression) + # into its 16 raw bytes. Embedded-IPv4 forms (e.g. "::ffff:1.2.3.4") are + # not handled — they are rare as connection targets in Invidious. + private def ipv6_bytes(addr : String) : Bytes + head, sep, tail = addr.partition("::") + head_groups = head.empty? ? [] of String : head.split(':') + tail_groups = tail.empty? ? [] of String : tail.split(':') + groups = + if sep.empty? + head_groups + else + head_groups + Array.new(8 - head_groups.size - tail_groups.size, "0") + tail_groups + end + raise Error.new("Malformed IPv6 address: #{addr}") unless groups.size == 8 + + bytes = Bytes.new(16) + groups.each_with_index do |group, i| + value = group.to_u16(16) + bytes[i * 2] = (value >> 8).to_u8 + bytes[i * 2 + 1] = (value & 0xff).to_u8 + end + bytes + end + + private def reply_message(code : UInt8) : String + reason = + case code + when 0x01_u8 then "general SOCKS server failure" + when 0x02_u8 then "connection not allowed by ruleset" + when 0x03_u8 then "network unreachable" + when 0x04_u8 then "host unreachable" + when 0x05_u8 then "connection refused" + when 0x06_u8 then "TTL expired" + when 0x07_u8 then "command not supported" + when 0x08_u8 then "address type not supported" + else "unknown error 0x#{code.to_s(16)}" + end + "SOCKS connect failed: #{reason}" + end + end +end + +# Plug a `SOCKS5::ProxyClient` into an `HTTP::Client`, mirroring the `#proxy=` +# setter that the `http_proxy` shard adds for HTTP CONNECT proxies. SOCKS auth +# is performed in-band during the handshake, so (unlike the HTTP variant) no +# `Proxy-Authorization` request header is added. +class HTTP::Client + def socks_proxy=(proxy_client : SOCKS5::ProxyClient) : Nil + @io = proxy_client.open( + host: @host, + port: @port, + tls: @tls, + dns_timeout: @dns_timeout, + connect_timeout: @connect_timeout, + read_timeout: @read_timeout, + write_timeout: @write_timeout, + ) + rescue ex : IO::Error + raise IO::Error.new("Failed to open SOCKS connection to #{@host}:#{@port} (#{ex.message})", cause: ex) + end +end