feat: add support for SOCKS5 proxy (#5865)

config/config.example.yml

@@ -247,15 +247,22 @@ https_only: false
 #force_resolve:
 
 ##
-## Configuration for using a HTTP proxy
-## If unset, then no HTTP proxy will be used.
-## Proxy type supported: HTTP, HTTPS
+## Configuration for using an outbound proxy.
+## If unset, then no proxy will be used.
+##
+## The 'type' field selects the proxy protocol:
+##   - "http"    : HTTP CONNECT proxy (default when 'type' is omitted)
+##   - "socks5"  : SOCKS5 proxy (target hostnames are resolved by the proxy)
+##   - "socks5h" : alias for "socks5"
+##
+## 'user' and 'password' are optional (leave empty for an unauthenticated proxy).
 ##
 ## This is not used for loading the video streams from YouTube servers (circumvent YouTube restrictions)
 ## Please instead configure the proxy in Invidious companion:
 ## https://github.com/iv-org/invidious-companion/blob/master/config/config.example.toml
 ##
 #http_proxy:
+#  type: http
 #  user:
 #  password:
 #  host:

spec/invidious/socks_proxy_spec.cr

@@ -0,0 +1,243 @@
+require "http/client"
+require "socket"
+require "openssl"
+require "spectator"
+require "../../src/invidious/yt_backend/socks_proxy"
+
+# A minimal in-process SOCKS5 server used to assert exactly what bytes our
+# client emits (auth negotiation, CONNECT address type, target host/port) and
+# that the returned IO is a usable tunnel. It handles a single connection.
+class MockSocksServer
+  record Captured,
+    methods : Array(UInt8),
+    username : String?,
+    password : String?,
+    atyp : UInt8,
+    address : Bytes,
+    port : UInt16
+
+  # Every server registers itself so specs can close all of them in after_each,
+  # even when an assertion fails before an explicit close.
+  @@instances = [] of MockSocksServer
+
+  def self.close_all
+    @@instances.each(&.close)
+    @@instances.clear
+  end
+
+  getter port : Int32
+
+  def initialize(@require_auth : Bool = false,
+                 @valid_user : String? = nil,
+                 @valid_pass : String? = nil,
+                 @echo : Bool = false,
+                 @http_reply : String? = nil)
+    @server = TCPServer.new("127.0.0.1", 0)
+    @port = @server.local_address.port
+    @captured = Channel(Captured | Exception).new(1)
+    @@instances << self
+    spawn run
+  end
+
+  # Blocks until the handshake completed, returning what the server observed.
+  def wait : Captured
+    result = @captured.receive
+    raise result if result.is_a?(Exception)
+    result
+  end
+
+  def close
+    @server.close
+  end
+
+  private def run
+    socket = @server.accept
+    begin
+      captured = handshake(socket)
+      @captured.send(captured)
+
+      if reply = @http_reply
+        # Drain the request headers, then send a canned HTTP response.
+        while (line = socket.gets) && line != ""
+        end
+        socket << reply
+        socket.flush
+      elsif @echo
+        if line = socket.gets
+          socket << "PONG:#{line}\n"
+          socket.flush
+        end
+      end
+    rescue ex
+      @captured.send(ex)
+    ensure
+      socket.close
+    end
+  end
+
+  private def handshake(io : IO) : Captured
+    raise "unexpected version" unless io.read_byte == 0x05_u8
+    nmethods = io.read_byte.not_nil!
+    method_bytes = Bytes.new(nmethods)
+    io.read_fully(method_bytes)
+    methods = method_bytes.to_a
+
+    username = nil
+    password = nil
+
+    if @require_auth
+      unless methods.includes?(0x02_u8)
+        io.write(Bytes[0x05_u8, 0xFF_u8]); io.flush
+        raise "no acceptable auth methods offered"
+      end
+
+      io.write(Bytes[0x05_u8, 0x02_u8]); io.flush
+
+      raise "unexpected auth version" unless io.read_byte == 0x01_u8
+      ulen = io.read_byte.not_nil!
+      ubuf = Bytes.new(ulen); io.read_fully(ubuf); username = String.new(ubuf)
+      plen = io.read_byte.not_nil!
+      pbuf = Bytes.new(plen); io.read_fully(pbuf); password = String.new(pbuf)
+
+      ok = username == @valid_user && password == @valid_pass
+      io.write(Bytes[0x01_u8, ok ? 0x00_u8 : 0x01_u8]); io.flush
+      raise "authentication rejected" unless ok
+    else
+      io.write(Bytes[0x05_u8, 0x00_u8]); io.flush
+    end
+
+    raise "unexpected request version" unless io.read_byte == 0x05_u8
+    raise "expected CONNECT command" unless io.read_byte == 0x01_u8
+    io.read_byte # RSV
+    atyp = io.read_byte.not_nil!
+
+    address =
+      case atyp
+      when 0x01_u8
+        buf = Bytes.new(4); io.read_fully(buf); buf
+      when 0x04_u8
+        buf = Bytes.new(16); io.read_fully(buf); buf
+      when 0x03_u8
+        dlen = io.read_byte.not_nil!
+        buf = Bytes.new(dlen); io.read_fully(buf); buf
+      else
+        raise "unknown address type"
+      end
+
+    port_bytes = Bytes.new(2); io.read_fully(port_bytes)
+    port = IO::ByteFormat::BigEndian.decode(UInt16, port_bytes)
+
+    # Reply: success, BND.ADDR/PORT = 0.0.0.0:0
+    io.write(Bytes[0x05_u8, 0x00_u8, 0x00_u8, 0x01_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8, 0_u8])
+    io.flush
+
+    Captured.new(methods, username, password, atyp, address, port)
+  end
+end
+
+Spectator.describe SOCKS5::ProxyClient do
+  after_each { MockSocksServer.close_all }
+
+  it "sends a hostname target as a domain-type address (ATYP 0x03) for proxy-side resolution" do
+    server = MockSocksServer.new
+    client = SOCKS5::ProxyClient.new("127.0.0.1", server.port)
+
+    io = client.open("www.youtube.com", 443)
+    captured = server.wait
+
+    expect(captured.methods).to eq([0x00_u8]) # only no-auth offered
+    expect(captured.atyp).to eq(0x03_u8)
+    expect(String.new(captured.address)).to eq("www.youtube.com")
+    expect(captured.port).to eq(443_u16)
+
+    io.close
+  end
+
+  it "encodes an IPv4 literal target as ATYP 0x01" do
+    server = MockSocksServer.new
+    client = SOCKS5::ProxyClient.new("127.0.0.1", server.port)
+
+    io = client.open("142.250.72.174", 80)
+    captured = server.wait
+
+    expect(captured.atyp).to eq(0x01_u8)
+    expect(captured.address.to_a).to eq([142_u8, 250_u8, 72_u8, 174_u8])
+    expect(captured.port).to eq(80_u16)
+
+    io.close
+  end
+
+  it "encodes an IPv6 literal target as ATYP 0x04 (with :: zero-compression)" do
+    server = MockSocksServer.new
+    client = SOCKS5::ProxyClient.new("127.0.0.1", server.port)
+
+    io = client.open("2607:f8b0::200e", 443)
+    captured = server.wait
+
+    expected = Bytes[0x26, 0x07, 0xf8, 0xb0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x20, 0x0e]
+    expect(captured.atyp).to eq(0x04_u8)
+    expect(captured.address.to_a).to eq(expected.to_a)
+    expect(captured.port).to eq(443_u16)
+
+    io.close
+  end
+
+  it "offers username/password auth and authenticates (RFC 1929)" do
+    server = MockSocksServer.new(require_auth: true, valid_user: "alice", valid_pass: "s3cret")
+    client = SOCKS5::ProxyClient.new("127.0.0.1", server.port, username: "alice", password: "s3cret")
+
+    io = client.open("example.com", 80)
+    captured = server.wait
+
+    expect(captured.methods.includes?(0x02_u8)).to be_true
+    expect(captured.username).to eq("alice")
+    expect(captured.password).to eq("s3cret")
+
+    io.close
+  end
+
+  it "raises when the proxy rejects the supplied credentials" do
+    server = MockSocksServer.new(require_auth: true, valid_user: "alice", valid_pass: "s3cret")
+    client = SOCKS5::ProxyClient.new("127.0.0.1", server.port, username: "alice", password: "wrong")
+
+    expect { client.open("example.com", 80) }.to raise_error(SOCKS5::Error, /authentication failed/)
+  end
+
+  it "raises when the proxy requires auth but no credentials are configured" do
+    server = MockSocksServer.new(require_auth: true, valid_user: "alice", valid_pass: "s3cret")
+    client = SOCKS5::ProxyClient.new("127.0.0.1", server.port)
+
+    expect { client.open("example.com", 80) }.to raise_error(SOCKS5::Error)
+  end
+
+  it "returns a usable tunnel IO after the handshake" do
+    server = MockSocksServer.new(echo: true)
+    client = SOCKS5::ProxyClient.new("127.0.0.1", server.port)
+
+    io = client.open("example.com", 80)
+    server.wait
+
+    io << "hi\n"
+    io.flush
+    expect(io.gets).to eq("PONG:hi")
+
+    io.close
+  end
+
+  it "drives a real HTTP::Client request through the SOCKS tunnel via #socks_proxy=" do
+    server = MockSocksServer.new(http_reply: "HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n")
+
+    client = HTTP::Client.new("example.com", 80)
+    client.socks_proxy = SOCKS5::ProxyClient.new("127.0.0.1", server.port)
+
+    response = client.get("/")
+    captured = server.wait
+
+    expect(captured.atyp).to eq(0x03_u8)
+    expect(String.new(captured.address)).to eq("example.com")
+    expect(captured.port).to eq(80_u16)
+    expect(response.status_code).to eq(204)
+
+    client.close
+  end
+end

src/invidious/config.cr

@@ -65,11 +65,17 @@ struct ConfigPreferences
   end
 end
 
+# Outbound proxy configuration. The name is kept for config backwards
+# compatibility; `type` selects the protocol (HTTP CONNECT or SOCKS5).
 struct HTTPProxyConfig
   include YAML::Serializable
 
-  property user : String
-  property password : String
+  # Proxy protocol: "http" (HTTP CONNECT, the default), "socks5", or "socks5h".
+  # SOCKS5 resolves target hostnames on the proxy side (SOCKS5h semantics).
+  property type : String = "http"
+  # Credentials are optional: omit both for an unauthenticated proxy.
+  property user : String? = nil
+  property password : String? = nil
   property host : String
   property port : Int32
 end

src/invidious/yt_backend/connection_pool.cr

@@ -16,7 +16,7 @@ struct YoutubeConnectionPool
   def client(&)
     conn = pool.checkout
     # Proxy needs to be reinstated every time we get a client from the pool
-    conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+    configure_proxy(conn) if CONFIG.http_proxy
 
     begin
       response = yield conn
@@ -121,7 +121,7 @@ end
 
 def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true)
   client = HTTP::Client.new(url)
-  client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy
+  configure_proxy(client) if CONFIG.http_proxy && use_http_proxy
 
   # Force the usage of a specific configured IP Family
   if force_resolve
@@ -145,6 +145,26 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_
   end
 end
 
+# Attaches the configured outbound proxy (HTTP CONNECT or SOCKS5) to a client.
+# Only called when an outbound proxy is configured.
+def configure_proxy(client : HTTP::Client) : Nil
+  config_proxy = CONFIG.http_proxy.not_nil!
+
+  case config_proxy.type.downcase
+  when "http"
+    client.proxy = make_configured_http_proxy_client
+  when "socks5", "socks5h"
+    client.socks_proxy = SOCKS5::ProxyClient.new(
+      config_proxy.host,
+      config_proxy.port,
+      username: config_proxy.user,
+      password: config_proxy.password,
+    )
+  else
+    raise %(Invalid http_proxy.type #{config_proxy.type.inspect} (expected "http", "socks5", or "socks5h"))
+  end
+end
+
 def make_configured_http_proxy_client
   # This method is only called when configuration for an HTTP proxy are set
   config_proxy = CONFIG.http_proxy.not_nil!

src/invidious/yt_backend/socks_proxy.cr

@@ -0,0 +1,226 @@
+# Minimal SOCKS5 (RFC 1928) client proxy support.
+#
+# Invidious already tunnels outbound requests through an HTTP CONNECT proxy via
+# the `http_proxy` shard, which works by giving `HTTP::Client` a socket factory
+# whose `#open(host, port, tls, ...)` returns a connected `IO`. This provides
+# the same contract for SOCKS5 so it can be wired in the exact same way (see
+# `configure_proxy` in `connection_pool.cr`).
+#
+# Supported: TCP CONNECT, IPv4/IPv6/hostname targets, optional username/password
+# authentication (RFC 1929). Hostnames are sent as domain-type addresses so the
+# proxy performs DNS resolution (SOCKS5h semantics) — this is what Invidious
+# wants for region/geo handling. BIND and UDP ASSOCIATE are not implemented.
+module SOCKS5
+  VERSION = 0x05_u8
+
+  # A SOCKS-level failure (bad handshake, rejected auth, refused CONNECT, ...).
+  # Subclasses IO::Error so callers that rescue transport failures — including
+  # Invidious's connection pool — treat it like any other connection error.
+  class Error < IO::Error
+  end
+
+  class ProxyClient
+    getter host : String
+    getter port : Int32
+    getter username : String?
+    getter password : String?
+
+    def initialize(@host : String, @port : Int32, *,
+                   username : String? = nil, password : String? = nil)
+      @username = username.presence
+      @password = password.presence
+    end
+
+    # Opens a TCP connection to the SOCKS server, negotiates the tunnel to
+    # `host`:`port`, and returns the resulting `IO` (TLS-wrapped when `tls` is
+    # set). Mirrors `HTTP::Proxy::Client#open`.
+    def open(host : String, port : Int32, tls = nil, *,
+             dns_timeout = nil, connect_timeout = nil,
+             read_timeout = nil, write_timeout = nil) : IO
+      socket = TCPSocket.new(@host, @port, dns_timeout, connect_timeout)
+      socket.read_timeout = read_timeout if read_timeout
+      socket.write_timeout = write_timeout if write_timeout
+      socket.sync = false
+
+      begin
+        negotiate(socket)
+        request_connect(socket, host, port)
+      rescue ex
+        socket.close
+        raise ex
+      end
+
+      {% if !flag?(:without_openssl) %}
+        if tls
+          socket = OpenSSL::SSL::Socket::Client.new(socket, context: tls, sync_close: true, hostname: host)
+        end
+      {% end %}
+
+      socket
+    end
+
+    # Method-selection handshake, followed by username/password auth if the
+    # server selects it.
+    private def negotiate(socket : IO) : Nil
+      methods = @username ? Bytes[0x00_u8, 0x02_u8] : Bytes[0x00_u8]
+
+      socket.write Bytes[VERSION, methods.size.to_u8]
+      socket.write methods
+      socket.flush
+
+      reply = uninitialized UInt8[2]
+      socket.read_fully(reply.to_slice)
+      raise Error.new("Unexpected SOCKS version in method reply") unless reply[0] == VERSION
+
+      case reply[1]
+      when 0x00_u8 then return # no authentication
+      when 0x02_u8 then authenticate(socket)
+      when 0xFF_u8 then raise Error.new("SOCKS proxy rejected all offered auth methods (credentials required?)")
+      else              raise Error.new("SOCKS proxy selected unsupported auth method 0x#{reply[1].to_s(16)}")
+      end
+    end
+
+    private def authenticate(socket : IO) : Nil
+      user = @username
+      raise Error.new("SOCKS proxy requested username/password auth but none is configured") unless user
+      pass = @password || ""
+
+      raise Error.new("SOCKS username exceeds 255 bytes") if user.bytesize > 255
+      raise Error.new("SOCKS password exceeds 255 bytes") if pass.bytesize > 255
+
+      io = IO::Memory.new
+      io.write_byte 0x01_u8 # auth sub-negotiation version
+      io.write_byte user.bytesize.to_u8
+      io << user
+      io.write_byte pass.bytesize.to_u8
+      io << pass
+      socket.write io.to_slice
+      socket.flush
+
+      reply = uninitialized UInt8[2]
+      socket.read_fully(reply.to_slice)
+      raise Error.new("Unexpected auth sub-negotiation version 0x#{reply[0].to_s(16)}") unless reply[0] == 0x01_u8
+      raise Error.new("SOCKS authentication failed") unless reply[1] == 0x00_u8
+    end
+
+    private def request_connect(socket : IO, host : String, port : Int32) : Nil
+      io = IO::Memory.new
+      io.write_byte VERSION
+      io.write_byte 0x01_u8 # CMD = CONNECT
+      io.write_byte 0x00_u8 # RSV
+      write_address(io, host)
+      io.write_bytes(port.to_u16, IO::ByteFormat::BigEndian)
+      socket.write io.to_slice
+      socket.flush
+
+      # Reply: VER REP RSV ATYP BND.ADDR BND.PORT
+      header = uninitialized UInt8[4]
+      socket.read_fully(header.to_slice)
+      raise Error.new("Unexpected SOCKS version in connect reply") unless header[0] == VERSION
+      raise Error.new(reply_message(header[1])) unless header[1] == 0x00_u8
+
+      # BND.ADDR length depends on ATYP; drain it plus the 2-byte BND.PORT.
+      # Invidious does not use the server-bound address.
+      bnd_len =
+        case header[3]
+        when 0x01_u8 then 4  # IPv4
+        when 0x04_u8 then 16 # IPv6
+        when 0x03_u8         # domain: 1 length byte + N
+          len = uninitialized UInt8[1]
+          socket.read_fully(len.to_slice)
+          len[0].to_i
+        else
+          raise Error.new("Unknown address type 0x#{header[3].to_s(16)} in SOCKS reply")
+        end
+      socket.skip(bnd_len + 2)
+    end
+
+    private def write_address(io : IO, host : String) : Nil
+      if addr = parse_ip(host)
+        case addr.family
+        when .inet?
+          io.write_byte 0x01_u8
+          addr.address.split('.').each { |octet| io.write_byte octet.to_u8 }
+          return
+        when .inet6?
+          io.write_byte 0x04_u8
+          io.write ipv6_bytes(addr.address)
+          return
+        end
+      end
+
+      # Hostname: let the proxy resolve it (SOCKS5h).
+      raise Error.new("Hostname exceeds 255 bytes: #{host}") if host.bytesize > 255
+      io.write_byte 0x03_u8
+      io.write_byte host.bytesize.to_u8
+      io << host
+    end
+
+    # Returns the parsed address only for IP literals; hostnames return nil and
+    # are sent as domain-type addresses. `.valid?` gates construction so we do
+    # not raise (and catch) an exception on every hostname request.
+    private def parse_ip(host : String) : Socket::IPAddress?
+      Socket::IPAddress.new(host, 0) if Socket::IPAddress.valid?(host)
+    end
+
+    # Converts an IPv6 address string (possibly using "::" zero-compression)
+    # into its 16 raw bytes. Embedded-IPv4 forms (e.g. "::ffff:1.2.3.4") are
+    # not handled — they are rare as connection targets in Invidious.
+    private def ipv6_bytes(addr : String) : Bytes
+      head, sep, tail = addr.partition("::")
+      head_groups = head.empty? ? [] of String : head.split(':')
+      tail_groups = tail.empty? ? [] of String : tail.split(':')
+      groups =
+        if sep.empty?
+          head_groups
+        else
+          head_groups + Array.new(8 - head_groups.size - tail_groups.size, "0") + tail_groups
+        end
+      raise Error.new("Malformed IPv6 address: #{addr}") unless groups.size == 8
+
+      bytes = Bytes.new(16)
+      groups.each_with_index do |group, i|
+        value = group.to_u16(16)
+        bytes[i * 2] = (value >> 8).to_u8
+        bytes[i * 2 + 1] = (value & 0xff).to_u8
+      end
+      bytes
+    end
+
+    private def reply_message(code : UInt8) : String
+      reason =
+        case code
+        when 0x01_u8 then "general SOCKS server failure"
+        when 0x02_u8 then "connection not allowed by ruleset"
+        when 0x03_u8 then "network unreachable"
+        when 0x04_u8 then "host unreachable"
+        when 0x05_u8 then "connection refused"
+        when 0x06_u8 then "TTL expired"
+        when 0x07_u8 then "command not supported"
+        when 0x08_u8 then "address type not supported"
+        else              "unknown error 0x#{code.to_s(16)}"
+        end
+      "SOCKS connect failed: #{reason}"
+    end
+  end
+end
+
+# Plug a `SOCKS5::ProxyClient` into an `HTTP::Client`, mirroring the `#proxy=`
+# setter that the `http_proxy` shard adds for HTTP CONNECT proxies. SOCKS auth
+# is performed in-band during the handshake, so (unlike the HTTP variant) no
+# `Proxy-Authorization` request header is added.
+class HTTP::Client
+  def socks_proxy=(proxy_client : SOCKS5::ProxyClient) : Nil
+    @io = proxy_client.open(
+      host: @host,
+      port: @port,
+      tls: @tls,
+      dns_timeout: @dns_timeout,
+      connect_timeout: @connect_timeout,
+      read_timeout: @read_timeout,
+      write_timeout: @write_timeout,
+    )
+  rescue ex : IO::Error
+    raise IO::Error.new("Failed to open SOCKS connection to #{@host}:#{@port} (#{ex.message})", cause: ex)
+  end
+end