diff --git a/src/invidious.cr b/src/invidious.cr index d655363e1..0c05b1ec4 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1487,7 +1487,7 @@ get "/feed/channel/:ucid" do |env| end page = 1 - videos, count = get_60_videos(ucid, page, auto_generated) + videos, count = get_60_videos(ucid, page, auto_generated, proxies) host_url = make_host_url(Kemal.config.ssl || CONFIG.https_only, env.request.headers["Host"]?) path = env.request.path @@ -1732,7 +1732,7 @@ get "/channel/:ucid" do |env| end end - videos, count = get_60_videos(ucid, page, auto_generated) + videos, count = get_60_videos(ucid, page, auto_generated, proxies) templated "channel" end @@ -1885,7 +1885,6 @@ get "/api/v1/comments/:id" do |env| proxy_client.read_timeout = 10.seconds proxy_client.connect_timeout = 10.seconds - proxy = list.sample(1)[0] proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) proxy_client.set_proxy(proxy) @@ -1894,13 +1893,10 @@ get "/api/v1/comments/:id" do |env| proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] proxy_html = response.body - if proxy_html.match(//) - bypass_channel.send(nil) - else + if !proxy_html.match(//) bypass_channel.send({proxy_html, proxy_client, proxy_headers}) + break end - - break rescue ex end end @@ -2493,7 +2489,7 @@ get "/api/v1/channels/:ucid" do |env| end page = 1 - videos, count = get_60_videos(ucid, page, auto_generated) + videos, count = get_60_videos(ucid, page, auto_generated, proxies) client = make_client(YT_URL) channel_html = client.get("/channel/#{ucid}/about?disable_polymer=1").body @@ -2631,7 +2627,7 @@ end halt env, status_code: 404, response: error_message end - videos, count = get_60_videos(ucid, page, auto_generated) + videos, count = get_60_videos(ucid, page, auto_generated, proxies) result = JSON.build do |json| json.array do diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 0ab8dd9bf..f7cae7495 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -176,7 +176,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil) continuation = Base64.urlsafe_encode(continuation) continuation = URI.escape(continuation) - url = "/browse_ajax?continuation=#{continuation}" + url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" return url end @@ -216,7 +216,7 @@ def get_about_info(ucid) return {author, ucid, auto_generated, sub_count} end -def get_60_videos(ucid, page, auto_generated) +def get_60_videos(ucid, page, auto_generated, proxies) count = 0 videos = [] of SearchVideo @@ -235,6 +235,49 @@ def get_60_videos(ucid, page, auto_generated) count += 30 end + if !json["load_more_widget_html"]?.try &.as_s.empty? && nodeset.size < 30 + bypass_channel = Channel(XML::NodeSet | Nil).new + + proxies.each do |region, list| + spawn do + list.each do |proxy| + begin + proxy_client = HTTPClient.new(YT_URL) + proxy_client.read_timeout = 10.seconds + proxy_client.connect_timeout = 10.seconds + + proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) + proxy_client.set_proxy(proxy) + + proxy_response = proxy_client.get(url) + json = JSON.parse(proxy_response.body) + + document = XML.parse_html(json["content_html"].as_s) + nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])) + + if nodeset.size == 30 + bypass_channel.send(nodeset) + break + end + rescue ex + end + end + + if nodeset.size != 30 + bypass_channel.send(nil) + end + end + end + + proxies.size.times do + response = bypass_channel.receive + if response + nodeset = response + break + end + end + end + if auto_generated videos += extract_videos(nodeset) else diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index ddee94413..1d823cc4d 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -573,11 +573,8 @@ def fetch_video(id, proxies) info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) if !info["reason"]? bypass_channel.send(proxy) - else - bypass_channel.send(nil) + break end - - break rescue ex end end