From 3a2bd4e928382a4fbb70d1f978912c72b8f889fe Mon Sep 17 00:00:00 2001 From: vhuynh3000 Date: Sun, 27 Sep 2020 10:19:44 -0700 Subject: [PATCH] add config to decrypt on demand instead of polling --- src/invidious.cr | 8 +- src/invidious/helpers/helpers.cr | 1 + src/invidious/helpers/signatures.cr | 100 +++++++++++------- .../jobs/update_decrypt_function_job.cr | 12 +-- src/invidious/videos.cr | 4 +- 5 files changed, 74 insertions(+), 51 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 8d4c2e587..dd862e236 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -168,7 +168,11 @@ end Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB, logger, config) Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB, logger, config) Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, logger, config, HMAC_KEY) -Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new + +DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling) +if config.decrypt_polling + Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new(logger) +end if config.statistics_enabled Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, config, SOFTWARE) @@ -191,8 +195,6 @@ def popular_videos Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get end -DECRYPT_FUNCTION = Invidious::Jobs::UpdateDecryptFunctionJob::DECRYPT_FUNCTION - before_all do |env| preferences = begin Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}") diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 2da49abb7..a6651a31b 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -67,6 +67,7 @@ class Config property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions) property feed_threads : Int32 # Number of threads to use for updating feeds property db : DBConfig # Database configuration + property decrypt_polling : Bool = true # Use polling to keep decryption function up to date property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https:// property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr index f811500f4..d8b1de659 100644 --- a/src/invidious/helpers/signatures.cr +++ b/src/invidious/helpers/signatures.cr @@ -1,53 +1,73 @@ alias SigProc = Proc(Array(String), Int32, Array(String)) -def fetch_decrypt_function(id = "CvFH_6DNRCY") - document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body - url = document.match(/src="(?\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"] - player = YT_POOL.client &.get(url).body +struct DecryptFunction + @decrypt_function = [] of {SigProc, Int32} + @decrypt_time = Time.monotonic - function_name = player.match(/^(?[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"] - function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?[^}]+)}/m).not_nil!["body"] - function_body = function_body.split(";")[1..-2] + def initialize(@use_polling = true) + end - var_name = function_body[0][0, 2] - var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?(.*?))};/).not_nil!["body"] + def update_decrypt_function + @decrypt_function = fetch_decrypt_function + end - operations = {} of String => SigProc - var_body.split("},").each do |operation| - op_name = operation.match(/^[^:]+/).not_nil![0] - op_body = operation.match(/\{[^}]+/).not_nil![0] + private def fetch_decrypt_function(id = "CvFH_6DNRCY") + document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body + url = document.match(/src="(?\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"] + player = YT_POOL.client &.get(url).body - case op_body - when "{a.reverse()" - operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse } - when "{a.splice(0,b)" - operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a } - else - operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a } + function_name = player.match(/^(?[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"] + function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?[^}]+)}/m).not_nil!["body"] + function_body = function_body.split(";")[1..-2] + + var_name = function_body[0][0, 2] + var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?(.*?))};/).not_nil!["body"] + + operations = {} of String => SigProc + var_body.split("},").each do |operation| + op_name = operation.match(/^[^:]+/).not_nil![0] + op_body = operation.match(/\{[^}]+/).not_nil![0] + + case op_body + when "{a.reverse()" + operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse } + when "{a.splice(0,b)" + operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a } + else + operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a } + end end + + decrypt_function = [] of {SigProc, Int32} + function_body.each do |function| + function = function.lchop(var_name).delete("[].") + + op_name = function.match(/[^\(]+/).not_nil![0] + value = function.match(/\(\w,(?[\d]+)\)/).not_nil!["value"].to_i + + decrypt_function << {operations[op_name], value} + end + + return decrypt_function end - decrypt_function = [] of {SigProc, Int32} - function_body.each do |function| - function = function.lchop(var_name).delete("[].") + def decrypt_signature(fmt : Hash(String, JSON::Any)) + return "" if !fmt["s"]? || !fmt["sp"]? - op_name = function.match(/[^\(]+/).not_nil![0] - value = function.match(/\(\w,(?[\d]+)\)/).not_nil!["value"].to_i + sp = fmt["sp"].as_s + sig = fmt["s"].as_s.split("") + if !@use_polling + now = Time.monotonic + if now - @decrypt_time > 60.seconds || @decrypt_function.size == 0 + @decrypt_function = fetch_decrypt_function + @decrypt_time = Time.monotonic + end + end - decrypt_function << {operations[op_name], value} + @decrypt_function.each do |proc, value| + sig = proc.call(sig, value) + end + + return "&#{sp}=#{sig.join("")}" end - - return decrypt_function -end - -def decrypt_signature(fmt : Hash(String, JSON::Any)) - return "" if !fmt["s"]? || !fmt["sp"]? - - sp = fmt["sp"].as_s - sig = fmt["s"].as_s.split("") - DECRYPT_FUNCTION.each do |proc, value| - sig = proc.call(sig, value) - end - - return "&#{sp}=#{sig.join("")}" end diff --git a/src/invidious/jobs/update_decrypt_function_job.cr b/src/invidious/jobs/update_decrypt_function_job.cr index 5332c6727..0a6c09c54 100644 --- a/src/invidious/jobs/update_decrypt_function_job.cr +++ b/src/invidious/jobs/update_decrypt_function_job.cr @@ -1,15 +1,15 @@ class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob - DECRYPT_FUNCTION = [] of {SigProc, Int32} + private getter logger : Invidious::LogHandler + + def initialize(@logger) + end def begin loop do begin - decrypt_function = fetch_decrypt_function - DECRYPT_FUNCTION.clear - decrypt_function.each { |df| DECRYPT_FUNCTION << df } + DECRYPT_FUNCTION.update_decrypt_function rescue ex - # TODO: Log error - next + logger.error("UpdateDecryptFunctionJob : #{ex.message}") ensure sleep 1.minute Fiber.yield diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 4a8311101..74edc1560 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -580,7 +580,7 @@ struct Video s.each do |k, v| fmt[k] = JSON::Any.new(v) end - fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}") + fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") end fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") @@ -599,7 +599,7 @@ struct Video s.each do |k, v| fmt[k] = JSON::Any.new(v) end - fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}") + fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") end fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")