Move the YoutubeConnectionPool code to its own file
This commit is contained in:
Samantaz Fox
parent
56dbe15943
commit
d300797e22
|
|
@ -1,4 +1,3 @@
|
||||||
require "lsquic"
|
|
||||||
require "db"
|
require "db"
|
||||||
|
|
||||||
def add_yt_headers(request)
|
def add_yt_headers(request)
|
||||||
|
|
@ -16,55 +15,6 @@ def add_yt_headers(request)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
struct YoutubeConnectionPool
|
|
||||||
property! url : URI
|
|
||||||
property! capacity : Int32
|
|
||||||
property! timeout : Float64
|
|
||||||
property pool : DB::Pool(QUIC::Client | HTTP::Client)
|
|
||||||
|
|
||||||
def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true)
|
|
||||||
@url = url
|
|
||||||
@pool = build_pool(use_quic)
|
|
||||||
end
|
|
||||||
|
|
||||||
def client(region = nil, &block)
|
|
||||||
if region
|
|
||||||
conn = make_client(url, region)
|
|
||||||
response = yield conn
|
|
||||||
else
|
|
||||||
conn = pool.checkout
|
|
||||||
begin
|
|
||||||
response = yield conn
|
|
||||||
rescue ex
|
|
||||||
conn.close
|
|
||||||
conn = QUIC::Client.new(url)
|
|
||||||
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
|
|
||||||
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
|
|
||||||
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
|
|
||||||
response = yield conn
|
|
||||||
ensure
|
|
||||||
pool.release(conn)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
response
|
|
||||||
end
|
|
||||||
|
|
||||||
private def build_pool(use_quic)
|
|
||||||
DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
|
|
||||||
if use_quic
|
|
||||||
conn = QUIC::Client.new(url)
|
|
||||||
else
|
|
||||||
conn = HTTP::Client.new(url)
|
|
||||||
end
|
|
||||||
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
|
|
||||||
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
|
|
||||||
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
|
|
||||||
conn
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
|
# See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
|
||||||
def ci_lower_bound(pos, n)
|
def ci_lower_bound(pos, n)
|
||||||
if n == 0
|
if n == 0
|
||||||
|
|
@ -85,37 +35,6 @@ def elapsed_text(elapsed)
|
||||||
"#{(millis * 1000).round(2)}µs"
|
"#{(millis * 1000).round(2)}µs"
|
||||||
end
|
end
|
||||||
|
|
||||||
def make_client(url : URI, region = nil)
|
|
||||||
# TODO: Migrate any applicable endpoints to QUIC
|
|
||||||
client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
|
|
||||||
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
|
|
||||||
client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
|
|
||||||
client.read_timeout = 10.seconds
|
|
||||||
client.connect_timeout = 10.seconds
|
|
||||||
|
|
||||||
if region
|
|
||||||
PROXY_LIST[region]?.try &.sample(40).each do |proxy|
|
|
||||||
begin
|
|
||||||
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
|
|
||||||
client.set_proxy(proxy)
|
|
||||||
break
|
|
||||||
rescue ex
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
return client
|
|
||||||
end
|
|
||||||
|
|
||||||
def make_client(url : URI, region = nil, &block)
|
|
||||||
client = make_client(url, region)
|
|
||||||
begin
|
|
||||||
yield client
|
|
||||||
ensure
|
|
||||||
client.close
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def decode_length_seconds(string)
|
def decode_length_seconds(string)
|
||||||
length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i
|
length_seconds = string.gsub(/[^0-9:]/, "").split(":").map &.to_i
|
||||||
length_seconds = [0] * (3 - length_seconds.size) + length_seconds
|
length_seconds = [0] * (3 - length_seconds.size) + length_seconds
|
||||||
|
|
|
||||||
81
src/invidious/yt_backend/connection_pool.cr
Normal file
81
src/invidious/yt_backend/connection_pool.cr
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
require "lsquic"
|
||||||
|
|
||||||
|
struct YoutubeConnectionPool
|
||||||
|
property! url : URI
|
||||||
|
property! capacity : Int32
|
||||||
|
property! timeout : Float64
|
||||||
|
property pool : DB::Pool(QUIC::Client | HTTP::Client)
|
||||||
|
|
||||||
|
def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true)
|
||||||
|
@url = url
|
||||||
|
@pool = build_pool(use_quic)
|
||||||
|
end
|
||||||
|
|
||||||
|
def client(region = nil, &block)
|
||||||
|
if region
|
||||||
|
conn = make_client(url, region)
|
||||||
|
response = yield conn
|
||||||
|
else
|
||||||
|
conn = pool.checkout
|
||||||
|
begin
|
||||||
|
response = yield conn
|
||||||
|
rescue ex
|
||||||
|
conn.close
|
||||||
|
conn = QUIC::Client.new(url)
|
||||||
|
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
|
||||||
|
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
|
||||||
|
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
|
||||||
|
response = yield conn
|
||||||
|
ensure
|
||||||
|
pool.release(conn)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
response
|
||||||
|
end
|
||||||
|
|
||||||
|
private def build_pool(use_quic)
|
||||||
|
DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
|
||||||
|
if use_quic
|
||||||
|
conn = QUIC::Client.new(url)
|
||||||
|
else
|
||||||
|
conn = HTTP::Client.new(url)
|
||||||
|
end
|
||||||
|
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
|
||||||
|
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
|
||||||
|
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
|
||||||
|
conn
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def make_client(url : URI, region = nil)
|
||||||
|
# TODO: Migrate any applicable endpoints to QUIC
|
||||||
|
client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
|
||||||
|
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
|
||||||
|
client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
|
||||||
|
client.read_timeout = 10.seconds
|
||||||
|
client.connect_timeout = 10.seconds
|
||||||
|
|
||||||
|
if region
|
||||||
|
PROXY_LIST[region]?.try &.sample(40).each do |proxy|
|
||||||
|
begin
|
||||||
|
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
|
||||||
|
client.set_proxy(proxy)
|
||||||
|
break
|
||||||
|
rescue ex
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
return client
|
||||||
|
end
|
||||||
|
|
||||||
|
def make_client(url : URI, region = nil, &block)
|
||||||
|
client = make_client(url, region)
|
||||||
|
begin
|
||||||
|
yield client
|
||||||
|
ensure
|
||||||
|
client.close
|
||||||
|
end
|
||||||
|
end
|
||||||
Loading…
Reference in a new issue