From a12571e748b0bcf37a6f6a80b16f3dec226c754c Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 12 Nov 2021 02:32:15 +0100 Subject: [PATCH 01/16] Fix minor error in config.example.yml Update the "note" for `locale` --- config/config.example.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 4cc6e7f4..3c79a61e 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -466,9 +466,9 @@ default_user_preferences: ## ## Default user interface language (locale). ## - ## Note: overridin the default (no preferred caption language) - ## is not recommended, in order to not penalize people using - ## other languages. + ## Note: When hosting a public instance, overriding the + ## default (english) is not recommended, as it may + ## people using other languages. ## ## Accepted values: ## ar (Arabic) From d379a36c0e89c66d47fe7faa005371e8b633a74a Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 30 Aug 2021 14:55:35 -0700 Subject: [PATCH 02/16] Add compile-time flag to remove code for QUIC --- src/invidious/routes/login.cr | 8 ++++- src/invidious/yt_backend/connection_pool.cr | 33 ++++++++++++++++----- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index b719b571..2d303713 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -53,7 +53,13 @@ module Invidious::Routes::Login # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82 begin - client = QUIC::Client.new(LOGIN_URL) + client = nil # Declare variable + {% unless flag?(:disable_quic) %} + client = QUIC::Client.new(LOGIN_URL) + {% else %} + client = HTTP::Client.new(LOGIN_URL) + {% end %} + headers = HTTP::Headers.new login_page = client.get("/ServiceLogin") diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 5ba2d73c..4bf200cc 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,5 +1,13 @@ require "lsquic" +{% unless flag?(:disable_quic) %} + require "lsquic" + + alias ConnectonClientType = QUIC::Client | HTTP::Client +{% else %} + alias ConnectonClientType = HTTP::Client +{% end %} + def add_yt_headers(request) request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" @@ -19,7 +27,7 @@ struct YoutubeConnectionPool property! url : URI property! capacity : Int32 property! timeout : Float64 - property pool : DB::Pool(QUIC::Client | HTTP::Client) + property pool : DB::Pool(ConnectonClientType) def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) @url = url @@ -36,7 +44,12 @@ struct YoutubeConnectionPool response = yield conn rescue ex conn.close - conn = QUIC::Client.new(url) + {% unless flag?(:disable_quic) %} + conn = QUIC::Client.new(url) + {% else %} + conn = HTTP::Client.new(url) + {% end %} + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -50,12 +63,18 @@ struct YoutubeConnectionPool end private def build_pool(use_quic) - DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - if use_quic - conn = QUIC::Client.new(url) - else + DB::Pool(ConnectonClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + conn = nil # Declare + {% unless flag?(:disable_quic) %} + if use_quic + conn = QUIC::Client.new(url) + else + conn = HTTP::Client.new(url) + end + {% else %} conn = HTTP::Client.new(url) - end + {% end %} + conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" From b0f127d4d89bc1a001c9ad37b27eec8d9d8e7961 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 28 Sep 2021 14:57:48 -0700 Subject: [PATCH 03/16] Fix gzip decompression with HTTP::Client --- src/invidious/yt_backend/youtube_api.cr | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 8ab2fe46..8904b589 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -404,10 +404,17 @@ module YoutubeAPI url = "#{endpoint}?key=#{client_config.api_key}" headers = HTTP::Headers{ - "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip", + "Content-Type" => "application/json; charset=UTF-8", } + # The normal HTTP client automatically applies accept-encoding: gzip, + # and decompresses. However, explicitly applying it will remove this functionality. + # + # https://github.com/crystal-lang/crystal/issues/11252#issuecomment-929594741 + {% unless flag?(:disable_quic) %} + headers["Accept-Encoding"] = "gzip" + {% end %} + # Logging LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") From 245122104a31608360f1749119716e6b8fc2a6b0 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 28 Sep 2021 17:21:26 -0700 Subject: [PATCH 04/16] Respect use_quic param and fix typos --- src/invidious/routes/login.cr | 2 +- src/invidious/yt_backend/connection_pool.cr | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 2d303713..562d88e5 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -55,7 +55,7 @@ module Invidious::Routes::Login begin client = nil # Declare variable {% unless flag?(:disable_quic) %} - client = QUIC::Client.new(LOGIN_URL) + client = CONFIG.use_quic ? QUIC::Client.new(LOGIN_URL) : HTTP::Client.new(LOGIN_URL) {% else %} client = HTTP::Client.new(LOGIN_URL) {% end %} diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 4bf200cc..fe7a572d 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -3,9 +3,9 @@ require "lsquic" {% unless flag?(:disable_quic) %} require "lsquic" - alias ConnectonClientType = QUIC::Client | HTTP::Client + alias HTTPClientType = QUIC::Client | HTTP::Client {% else %} - alias ConnectonClientType = HTTP::Client + alias HTTPClientType = HTTP::Client {% end %} def add_yt_headers(request) @@ -27,7 +27,7 @@ struct YoutubeConnectionPool property! url : URI property! capacity : Int32 property! timeout : Float64 - property pool : DB::Pool(ConnectonClientType) + property pool : DB::Pool(HTTPClientType) def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) @url = url @@ -45,7 +45,7 @@ struct YoutubeConnectionPool rescue ex conn.close {% unless flag?(:disable_quic) %} - conn = QUIC::Client.new(url) + conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url) {% else %} conn = HTTP::Client.new(url) {% end %} @@ -63,7 +63,7 @@ struct YoutubeConnectionPool end private def build_pool(use_quic) - DB::Pool(ConnectonClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = nil # Declare {% unless flag?(:disable_quic) %} if use_quic From c3747c2d49ec592fac9a4583bd85fbd4333674d6 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Oct 2021 05:55:34 -0700 Subject: [PATCH 05/16] Allow ggpht queries with QUIC disabled --- src/invidious/routes/images.cr | 60 +++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index bb924cdf..e02dc9de 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -3,31 +3,61 @@ module Invidious::Routes::Images def self.ggpht(env) url = env.request.path.lchop("/ggpht") - headers = HTTP::Headers{":authority" => "yt3.ggpht.com"} + headers = ( + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + HTTP::Headers{":authority" => "yt3.ggpht.com"} + else + HTTP::Headers.new + end + {% else %} + HTTP::Headers.new + {% end %} + ) + REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? headers[header] = env.request.headers[header] end end + # We're encapsulating this into a proc in order to easily reuse this + # portion of the code for each request block below. + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 + env.response.headers.delete("Transfer-Encoding") + return + end + + proxy_file(response, env) + } + begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("yt3.ggpht.com#{url}") do |resp| + return request_proc.call(resp) end end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - break + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("yt3.ggpht.com#{url}") do |resp| + return request_proc.call(resp) end - - proxy_file(response, env) - end + {% end %} rescue ex end end From 6b8450558d7123b90ca16ccd3bee1067cf3f57b7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Oct 2021 06:03:33 -0700 Subject: [PATCH 06/16] Allow storyboard queries with QUIC disabled --- src/invidious/routes/images.cr | 108 +++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index e02dc9de..d14427f7 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -78,7 +78,9 @@ module Invidious::Routes::Images headers = HTTP::Headers.new - headers[":authority"] = "#{authority}.ytimg.com" + {% unless flag?(:disable_quic) %} + headers[":authority"] = "#{authority}.ytimg.com" + {% end %} REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? @@ -86,25 +88,41 @@ module Invidious::Routes::Images end end + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Connection"] = "close" + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") + end + + proxy_file(response, env) + } + begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("#{authority}.ytimg.com#{url}") do |resp| + return request_proc.call(resp) end end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - break + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("#{authority}.ytimg.com#{url}") do |resp| + return request_proc.call(resp) end - - proxy_file(response, env) - end + {% end %} rescue ex end end @@ -113,34 +131,60 @@ module Invidious::Routes::Images def self.s_p_image(env) id = env.params.url["id"] name = env.params.url["name"] - url = env.request.resource - headers = HTTP::Headers{":authority" => "i9.ytimg.com"} + headers = ( + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + HTTP::Headers{":authority" => "i9.ytimg.com"} + else + HTTP::Headers.new + end + {% else %} + HTTP::Headers.new + {% end %} + ) + REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? headers[header] = env.request.headers[header] end end + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 && response.status_code != 404 + return env.response.headers.delete("Transfer-Encoding") + end + + proxy_file(response, env) + } + begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("i9.ytimg.com#{url}") do |resp| + return request_proc.call(resp) end end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - break + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("i9.ytimg.com#{url}") do |resp| + return request_proc.call(resp) end - - proxy_file(response, env) - end + {% end %} rescue ex end end From 547abe17d974fe783ad3d83234c48cd042fd42dd Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Oct 2021 06:11:10 -0700 Subject: [PATCH 07/16] Use https for ggpht requests --- src/invidious/routes/images.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index d14427f7..1ad0bd3a 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -48,13 +48,13 @@ module Invidious::Routes::Images return request_proc.call(resp) end else - HTTP::Client.get("yt3.ggpht.com#{url}") do |resp| + HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| return request_proc.call(resp) end end {% else %} # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("yt3.ggpht.com#{url}") do |resp| + HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| return request_proc.call(resp) end {% end %} From 814c9e6c3ae81664acec6ab91e782eaca8ed0f75 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Oct 2021 06:14:37 -0700 Subject: [PATCH 08/16] Use https for storyboard image requests --- src/invidious/routes/images.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 1ad0bd3a..a10e0963 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -113,13 +113,13 @@ module Invidious::Routes::Images return request_proc.call(resp) end else - HTTP::Client.get("#{authority}.ytimg.com#{url}") do |resp| + HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| return request_proc.call(resp) end end {% else %} # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("#{authority}.ytimg.com#{url}") do |resp| + HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| return request_proc.call(resp) end {% end %} @@ -175,13 +175,13 @@ module Invidious::Routes::Images return request_proc.call(resp) end else - HTTP::Client.get("i9.ytimg.com#{url}") do |resp| + HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| return request_proc.call(resp) end end {% else %} # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("i9.ytimg.com#{url}") do |resp| + HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| return request_proc.call(resp) end {% end %} From 83556bace29cff730d0461916cb0d9882b777f98 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Oct 2021 06:30:47 -0700 Subject: [PATCH 09/16] Allow thumbnail queries with QUIC disabled --- src/invidious/routes/images.cr | 59 +++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index a10e0963..2272a941 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -223,7 +223,17 @@ module Invidious::Routes::Images id = env.params.url["id"] name = env.params.url["name"] - headers = HTTP::Headers{":authority" => "i.ytimg.com"} + headers = ( + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + HTTP::Headers{":authority" => "i.ytimg.com"} + else + HTTP::Headers.new + end + {% else %} + HTTP::Headers.new + {% end %} + ) if name == "maxres.jpg" build_thumbnails(id).each do |thumb| @@ -233,6 +243,7 @@ module Invidious::Routes::Images end end end + url = "/vi/#{id}/#{name}" REQUEST_HEADERS_WHITELIST.each do |header| @@ -241,25 +252,43 @@ module Invidious::Routes::Images end end + request_proc = ->(response : HTTP::Client::Response) { + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 && response.status_code != 404 + env.response.headers.delete("Transfer-Encoding") + return + end + + proxy_file(response, env) + } + begin - YT_POOL.client &.get(url, headers) do |response| - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + YT_POOL.client &.get(url, headers) do |resp| + return request_proc.call(resp) + end + else + HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + return request_proc.call(resp) end end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - break + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + return request_proc.call(resp) end - - proxy_file(response, env) - end + {% end %} rescue ex end + end end From 48191aca6eb5e128191322ae5bb5d919f28b2770 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Oct 2021 06:35:45 -0700 Subject: [PATCH 10/16] Fix copy-paste error --- src/invidious/routes/images.cr | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 2272a941..823cf13c 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -263,8 +263,7 @@ module Invidious::Routes::Images env.response.headers["Access-Control-Allow-Origin"] = "*" if response.status_code >= 300 && response.status_code != 404 - env.response.headers.delete("Transfer-Encoding") - return + return env.response.headers.delete("Transfer-Encoding") end proxy_file(response, env) From 6ec4dcfafdcad2eb31abfe519553971f09a422a7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Oct 2021 06:58:06 -0700 Subject: [PATCH 11/16] Fix handling for maxres thumbnail --- src/invidious/routes/images.cr | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 823cf13c..594a7869 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -237,10 +237,27 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| - if YT_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg", headers).status_code == 200 - name = thumb[:url] + ".jpg" - break - end + thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" + # Logic here is short enough that manually typing them out should be fine. + {% unless flag?(:disable_quic) %} + if CONFIG.use_quic + if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200 + name = thumb[:url] + ".jpg" + break + end + else + if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + name = thumb[:url] + ".jpg" + break + end + end + {% else %} + # This can likely be optimized into a (small) pool sometime in the future. + if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + name = thumb[:url] + ".jpg" + break + end + {% end %} end end @@ -277,7 +294,7 @@ module Invidious::Routes::Images end else HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) + return request_proc.call(resp) end end {% else %} @@ -288,6 +305,5 @@ module Invidious::Routes::Images {% end %} rescue ex end - end end From 65fbdbff6af2869dde94abb3b0b9f65286d56500 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 12 Nov 2021 03:52:50 -0800 Subject: [PATCH 12/16] Remove of gzip header w/ use_quic config Continuation of b0f127d4d89bc1a001c9ad37b27eec8d9d8e7961 --- src/invidious/yt_backend/youtube_api.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 8904b589..27f25036 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -412,7 +412,9 @@ module YoutubeAPI # # https://github.com/crystal-lang/crystal/issues/11252#issuecomment-929594741 {% unless flag?(:disable_quic) %} - headers["Accept-Encoding"] = "gzip" + if CONFIG.use_quic + headers["Accept-Encoding"] = "gzip" + end {% end %} # Logging From a120f143d7bbe67379d0ff51e96b111661e1d385 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 12 Nov 2021 04:03:23 -0800 Subject: [PATCH 13/16] Disable quic by default See #2577 --- src/invidious/config.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index bacdb4ac..578e31fd 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -93,7 +93,7 @@ class Config property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument) property host_binding : String = "0.0.0.0" # Host to bind (overrided by command line argument) property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) - property use_quic : Bool = true # Use quic transport for youtube api + property use_quic : Bool = false # Use quic transport for youtube api @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new # Saved cookies in "name1=value1; name2=value2..." format From be1a390b505487aa2a2c9f1790cb9a80b6ca7fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20Devos?= Date: Fri, 12 Nov 2021 14:17:07 +0100 Subject: [PATCH 14/16] Upgrade crystal on ARM64 docker image --- docker/Dockerfile.arm64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 4fe6b1ea..4d5d46bf 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,5 +1,5 @@ FROM alpine:edge AS builder -RUN apk add --no-cache 'crystal=1.2.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev +RUN apk add --no-cache 'crystal=1.2.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev ARG release From 49407596278c2977511eab8df4b86dad0a0c22b2 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 12 Nov 2021 18:53:51 +0100 Subject: [PATCH 15/16] Example config: Change 'use_quic' default value Related to commit a120f143d7bbe67379d0ff51e96b111661e1d385 Also point towards the Github issue. --- config/config.example.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 3c79a61e..c3d52d32 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -157,10 +157,14 @@ https_only: false ## Note 2: Using QUIC prevents some captcha challenges from appearing. ## See: https://github.com/iv-org/invidious/issues/957#issuecomment-576424042 ## -## Accepted values: true, false -## Default: true +## Note 3: As of 2021-11-12, Google seems to have disabled QUIC on +## their servers. The default has been changed to 'false'.Read more +## at: https://github.com/iv-org/invidious/issues/2577 ## -#use_quic: true +## Accepted values: true, false +## Default: false +## +#use_quic: false ## ## Additionnal cookies to be sent when requesting the youtube API. From d214a0b3330e6cbb3f42fc321510076238467731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20Devos?= Date: Fri, 12 Nov 2021 23:02:43 +0000 Subject: [PATCH 16/16] remove duplicate lsquic requirement --- src/invidious/yt_backend/connection_pool.cr | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index fe7a572d..3feb9233 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,5 +1,3 @@ -require "lsquic" - {% unless flag?(:disable_quic) %} require "lsquic"