Merge branch 'master' of github.com:iv-org/invidious

config/config.example.yml

@@ -157,10 +157,14 @@ https_only: false
 ## Note 2: Using QUIC prevents some captcha challenges from appearing.
 ## See: https://github.com/iv-org/invidious/issues/957#issuecomment-576424042
 ##
-## Accepted values: true, false
+## Note 3: As of 2021-11-12, Google seems to have disabled QUIC on
-## Default: true
+## their servers. The default has been changed to 'false'.Read more
+## at: https://github.com/iv-org/invidious/issues/2577
 ##
-#use_quic: true
+## Accepted values: true, false
+## Default: false
+##
+#use_quic: false
 
 ##
 ## Additionnal cookies to be sent when requesting the youtube API.
@@ -466,9 +470,9 @@ default_user_preferences:
   ##
   ## Default user interface language (locale).
   ##
-  ## Note: overridin the default (no preferred caption language)
+  ## Note: When hosting a public instance, overriding the
-  ## is not recommended, in order to not penalize people using
+  ## default (english) is not recommended, as it may
-  ## other languages.
+  ## people using other languages.
   ##
   ## Accepted values:
   ##   ar      (Arabic)

docker/Dockerfile.arm64

@@ -1,5 +1,5 @@
 FROM alpine:edge AS builder
-RUN apk add --no-cache 'crystal=1.2.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev
+RUN apk add --no-cache 'crystal=1.2.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev
 
 ARG release
 

src/invidious/config.cr

@@ -93,7 +93,7 @@ class Config
   property port : Int32 = 3000                                     # Port to listen for connections (overrided by command line argument)
   property host_binding : String = "0.0.0.0"                       # Host to bind (overrided by command line argument)
   property pool_size : Int32 = 100                                 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
-  property use_quic : Bool = true                                  # Use quic transport for youtube api
+  property use_quic : Bool = false                                 # Use quic transport for youtube api
 
   @[YAML::Field(converter: Preferences::StringToCookies)]
   property cookies : HTTP::Cookies = HTTP::Cookies.new               # Saved cookies in "name1=value1; name2=value2..." format

src/invidious/routes/images.cr

@@ -3,31 +3,61 @@ module Invidious::Routes::Images
   def self.ggpht(env)
     url = env.request.path.lchop("/ggpht")
 
-    headers = HTTP::Headers{":authority" => "yt3.ggpht.com"}
+    headers = (
+      {% unless flag?(:disable_quic) %}
+        if CONFIG.use_quic
+          HTTP::Headers{":authority" => "yt3.ggpht.com"}
+        else
+          HTTP::Headers.new
+        end
+      {% else %}
+        HTTP::Headers.new
+      {% end %}
+    )
+
     REQUEST_HEADERS_WHITELIST.each do |header|
       if env.request.headers[header]?
         headers[header] = env.request.headers[header]
       end
     end
 
+    # We're encapsulating this into a proc in order to easily reuse this
+    # portion of the code for each request block below.
+    request_proc = ->(response : HTTP::Client::Response) {
+      env.response.status_code = response.status_code
+      response.headers.each do |key, value|
+        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+          env.response.headers[key] = value
+        end
+      end
+
+      env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+      if response.status_code >= 300
+        env.response.headers.delete("Transfer-Encoding")
+        return
+      end
+
+      proxy_file(response, env)
+    }
+
     begin
-      YT_POOL.client &.get(url, headers) do |response|
+      {% unless flag?(:disable_quic) %}
-        env.response.status_code = response.status_code
+        if CONFIG.use_quic
-        response.headers.each do |key, value|
+          YT_POOL.client &.get(url, headers) do |resp|
-          if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+            return request_proc.call(resp)
-            env.response.headers[key] = value
+          end
+        else
+          HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
+            return request_proc.call(resp)
           end
         end
-
+      {% else %}
-        env.response.headers["Access-Control-Allow-Origin"] = "*"
+        # This can likely be optimized into a (small) pool sometime in the future.
-
+        HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
-        if response.status_code >= 300
+          return request_proc.call(resp)
-          env.response.headers.delete("Transfer-Encoding")
-          break
         end
-
+      {% end %}
-        proxy_file(response, env)
-      end
     rescue ex
     end
   end
@@ -48,7 +78,9 @@ module Invidious::Routes::Images
 
     headers = HTTP::Headers.new
 
-    headers[":authority"] = "#{authority}.ytimg.com"
+    {% unless flag?(:disable_quic) %}
+      headers[":authority"] = "#{authority}.ytimg.com"
+    {% end %}
 
     REQUEST_HEADERS_WHITELIST.each do |header|
       if env.request.headers[header]?
@@ -56,25 +88,41 @@ module Invidious::Routes::Images
       end
     end
 
+    request_proc = ->(response : HTTP::Client::Response) {
+      env.response.status_code = response.status_code
+      response.headers.each do |key, value|
+        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+          env.response.headers[key] = value
+        end
+      end
+
+      env.response.headers["Connection"] = "close"
+      env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+      if response.status_code >= 300
+        return env.response.headers.delete("Transfer-Encoding")
+      end
+
+      proxy_file(response, env)
+    }
+
     begin
-      YT_POOL.client &.get(url, headers) do |response|
+      {% unless flag?(:disable_quic) %}
-        env.response.status_code = response.status_code
+        if CONFIG.use_quic
-        response.headers.each do |key, value|
+          YT_POOL.client &.get(url, headers) do |resp|
-          if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+            return request_proc.call(resp)
-            env.response.headers[key] = value
+          end
+        else
+          HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
+            return request_proc.call(resp)
           end
         end
-
+      {% else %}
-        env.response.headers["Connection"] = "close"
+        # This can likely be optimized into a (small) pool sometime in the future.
-        env.response.headers["Access-Control-Allow-Origin"] = "*"
+        HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
-
+          return request_proc.call(resp)
-        if response.status_code >= 300
-          env.response.headers.delete("Transfer-Encoding")
-          break
         end
-
+      {% end %}
-        proxy_file(response, env)
-      end
     rescue ex
     end
   end
@@ -83,34 +131,60 @@ module Invidious::Routes::Images
   def self.s_p_image(env)
     id = env.params.url["id"]
     name = env.params.url["name"]
-
     url = env.request.resource
 
-    headers = HTTP::Headers{":authority" => "i9.ytimg.com"}
+    headers = (
+      {% unless flag?(:disable_quic) %}
+        if CONFIG.use_quic
+          HTTP::Headers{":authority" => "i9.ytimg.com"}
+        else
+          HTTP::Headers.new
+        end
+      {% else %}
+        HTTP::Headers.new
+      {% end %}
+    )
+
     REQUEST_HEADERS_WHITELIST.each do |header|
       if env.request.headers[header]?
         headers[header] = env.request.headers[header]
       end
     end
 
+    request_proc = ->(response : HTTP::Client::Response) {
+      env.response.status_code = response.status_code
+      response.headers.each do |key, value|
+        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+          env.response.headers[key] = value
+        end
+      end
+
+      env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+      if response.status_code >= 300 && response.status_code != 404
+        return env.response.headers.delete("Transfer-Encoding")
+      end
+
+      proxy_file(response, env)
+    }
+
     begin
-      YT_POOL.client &.get(url, headers) do |response|
+      {% unless flag?(:disable_quic) %}
-        env.response.status_code = response.status_code
+        if CONFIG.use_quic
-        response.headers.each do |key, value|
+          YT_POOL.client &.get(url, headers) do |resp|
-          if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+            return request_proc.call(resp)
-            env.response.headers[key] = value
+          end
+        else
+          HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
+            return request_proc.call(resp)
           end
         end
-
+      {% else %}
-        env.response.headers["Access-Control-Allow-Origin"] = "*"
+        # This can likely be optimized into a (small) pool sometime in the future.
-
+        HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
-        if response.status_code >= 300 && response.status_code != 404
+          return request_proc.call(resp)
-          env.response.headers.delete("Transfer-Encoding")
-          break
         end
-
+      {% end %}
-        proxy_file(response, env)
-      end
     rescue ex
     end
   end
@@ -149,16 +223,44 @@ module Invidious::Routes::Images
     id = env.params.url["id"]
     name = env.params.url["name"]
 
-    headers = HTTP::Headers{":authority" => "i.ytimg.com"}
+    headers = (
+      {% unless flag?(:disable_quic) %}
+        if CONFIG.use_quic
+          HTTP::Headers{":authority" => "i.ytimg.com"}
+        else
+          HTTP::Headers.new
+        end
+      {% else %}
+        HTTP::Headers.new
+      {% end %}
+    )
 
     if name == "maxres.jpg"
       build_thumbnails(id).each do |thumb|
-        if YT_POOL.client &.head("/vi/#{id}/#{thumb[:url]}.jpg", headers).status_code == 200
+        thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
-          name = thumb[:url] + ".jpg"
+        # Logic here is short enough that manually typing them out should be fine.
-          break
+        {% unless flag?(:disable_quic) %}
-        end
+          if CONFIG.use_quic
+            if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200
+              name = thumb[:url] + ".jpg"
+              break
+            end
+          else
+            if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
+              name = thumb[:url] + ".jpg"
+              break
+            end
+          end
+        {% else %}
+          # This can likely be optimized into a (small) pool sometime in the future.
+          if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
+            name = thumb[:url] + ".jpg"
+            break
+          end
+        {% end %}
       end
     end
+
     url = "/vi/#{id}/#{name}"
 
     REQUEST_HEADERS_WHITELIST.each do |header|
@@ -167,24 +269,40 @@ module Invidious::Routes::Images
       end
     end
 
+    request_proc = ->(response : HTTP::Client::Response) {
+      env.response.status_code = response.status_code
+      response.headers.each do |key, value|
+        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+          env.response.headers[key] = value
+        end
+      end
+
+      env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+      if response.status_code >= 300 && response.status_code != 404
+        return env.response.headers.delete("Transfer-Encoding")
+      end
+
+      proxy_file(response, env)
+    }
+
     begin
-      YT_POOL.client &.get(url, headers) do |response|
+      {% unless flag?(:disable_quic) %}
-        env.response.status_code = response.status_code
+        if CONFIG.use_quic
-        response.headers.each do |key, value|
+          YT_POOL.client &.get(url, headers) do |resp|
-          if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+            return request_proc.call(resp)
-            env.response.headers[key] = value
+          end
+        else
+          HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
+            return request_proc.call(resp)
           end
         end
-
+      {% else %}
-        env.response.headers["Access-Control-Allow-Origin"] = "*"
+        # This can likely be optimized into a (small) pool sometime in the future.
-
+        HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
-        if response.status_code >= 300 && response.status_code != 404
+          return request_proc.call(resp)
-          env.response.headers.delete("Transfer-Encoding")
-          break
         end
-
+      {% end %}
-        proxy_file(response, env)
-      end
     rescue ex
     end
   end

src/invidious/routes/login.cr

@@ -53,7 +53,13 @@ module Invidious::Routes::Login
 
       # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
       begin
-        client = QUIC::Client.new(LOGIN_URL)
+        client = nil # Declare variable
+        {% unless flag?(:disable_quic) %}
+          client = CONFIG.use_quic ? QUIC::Client.new(LOGIN_URL) : HTTP::Client.new(LOGIN_URL)
+        {% else %}
+          client = HTTP::Client.new(LOGIN_URL)
+        {% end %}
+
         headers = HTTP::Headers.new
 
         login_page = client.get("/ServiceLogin")

src/invidious/yt_backend/connection_pool.cr

@@ -1,4 +1,10 @@
-require "lsquic"
+{% unless flag?(:disable_quic) %}
+  require "lsquic"
+
+  alias HTTPClientType = QUIC::Client | HTTP::Client
+{% else %}
+  alias HTTPClientType = HTTP::Client
+{% end %}
 
 def add_yt_headers(request)
   request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
@@ -19,7 +25,7 @@ struct YoutubeConnectionPool
   property! url : URI
   property! capacity : Int32
   property! timeout : Float64
-  property pool : DB::Pool(QUIC::Client | HTTP::Client)
+  property pool : DB::Pool(HTTPClientType)
 
   def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true)
     @url = url
@@ -36,7 +42,12 @@ struct YoutubeConnectionPool
         response = yield conn
       rescue ex
         conn.close
-        conn = QUIC::Client.new(url)
+        {% unless flag?(:disable_quic) %}
+          conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url)
+        {% else %}
+          conn = HTTP::Client.new(url)
+        {% end %}
+
         conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
         conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
         conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@@ -50,12 +61,18 @@ struct YoutubeConnectionPool
   end
 
   private def build_pool(use_quic)
-    DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
+    DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
-      if use_quic
+      conn = nil # Declare
-        conn = QUIC::Client.new(url)
+      {% unless flag?(:disable_quic) %}
-      else
+        if use_quic
+          conn = QUIC::Client.new(url)
+        else
+          conn = HTTP::Client.new(url)
+        end
+      {% else %}
         conn = HTTP::Client.new(url)
-      end
+      {% end %}
+
       conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"

src/invidious/yt_backend/youtube_api.cr

@@ -404,10 +404,19 @@ module YoutubeAPI
     url = "#{endpoint}?key=#{client_config.api_key}"
 
     headers = HTTP::Headers{
-      "Content-Type"    => "application/json; charset=UTF-8",
+      "Content-Type" => "application/json; charset=UTF-8",
-      "Accept-Encoding" => "gzip",
     }
 
+    # The normal HTTP client automatically applies accept-encoding: gzip,
+    # and decompresses. However, explicitly applying it will remove this functionality.
+    #
+    # https://github.com/crystal-lang/crystal/issues/11252#issuecomment-929594741
+    {% unless flag?(:disable_quic) %}
+      if CONFIG.use_quic
+        headers["Accept-Encoding"] = "gzip"
+      end
+    {% end %}
+
     # Logging
     LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"")
     LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}")