videos: improve fetching of streaming data

src/invidious/videos.cr

@@ -380,12 +380,6 @@ def fetch_video(id, region)
     end
   end
 
-  # Try to fetch video info using an embedded client
-  if info["reason"]?
-    embed_info = extract_video_info(video_id: id, context_screen: "embed")
-    info = embed_info if !embed_info["reason"]?
-  end
-
   if reason = info["reason"]?
     if reason == "Video unavailable"
       raise NotFoundException.new(reason.as_s || "")

src/invidious/videos/parser.cr

@@ -50,12 +50,9 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
   }
 end
 
-def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil)
+def extract_video_info(video_id : String, proxy_region : String? = nil)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region)
-  if context_screen == "embed"
-    client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
-  end
 
   # Fetch data from the player endpoint
   player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
@@ -69,7 +66,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
     reason ||= player_response.dig("playabilityStatus", "reason").as_s
 
     # Stop here if video is not a scheduled livestream
-    if playability_status != "LIVE_STREAM_OFFLINE"
+    if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
       return {
         "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
         "reason"  => JSON::Any.new(reason),
@@ -84,7 +81,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
   end
 
   # Don't fetch the next endpoint if the video is unavailable.
-  if {"OK", "LIVE_STREAM_OFFLINE"}.any?(playability_status)
+  if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
     next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
     player_response = player_response.merge(next_response)
   end
@@ -92,33 +89,34 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
   params = parse_video_info(video_id, player_response)
   params["reason"] = JSON::Any.new(reason) if reason
 
-  # Fetch the video streams using an Android client in order to get the decrypted URLs and
+  new_player_response = nil
-  # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs:
+
-  # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
   if reason.nil?
-    if context_screen == "embed"
+    # Fetch the video streams using an Android client in order to get the
-      client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
+    # decrypted URLs and maybe fix throttling issues (#2194). See the
-    else
+    # following issue for an explanation about decrypted URLs:
+    # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
     client_config.client_type = YoutubeAPI::ClientType::Android
-    end
+    new_player_response = try_fetch_streaming_data(video_id, client_config)
-    android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
+  elsif !reason.includes?("your country") # Handled separately
-
+    # The Android embedded client could help here
-    # Sometimes, the video is available from the web client, but not on Android, so check
+    client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
-    # that here, and fallback to the streaming data from the web client if needed.
+    new_player_response = try_fetch_streaming_data(video_id, client_config)
-    # See: https://github.com/iv-org/invidious/issues/2549
-    if video_id != android_player.dig("videoDetails", "videoId")
-      # YouTube may return a different video player response than expected.
-      # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
-      raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (ANDROID client)")
-    elsif android_player["playabilityStatus"]["status"] == "OK"
-      params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("")
-    else
-      params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("")
-    end
   end
 
-  # TODO: clean that up
+  # Last hope
-  {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f|
+  if new_player_response.nil?
+    client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
+    new_player_response = try_fetch_streaming_data(video_id, client_config)
+  end
+
+  # Replace player response and reset reason
+  if !new_player_response.nil?
+    player_response = new_player_response
+    params.delete("reason")
+  end
+
+  {"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f|
     params[f] = player_response[f] if player_response[f]?
   end
 
@@ -128,6 +126,26 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
   return params
 end
 
+def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
+  LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
+  response = YoutubeAPI.player(video_id: id, params: "", client_config: client_config)
+
+  playability_status = response["playabilityStatus"]["status"]
+  LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
+
+  if id != response.dig("videoDetails", "videoId")
+    # YouTube may return a different video player response than expected.
+    # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
+    raise VideoNotAvailableException.new(
+      "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)"
+    )
+  elsif playability_status == "OK"
+    return response
+  else
+    return nil
+  end
+end
+
 def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any)
   # Top level elements