CAPTION_LANGUAGES = { "", "English", "English (auto-generated)", "Afrikaans", "Albanian", "Amharic", "Arabic", "Armenian", "Azerbaijani", "Bangla", "Basque", "Belarusian", "Bosnian", "Bulgarian", "Burmese", "Catalan", "Cebuano", "Chinese (Simplified)", "Chinese (Traditional)", "Corsican", "Croatian", "Czech", "Danish", "Dutch", "Esperanto", "Estonian", "Filipino", "Finnish", "French", "Galician", "Georgian", "German", "Greek", "Gujarati", "Haitian Creole", "Hausa", "Hawaiian", "Hebrew", "Hindi", "Hmong", "Hungarian", "Icelandic", "Igbo", "Indonesian", "Irish", "Italian", "Japanese", "Javanese", "Kannada", "Kazakh", "Khmer", "Korean", "Kurdish", "Kyrgyz", "Lao", "Latin", "Latvian", "Lithuanian", "Luxembourgish", "Macedonian", "Malagasy", "Malay", "Malayalam", "Maltese", "Maori", "Marathi", "Mongolian", "Nepali", "Norwegian BokmÃ¥l", "Nyanja", "Pashto", "Persian", "Polish", "Portuguese", "Punjabi", "Romanian", "Russian", "Samoan", "Scottish Gaelic", "Serbian", "Shona", "Sindhi", "Sinhala", "Slovak", "Slovenian", "Somali", "Southern Sotho", "Spanish", "Spanish (Latin America)", "Sundanese", "Swahili", "Swedish", "Tajik", "Tamil", "Telugu", "Thai", "Turkish", "Ukrainian", "Urdu", "Uzbek", "Vietnamese", "Welsh", "Western Frisian", "Xhosa", "Yiddish", "Yoruba", "Zulu", } REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"} # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 VIDEO_FORMATS = { "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, # 3D videos "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, # Apple HTTP Live Streaming "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, # DASH mp4 video "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559) "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, # Dash mp4 audio "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, # Dash webm "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, # Dash webm audio "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, # Dash webm audio with opus inside "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, # av01 video only formats sometimes served with "unknown" codecs "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"}, "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"}, "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"}, "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, } struct VideoPreferences include JSON::Serializable property annotations : Bool property autoplay : Bool property comments : Array(String) property continue : Bool property continue_autoplay : Bool property controls : Bool property listen : Bool property local : Bool property preferred_captions : Array(String) property player_style : String property quality : String property quality_dash : String property raw : Bool property region : String? property related_videos : Bool property speed : Float32 | Float64 property video_end : Float64 | Int32 property video_loop : Bool property extend_desc : Bool property video_start : Float64 | Int32 property volume : Int32 property vr_mode : Bool end struct Video include DB::Serializable property id : String @[DB::Field(converter: Video::JSONConverter)] property info : Hash(String, JSON::Any) property updated : Time @[DB::Field(ignore: true)] property captions : Array(Caption)? @[DB::Field(ignore: true)] property adaptive_fmts : Array(Hash(String, JSON::Any))? @[DB::Field(ignore: true)] property fmt_stream : Array(Hash(String, JSON::Any))? @[DB::Field(ignore: true)] property description : String? module JSONConverter def self.from_rs(rs) JSON.parse(rs.read(String)).as_h end end def to_json(locale, json : JSON::Builder) json.object do json.field "type", "video" json.field "title", self.title json.field "videoId", self.id json.field "error", info["reason"] if info["reason"]? json.field "videoThumbnails" do generate_thumbnails(json, self.id) end json.field "storyboards" do generate_storyboards(json, self.id, self.storyboards) end json.field "description", self.description json.field "descriptionHtml", self.description_html json.field "published", self.published.to_unix json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) json.field "keywords", self.keywords json.field "viewCount", self.views json.field "likeCount", self.likes json.field "dislikeCount", self.dislikes json.field "paid", self.paid json.field "premium", self.premium json.field "isFamilyFriendly", self.is_family_friendly json.field "allowedRegions", self.allowed_regions json.field "genre", self.genre json.field "genreUrl", self.genre_url json.field "author", self.author json.field "authorId", self.ucid json.field "authorUrl", "/channel/#{self.ucid}" json.field "authorThumbnails" do json.array do qualities = {32, 48, 76, 100, 176, 512} qualities.each do |quality| json.object do json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") json.field "width", quality json.field "height", quality end end end end json.field "subCountText", self.sub_count_text json.field "lengthSeconds", self.length_seconds json.field "allowRatings", self.allow_ratings json.field "rating", self.average_rating json.field "isListed", self.is_listed json.field "liveNow", self.live_now json.field "isUpcoming", self.is_upcoming if self.premiere_timestamp json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix end if hlsvp = self.hls_manifest_url hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL) json.field "hlsUrl", hlsvp end json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}" json.field "adaptiveFormats" do json.array do self.adaptive_fmts.each do |fmt| json.object do json.field "index", "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}" json.field "bitrate", fmt["bitrate"].as_i.to_s json.field "init", "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}" json.field "url", fmt["url"] json.field "itag", fmt["itag"].as_i.to_s json.field "type", fmt["mimeType"] json.field "clen", fmt["contentLength"] json.field "lmt", fmt["lastModified"] json.field "projectionType", fmt["projectionType"] fmt_info = itag_to_metadata?(fmt["itag"]) if fmt_info fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 json.field "fps", fps json.field "container", fmt_info["ext"] json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] if fmt_info["height"]? json.field "resolution", "#{fmt_info["height"]}p" quality_label = "#{fmt_info["height"]}p" if fps > 30 quality_label += "60" end json.field "qualityLabel", quality_label if fmt_info["width"]? json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" end end end end end end end json.field "formatStreams" do json.array do self.fmt_stream.each do |fmt| json.object do json.field "url", fmt["url"] json.field "itag", fmt["itag"].as_i.to_s json.field "type", fmt["mimeType"] json.field "quality", fmt["quality"] fmt_info = itag_to_metadata?(fmt["itag"]) if fmt_info fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30 json.field "fps", fps json.field "container", fmt_info["ext"] json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] if fmt_info["height"]? json.field "resolution", "#{fmt_info["height"]}p" quality_label = "#{fmt_info["height"]}p" if fps > 30 quality_label += "60" end json.field "qualityLabel", quality_label if fmt_info["width"]? json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" end end end end end end end json.field "captions" do json.array do self.captions.each do |caption| json.object do json.field "label", caption.name json.field "languageCode", caption.languageCode json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}" end end end end json.field "recommendedVideos" do json.array do self.related_videos.each do |rv| if rv["id"]? json.object do json.field "videoId", rv["id"] json.field "title", rv["title"] json.field "videoThumbnails" do generate_thumbnails(json, rv["id"]) end json.field "author", rv["author"] json.field "authorUrl", rv["author_url"]? json.field "authorId", rv["ucid"]? if rv["author_thumbnail"]? json.field "authorThumbnails" do json.array do qualities = {32, 48, 76, 100, 176, 512} qualities.each do |quality| json.object do json.field "url", rv["author_thumbnail"]?.try &.gsub(/s\d+-/, "s#{quality}-") json.field "width", quality json.field "height", quality end end end end end json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i json.field "viewCountText", rv["short_view_count_text"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 end end end end end end end def to_json(locale, json : JSON::Builder | Nil = nil) if json to_json(locale, json) else JSON.build do |json| to_json(locale, json) end end end def title info["videoDetails"]["title"]?.try &.as_s || "" end def ucid info["videoDetails"]["channelId"]?.try &.as_s || "" end def author info["videoDetails"]["author"]?.try &.as_s || "" end def length_seconds : Int32 info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["lengthSeconds"]?.try &.as_s.to_i || info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0 end def views : Int64 info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64 end def likes : Int64 info["likes"]?.try &.as_i64 || 0_i64 end def dislikes : Int64 info["dislikes"]?.try &.as_i64 || 0_i64 end def average_rating : Float64 # (likes / (likes + dislikes) * 4 + 1) info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0 end def published : Time info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc end def published=(other : Time) info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d")) end def allow_ratings r = info["videoDetails"]["allowRatings"]?.try &.as_bool r.nil? ? false : r end def live_now info["microformat"]?.try &.["playerMicroformatRenderer"]? .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false end def is_listed info["videoDetails"]["isCrawlable"]?.try &.as_bool || false end def is_upcoming info["videoDetails"]["isUpcoming"]?.try &.as_bool || false end def premiere_timestamp : Time? info["microformat"]?.try &.["playerMicroformatRenderer"]? .try &.["liveBroadcastDetails"]?.try &.["startTimestamp"]?.try { |t| Time.parse_rfc3339(t.as_s) } end def keywords info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String end def related_videos info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String) end def allowed_regions info["microformat"]?.try &.["playerMicroformatRenderer"]? .try &.["availableCountries"]?.try &.as_a.map &.as_s || [] of String end def author_thumbnail : String info["authorThumbnail"]?.try &.as_s || "" end def sub_count_text : String info["subCountText"]?.try &.as_s || "-" end def fmt_stream return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) fmt_stream.each do |fmt| if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } s.each do |k, v| fmt[k] = JSON::Any.new(v) end fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") end fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? end fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } @fmt_stream = fmt_stream return @fmt_stream.as(Array(Hash(String, JSON::Any))) end def adaptive_fmts return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any) fmt_stream.each do |fmt| if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) } s.each do |k, v| fmt[k] = JSON::Any.new(v) end fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}") end fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}") fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]? end # See https://github.com/TeamNewPipe/NewPipe/issues/2415 # Some streams are segmented by URL `sq/` rather than index, for now we just filter them out fmt_stream.reject! { |f| !f["indexRange"]? } fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 } @adaptive_fmts = fmt_stream return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) end def video_streams adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("video") end def audio_streams adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio") end def storyboards storyboards = info["storyboards"]? .try &.as_h .try &.["playerStoryboardSpecRenderer"]? .try &.["spec"]? .try &.as_s.split("|") if !storyboards if storyboard = info["storyboards"]? .try &.as_h .try &.["playerLiveStoryboardSpecRenderer"]? .try &.["spec"]? .try &.as_s return [{ url: storyboard.split("#")[0], width: 106, height: 60, count: -1, interval: 5000, storyboard_width: 3, storyboard_height: 3, storyboard_count: -1, }] end end items = [] of NamedTuple( url: String, width: Int32, height: Int32, count: Int32, interval: Int32, storyboard_width: Int32, storyboard_height: Int32, storyboard_count: Int32) return items if !storyboards url = URI.parse(storyboards.shift) params = HTTP::Params.parse(url.query || "") storyboards.each_with_index do |storyboard, i| width, height, count, storyboard_width, storyboard_height, interval, _, sigh = storyboard.split("#") params["sigh"] = sigh url.query = params.to_s width = width.to_i height = height.to_i count = count.to_i interval = interval.to_i storyboard_width = storyboard_width.to_i storyboard_height = storyboard_height.to_i storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i items << { url: url.to_s.sub("$L", i).sub("$N", "M$M"), width: width, height: height, count: count, interval: interval, storyboard_width: storyboard_width, storyboard_height: storyboard_height, storyboard_count: storyboard_count, } end items end def paid reason = info["playabilityStatus"]?.try &.["reason"]? paid = reason == "This video requires payment to watch." ? true : false paid end def premium keywords.includes? "YouTube Red" end def captions : Array(Caption) return @captions.as(Array(Caption)) if @captions captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption| name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"] languageCode = caption["languageCode"].to_s baseUrl = caption["baseUrl"].to_s caption = Caption.new(name.to_s, languageCode, baseUrl) caption.name = caption.name.split(" - ")[0] caption end captions ||= [] of Caption @captions = captions return @captions.as(Array(Caption)) end def description description = info["microformat"]?.try &.["playerMicroformatRenderer"]? .try &.["description"]?.try &.["simpleText"]?.try &.as_s || "" end # TODO def description=(value : String) @description = value end def description_html info["descriptionHtml"]?.try &.as_s || "

" end def description_html=(value : String) info["descriptionHtml"] = JSON::Any.new(value) end def short_description info["shortDescription"]?.try &.as_s? || "" end def hls_manifest_url : String? info["streamingData"]?.try &.["hlsManifestUrl"]?.try &.as_s end def dash_manifest_url info["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s end def genre : String info["genre"]?.try &.as_s || "" end def genre_url : String? info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil end def license : String? info["license"]?.try &.as_s end def is_family_friendly : Bool info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false end def is_vr : Bool? info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s == "MESH" end def wilson_score : Float64 ci_lower_bound(likes, likes + dislikes).round(4) end def engagement : Float64 (((likes + dislikes) / views) * 100).round(4) end def reason : String? info["reason"]?.try &.as_s end end struct Caption property name property languageCode property baseUrl getter name : String getter languageCode : String getter baseUrl : String setter name def initialize(@name, @languageCode, @baseUrl) end end class VideoRedirect < Exception property video_id : String def initialize(@video_id) end end def parse_related(r : JSON::Any) : JSON::Any? # TODO: r["endScreenPlaylistRenderer"], etc. return if !r["endScreenVideoRenderer"]? r = r["endScreenVideoRenderer"].as_h return if !r["lengthInSeconds"]? rv = {} of String => JSON::Any rv["author"] = r["shortBylineText"]["runs"][0]?.try &.["text"] || JSON::Any.new("") rv["ucid"] = r["shortBylineText"]["runs"][0]?.try &.["navigationEndpoint"]["browseEndpoint"]["browseId"] || JSON::Any.new("") rv["author_url"] = JSON::Any.new("/channel/#{rv["ucid"]}") rv["length_seconds"] = JSON::Any.new(r["lengthInSeconds"].as_i.to_s) rv["title"] = r["title"]["simpleText"] rv["short_view_count_text"] = JSON::Any.new(r["shortViewCountText"]?.try &.["simpleText"]?.try &.as_s || "") rv["view_count"] = JSON::Any.new(r["title"]["accessibility"]?.try &.["accessibilityData"]["label"].as_s.match(/(?[1-9](\d+,?)*) views/).try &.["views"].gsub(/\D/, "") || "") rv["id"] = r["videoId"] JSON::Any.new(rv) end def extract_video_info(video_id : String, proxy_region : String? = nil) params = {} of String => JSON::Any client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) if player_response["playabilityStatus"]?.try &.["status"]?.try &.as_s != "OK" reason = player_response["playabilityStatus"]["errorScreen"]?.try &.["playerErrorMessageRenderer"]?.try &.["subreason"]?.try { |s| s["simpleText"]?.try &.as_s || s["runs"].as_a.map { |r| r["text"] }.join("") } || player_response["playabilityStatus"]["reason"].as_s params["reason"] = JSON::Any.new(reason) end params["shortDescription"] = player_response.dig?("videoDetails", "shortDescription") || JSON::Any.new(nil) # Don't fetch the next endpoint if the video is unavailable. if !params["reason"]? next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) player_response = player_response.merge(next_response) end # Fetch the video streams using an Android client in order to get the decrypted URLs and # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs: # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 if !params["reason"]? client_config.client_type = YoutubeAPI::ClientType::Android stream_data = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) params["streamingData"] = stream_data["streamingData"]? || JSON::Any.new("") end {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f| params[f] = player_response[f] if player_response[f]? end params["relatedVideos"] = ( player_response .dig?("playerOverlays", "playerOverlayRenderer", "endScreen", "watchNextEndScreenRenderer", "results") .try &.as_a.compact_map { |r| parse_related r } || \ player_response .dig?("webWatchNextResponseExtensionData", "relatedVideoArgs") .try &.as_s.split(",").map { |r| r = HTTP::Params.parse(r).to_h JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) })) } ).try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) primary_results = player_response.try &.["contents"]?.try &.["twoColumnWatchNextResults"]?.try &.["results"]? .try &.["results"]?.try &.["contents"]? sentiment_bar = primary_results.try &.as_a.select { |object| object["videoPrimaryInfoRenderer"]? }[0]? .try &.["videoPrimaryInfoRenderer"]? .try &.["sentimentBar"]? .try &.["sentimentBarRenderer"]? .try &.["tooltip"]? .try &.as_s likes, dislikes = sentiment_bar.try &.split(" / ", 2).map &.gsub(/\D/, "").to_i64 || {0_i64, 0_i64} params["likes"] = JSON::Any.new(likes) params["dislikes"] = JSON::Any.new(dislikes) params["descriptionHtml"] = JSON::Any.new(primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? .try &.["videoSecondaryInfoRenderer"]?.try &.["description"]?.try &.["runs"]? .try &.as_a.try { |t| content_to_comment_html(t).gsub("\n", "
") } || "

") metadata = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? .try &.["videoSecondaryInfoRenderer"]? .try &.["metadataRowContainer"]? .try &.["metadataRowContainerRenderer"]? .try &.["rows"]? .try &.as_a params["genre"] = params["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["category"]? || JSON::Any.new("") params["genreUrl"] = JSON::Any.new(nil) metadata.try &.each do |row| title = row["metadataRowRenderer"]?.try &.["title"]?.try &.["simpleText"]?.try &.as_s contents = row["metadataRowRenderer"]? .try &.["contents"]? .try &.as_a[0]? if title.try &.== "Category" contents = contents.try &.["runs"]? .try &.as_a[0]? params["genre"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "") params["genreUcid"] = JSON::Any.new(contents.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]? .try &.["browseId"]?.try &.as_s || "") elsif title.try &.== "License" contents = contents.try &.["runs"]? .try &.as_a[0]? params["license"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "") elsif title.try &.== "Licensed to YouTube by" params["license"] = JSON::Any.new(contents.try &.["simpleText"]?.try &.as_s || "") end end author_info = primary_results.try &.as_a.select { |object| object["videoSecondaryInfoRenderer"]? }[0]? .try &.["videoSecondaryInfoRenderer"]?.try &.["owner"]?.try &.["videoOwnerRenderer"]? params["authorThumbnail"] = JSON::Any.new(author_info.try &.["thumbnail"]? .try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"]? .try &.as_s || "") params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]? .try { |t| t["simpleText"]? || t["runs"]?.try &.[0]?.try &.["text"]? }.try &.as_s.split(" ", 2)[0] || "-") params end def get_video(id, db, refresh = true, region = nil, force_refresh = false) if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) if (refresh && (Time.utc - video.updated > 10.minutes) || (video.premiere_timestamp.try &.< Time.utc)) || force_refresh begin video = fetch_video(id, region) db.exec("UPDATE videos SET (id, info, updated) = ($1, $2, $3) WHERE id = $1", video.id, video.info.to_json, video.updated) rescue ex db.exec("DELETE FROM videos * WHERE id = $1", id) raise ex end end else video = fetch_video(id, region) if !region db.exec("INSERT INTO videos VALUES ($1, $2, $3) ON CONFLICT (id) DO NOTHING", video.id, video.info.to_json, video.updated) end end return video end def fetch_video(id, region) info = extract_video_info(video_id: id) allowed_regions = info .dig?("microformat", "playerMicroformatRenderer", "availableCountries") .try &.as_a.map &.as_s || [] of String # Check for region-blocks if info["reason"]?.try &.as_s.includes?("your country") bypass_regions = PROXY_LIST.keys & allowed_regions if !bypass_regions.empty? region = bypass_regions[rand(bypass_regions.size)] region_info = extract_video_info(video_id: id, proxy_region: region) region_info["region"] = JSON::Any.new(region) if region info = region_info if !region_info["reason"]? end end # Try to pull streams from embed URL if info["reason"]? required_parameters = { "video_id" => id, "eurl" => "https://youtube.googleapis.com/v/#{id}", "html5" => "1", "gl" => "US", "hl" => "en", } if info["reason"].as_s.includes?("inappropriate") # The html5, c and cver parameters are required in order to extract age-restricted videos # See https://github.com/yt-dlp/yt-dlp/commit/4e6767b5f2e2523ebd3dd1240584ead53e8c8905 required_parameters.merge!({ "c" => "TVHTML5", "cver" => "6.20180913", }) # In order to actually extract video info without error, the `x-youtube-client-version` # has to be set to the same version as `cver` above. additional_headers = HTTP::Headers{"x-youtube-client-version" => "6.20180913"} else embed_page = YT_POOL.client &.get("/embed/#{id}").body sts = embed_page.match(/"sts"\s*:\s*(?\d+)/).try &.["sts"]? || "" required_parameters["sts"] = sts additional_headers = HTTP::Headers{} of String => String end embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?#{URI::Params.encode(required_parameters)}", headers: additional_headers).body) if embed_info["player_response"]? player_response = JSON.parse(embed_info["player_response"]) {"captions", "microformat", "playabilityStatus", "streamingData", "videoDetails", "storyboards"}.each do |f| info[f] = player_response[f] if player_response[f]? end end initial_data = JSON.parse(embed_info["watch_next_response"]) if embed_info["watch_next_response"]? info["relatedVideos"] = initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]? .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r| parse_related r }.try { |a| JSON::Any.new(a) } || embed_info["rvs"]?.try &.split(",").map { |r| r = HTTP::Params.parse(r).to_h JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) })) }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) end raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]? video = Video.new({ id: id, info: info, updated: Time.utc, }) return video end def itag_to_metadata?(itag : JSON::Any) return VIDEO_FORMATS[itag.to_s]? end def process_continuation(db, query, plid, id) continuation = nil if plid if index = query["index"]?.try &.to_i? continuation = index else continuation = id end continuation ||= 0 end continuation end def process_video_params(query, preferences) annotations = query["iv_load_policy"]?.try &.to_i? autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } comments = query["comments"]?.try &.split(",").map { |a| a.downcase } continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } player_style = query["player_style"]? preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase } quality = query["quality"]? quality_dash = query["quality_dash"]? region = query["region"]? related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } speed = query["speed"]?.try &.rchop("x").to_f? video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } volume = query["volume"]?.try &.to_i? vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } if preferences # region ||= preferences.region annotations ||= preferences.annotations.to_unsafe autoplay ||= preferences.autoplay.to_unsafe comments ||= preferences.comments continue ||= preferences.continue.to_unsafe continue_autoplay ||= preferences.continue_autoplay.to_unsafe listen ||= preferences.listen.to_unsafe local ||= preferences.local.to_unsafe player_style ||= preferences.player_style preferred_captions ||= preferences.captions quality ||= preferences.quality quality_dash ||= preferences.quality_dash related_videos ||= preferences.related_videos.to_unsafe speed ||= preferences.speed video_loop ||= preferences.video_loop.to_unsafe extend_desc ||= preferences.extend_desc.to_unsafe volume ||= preferences.volume vr_mode ||= preferences.vr_mode.to_unsafe end annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe comments ||= CONFIG.default_user_preferences.comments continue ||= CONFIG.default_user_preferences.continue.to_unsafe continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe listen ||= CONFIG.default_user_preferences.listen.to_unsafe local ||= CONFIG.default_user_preferences.local.to_unsafe player_style ||= CONFIG.default_user_preferences.player_style preferred_captions ||= CONFIG.default_user_preferences.captions quality ||= CONFIG.default_user_preferences.quality quality_dash ||= CONFIG.default_user_preferences.quality_dash related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe speed ||= CONFIG.default_user_preferences.speed video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe volume ||= CONFIG.default_user_preferences.volume vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe annotations = annotations == 1 autoplay = autoplay == 1 continue = continue == 1 continue_autoplay = continue_autoplay == 1 listen = listen == 1 local = local == 1 related_videos = related_videos == 1 video_loop = video_loop == 1 extend_desc = extend_desc == 1 vr_mode = vr_mode == 1 if CONFIG.disabled?("dash") && quality == "dash" quality = "high" end if CONFIG.disabled?("local") && local local = false end if start = query["t"]? || query["time_continue"]? || query["start"]? video_start = decode_time(start) end video_start ||= 0 if query["end"]? video_end = decode_time(query["end"]) end video_end ||= -1 raw = query["raw"]?.try &.to_i? raw ||= 0 raw = raw == 1 controls = query["controls"]?.try &.to_i? controls ||= 1 controls = controls >= 1 params = VideoPreferences.new({ annotations: annotations, autoplay: autoplay, comments: comments, continue: continue, continue_autoplay: continue_autoplay, controls: controls, listen: listen, local: local, player_style: player_style, preferred_captions: preferred_captions, quality: quality, quality_dash: quality_dash, raw: raw, region: region, related_videos: related_videos, speed: speed, video_end: video_end, video_loop: video_loop, extend_desc: extend_desc, video_start: video_start, volume: volume, vr_mode: vr_mode, }) return params end def build_thumbnails(id) return { {host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"}, {host: HOST_URL, height: 720, width: 1280, name: "maxresdefault", url: "maxresdefault"}, {host: HOST_URL, height: 480, width: 640, name: "sddefault", url: "sddefault"}, {host: HOST_URL, height: 360, width: 480, name: "high", url: "hqdefault"}, {host: HOST_URL, height: 180, width: 320, name: "medium", url: "mqdefault"}, {host: HOST_URL, height: 90, width: 120, name: "default", url: "default"}, {host: HOST_URL, height: 90, width: 120, name: "start", url: "1"}, {host: HOST_URL, height: 90, width: 120, name: "middle", url: "2"}, {host: HOST_URL, height: 90, width: 120, name: "end", url: "3"}, } end def generate_thumbnails(json, id) json.array do build_thumbnails(id).each do |thumbnail| json.object do json.field "quality", thumbnail[:name] json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" json.field "width", thumbnail[:width] json.field "height", thumbnail[:height] end end end end def generate_storyboards(json, id, storyboards) json.array do storyboards.each do |storyboard| json.object do json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}" json.field "templateUrl", storyboard[:url] json.field "width", storyboard[:width] json.field "height", storyboard[:height] json.field "count", storyboard[:count] json.field "interval", storyboard[:interval] json.field "storyboardWidth", storyboard[:storyboard_width] json.field "storyboardHeight", storyboard[:storyboard_height] json.field "storyboardCount", storyboard[:storyboard_count] end end end end