struct PlaylistVideo def to_xml(host_url, auto_generated, xml : XML::Builder) xml.element("entry") do xml.element("id") { xml.text "yt:video:#{self.id}" } xml.element("yt:videoId") { xml.text self.id } xml.element("yt:channelId") { xml.text self.ucid } xml.element("title") { xml.text self.title } xml.element("link", rel: "alternate", href: "#{host_url}/watch?v=#{self.id}") xml.element("author") do if auto_generated xml.element("name") { xml.text self.author } xml.element("uri") { xml.text "#{host_url}/channel/#{self.ucid}" } else xml.element("name") { xml.text author } xml.element("uri") { xml.text "#{host_url}/channel/#{ucid}" } end end xml.element("content", type: "xhtml") do xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do xml.element("a", href: "#{host_url}/watch?v=#{self.id}") do xml.element("img", src: "#{host_url}/vi/#{self.id}/mqdefault.jpg") end end end xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") } xml.element("media:group") do xml.element("media:title") { xml.text self.title } xml.element("media:thumbnail", url: "#{host_url}/vi/#{self.id}/mqdefault.jpg", width: "320", height: "180") end end end def to_xml(host_url, auto_generated, xml : XML::Builder? = nil) if xml to_xml(host_url, auto_generated, xml) else XML.build do |json| to_xml(host_url, auto_generated, xml) end end end def to_json(locale, config, kemal_config, json : JSON::Builder, index : Int32?) json.object do json.field "title", self.title json.field "videoId", self.id json.field "author", self.author json.field "authorId", self.ucid json.field "authorUrl", "/channel/#{self.ucid}" json.field "videoThumbnails" do generate_thumbnails(json, self.id, config, kemal_config) end if index json.field "index", index json.field "indexId", self.index.to_u64.to_s(16).upcase else json.field "index", self.index end json.field "lengthSeconds", self.length_seconds end end def to_json(locale, config, kemal_config, json : JSON::Builder? = nil, index : Int32? = nil) if json to_json(locale, config, kemal_config, json, index: index) else JSON.build do |json| to_json(locale, config, kemal_config, json, index: index) end end end db_mapping({ title: String, id: String, author: String, ucid: String, length_seconds: Int32, published: Time, plid: String, index: Int64, live_now: Bool, }) end struct Playlist def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil) json.object do json.field "type", "playlist" json.field "title", self.title json.field "playlistId", self.id json.field "playlistThumbnail", self.thumbnail json.field "author", self.author json.field "authorId", self.ucid json.field "authorUrl", "/channel/#{self.ucid}" json.field "authorThumbnails" do json.array do qualities = {32, 48, 76, 100, 176, 512} qualities.each do |quality| json.object do json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}") json.field "width", quality json.field "height", quality end end end end json.field "description", html_to_content(self.description_html) json.field "descriptionHtml", self.description_html json.field "videoCount", self.video_count json.field "viewCount", self.views json.field "updated", self.updated.to_unix json.field "isListed", self.privacy.public? json.field "videos" do json.array do videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) videos.each_with_index do |video, index| video.to_json(locale, config, Kemal.config, json) end end end end end def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil) if json to_json(offset, locale, config, kemal_config, json, continuation: continuation) else JSON.build do |json| to_json(offset, locale, config, kemal_config, json, continuation: continuation) end end end db_mapping({ title: String, id: String, author: String, author_thumbnail: String, ucid: String, description_html: String, video_count: Int32, views: Int64, updated: Time, thumbnail: String?, }) def privacy PlaylistPrivacy::Public end end enum PlaylistPrivacy Public = 0 Unlisted = 1 Private = 2 end struct InvidiousPlaylist def to_json(offset, locale, config, kemal_config, json : JSON::Builder, continuation : String? = nil) json.object do json.field "type", "invidiousPlaylist" json.field "title", self.title json.field "playlistId", self.id json.field "author", self.author json.field "authorId", self.ucid json.field "authorUrl", nil json.field "authorThumbnails", [] of String json.field "description", html_to_content(self.description_html) json.field "descriptionHtml", self.description_html json.field "videoCount", self.video_count json.field "viewCount", self.views json.field "updated", self.updated.to_unix json.field "isListed", self.privacy.public? json.field "videos" do json.array do videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation) videos.each_with_index do |video, index| video.to_json(locale, config, Kemal.config, json, offset + index) end end end end end def to_json(offset, locale, config, kemal_config, json : JSON::Builder? = nil, continuation : String? = nil) if json to_json(offset, locale, config, kemal_config, json, continuation: continuation) else JSON.build do |json| to_json(offset, locale, config, kemal_config, json, continuation: continuation) end end end property thumbnail_id module PlaylistPrivacyConverter def self.from_rs(rs) return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8)))) end end db_mapping({ title: String, id: String, author: String, description: {type: String, default: ""}, video_count: Int32, created: Time, updated: Time, privacy: {type: PlaylistPrivacy, default: PlaylistPrivacy::Private, converter: PlaylistPrivacyConverter}, index: Array(Int64), }) def thumbnail @thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------" "/vi/#{@thumbnail_id}/mqdefault.jpg" end def author_thumbnail nil end def ucid nil end def views 0_i64 end def description_html HTML.escape(self.description).gsub("\n", "
") end end def create_playlist(db, title, privacy, user) plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}" playlist = InvidiousPlaylist.new( title: title.byte_slice(0, 150), id: plid, author: user.email, description: "", # Max 5000 characters video_count: 0, created: Time.utc, updated: Time.utc, privacy: privacy, index: [] of Int64, ) playlist_array = playlist.to_a args = arg_array(playlist_array) db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array) return playlist end def extract_playlist(plid, nodeset, index) videos = [] of PlaylistVideo nodeset.each_with_index do |video, offset| anchor = video.xpath_node(%q(.//td[@class="pl-video-title"])) if !anchor next end title = anchor.xpath_node(%q(.//a)).not_nil!.content.strip(" \n") id = anchor.xpath_node(%q(.//a)).not_nil!["href"].lchop("/watch?v=")[0, 11] anchor = anchor.xpath_node(%q(.//div[@class="pl-video-owner"]/a)) if anchor author = anchor.content ucid = anchor["href"].split("/")[2] else author = "" ucid = "" end anchor = video.xpath_node(%q(.//td[@class="pl-video-time"]/div/div[1])) if anchor && !anchor.content.empty? length_seconds = decode_length_seconds(anchor.content) live_now = false else length_seconds = 0 live_now = true end videos << PlaylistVideo.new( title: title, id: id, author: author, ucid: ucid, length_seconds: length_seconds, published: Time.utc, plid: plid, index: (index + offset).to_i64, live_now: live_now ) end return videos end def produce_playlist_url(id, index) if id.starts_with? "UC" id = "UU" + id.lchop("UC") end ucid = "VL" + id data = IO::Memory.new data.write_byte 0x08 VarInt.to_io(data, index) data.rewind data = Base64.urlsafe_encode(data, false) data = "PT:#{data}" continuation = IO::Memory.new continuation.write_byte 0x7a VarInt.to_io(continuation, data.bytesize) continuation.print data data = Base64.urlsafe_encode(continuation) cursor = URI.encode_www_form(data) data = IO::Memory.new data.write_byte 0x12 VarInt.to_io(data, ucid.bytesize) data.print ucid data.write_byte 0x1a VarInt.to_io(data, cursor.bytesize) data.print cursor data.rewind buffer = IO::Memory.new buffer.write Bytes[0xe2, 0xa9, 0x85, 0xb2, 0x02] VarInt.to_io(buffer, data.bytesize) IO.copy data, buffer continuation = Base64.urlsafe_encode(buffer) continuation = URI.encode_www_form(continuation) url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" return url end def get_playlist(db, plid, locale, refresh = true, force_refresh = false) if plid.starts_with? "IV" if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) return playlist else raise "Playlist does not exist." end else return fetch_playlist(plid, locale) end end def fetch_playlist(plid, locale) client = make_client(YT_URL) if plid.starts_with? "UC" plid = "UU#{plid.lchop("UC")}" end response = client.get("/playlist?list=#{plid}&hl=en&disable_polymer=1") if response.status_code != 200 raise translate(locale, "Not a playlist.") end body = response.body.gsub(/]+>]+>\s*less\s*]+>\n<\/span><\/button>/, "") document = XML.parse_html(body) title = document.xpath_node(%q(//h1[@class="pl-header-title"])) if !title raise translate(locale, "Playlist does not exist.") end title = title.content.strip(" \n") description_html = document.xpath_node(%q(//span[@class="pl-header-description-text"]/div/div[1])).try &.to_s || document.xpath_node(%q(//span[@class="pl-header-description-text"])).try &.to_s || "" playlist_thumbnail = document.xpath_node(%q(//div[@class="pl-header-thumb"]/img)).try &.["data-thumb"]? || document.xpath_node(%q(//div[@class="pl-header-thumb"]/img)).try &.["src"] # YouTube allows anonymous playlists, so most of this can be empty or optional anchor = document.xpath_node(%q(//ul[@class="pl-header-details"])) author = anchor.try &.xpath_node(%q(.//li[1]/a)).try &.content author ||= "" author_thumbnail = document.xpath_node(%q(//img[@class="channel-header-profile-image"])).try &.["src"] author_thumbnail ||= "" ucid = anchor.try &.xpath_node(%q(.//li[1]/a)).try &.["href"].split("/")[-1] ucid ||= "" video_count = anchor.try &.xpath_node(%q(.//li[2])).try &.content.gsub(/\D/, "").to_i? video_count ||= 0 views = anchor.try &.xpath_node(%q(.//li[3])).try &.content.gsub(/\D/, "").to_i64? views ||= 0_i64 updated = anchor.try &.xpath_node(%q(.//li[4])).try &.content.lchop("Last updated on ").lchop("Updated ").try { |date| decode_date(date) } updated ||= Time.utc playlist = Playlist.new( title: title, id: plid, author: author, author_thumbnail: author_thumbnail, ucid: ucid, description_html: description_html, video_count: video_count, views: views, updated: updated, thumbnail: playlist_thumbnail, ) return playlist end def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil) if playlist.is_a? InvidiousPlaylist if !offset index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", playlist.id, continuation, as: Int64) offset = playlist.index.index(index) || 0 end db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", playlist.id, playlist.index, offset, as: PlaylistVideo) else fetch_playlist_videos(playlist.id, playlist.video_count, offset, locale, continuation) end end def fetch_playlist_videos(plid, video_count, offset = 0, locale = nil, continuation = nil) client = make_client(YT_URL) if continuation html = client.get("/watch?v=#{continuation}&list=#{plid}&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999") html = XML.parse_html(html.body) index = html.xpath_node(%q(//span[@id="playlist-current-index"])).try &.content.to_i?.try &.- 1 offset = index || offset end if video_count > 100 url = produce_playlist_url(plid, offset) response = client.get(url) response = JSON.parse(response.body) if !response["content_html"]? || response["content_html"].as_s.empty? raise translate(locale, "Empty playlist") end document = XML.parse_html(response["content_html"].as_s) nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")])) videos = extract_playlist(plid, nodeset, offset) elsif offset > 100 return [] of PlaylistVideo else # Extract first page of videos response = client.get("/playlist?list=#{plid}&gl=US&hl=en&disable_polymer=1") document = XML.parse_html(response.body) nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")])) videos = extract_playlist(plid, nodeset, 0) end until videos.empty? || videos[0].index == offset videos.shift end return videos end def template_playlist(playlist) html = <<-END_HTML

#{playlist["title"]}

    END_HTML playlist["videos"].as_a.each do |video| html += <<-END_HTML
  1. #{recode_length_seconds(video["lengthSeconds"].as_i)}

    #{video["title"]}

    #{video["author"]}

  2. END_HTML end html += <<-END_HTML

END_HTML html end