Merge pull request #51 from omarroth/data-control
Add options to import and export user data
This commit is contained in:
Omar Roth
GitHub
commit
381b644dab
|
|
@ -11,13 +11,16 @@ targets:
|
|||
dependencies:
|
||||
kemal:
|
||||
github: kemalcr/kemal
|
||||
branch: master
|
||||
branch: rework-param-parser
|
||||
pg:
|
||||
github: will/crystal-pg
|
||||
branch: master
|
||||
detect_language:
|
||||
github: detectlanguage/detectlanguage-crystal
|
||||
branch: master
|
||||
sqlite3:
|
||||
github: crystal-lang/crystal-sqlite3
|
||||
branch: master
|
||||
|
||||
crystal: 0.25.1
|
||||
|
||||
|
|
|
|||
181
src/invidious.cr
181
src/invidious.cr
|
|
@ -22,6 +22,7 @@ require "option_parser"
|
|||
require "pg"
|
||||
require "xml"
|
||||
require "yaml"
|
||||
require "zip"
|
||||
require "./invidious/*"
|
||||
|
||||
CONFIG = Config.from_yaml(File.read("config/config.yml"))
|
||||
|
|
@ -2174,15 +2175,195 @@ get "/subscription_manager" do |env|
|
|||
end
|
||||
subscriptions = user.subscriptions
|
||||
|
||||
action_takeout = env.params.query["action_takeout"]?.try &.to_i?
|
||||
action_takeout ||= 0
|
||||
action_takeout = action_takeout == 1
|
||||
|
||||
format = env.params.query["format"]?
|
||||
format ||= "rss"
|
||||
|
||||
client = make_client(YT_URL)
|
||||
subscriptions = subscriptions.map do |ucid|
|
||||
get_channel(ucid, client, PG_DB, false)
|
||||
end
|
||||
subscriptions.sort_by! { |channel| channel.author.downcase }
|
||||
|
||||
if action_takeout
|
||||
if Kemal.config.ssl || CONFIG.https_only
|
||||
scheme = "https://"
|
||||
else
|
||||
scheme = "http://"
|
||||
end
|
||||
host = env.request.headers["Host"]
|
||||
|
||||
url = "#{scheme}#{host}"
|
||||
|
||||
if format == "json"
|
||||
env.response.content_type = "application/json"
|
||||
env.response.headers["content-disposition"] = "attachment"
|
||||
next {
|
||||
"subscriptions" => user.subscriptions,
|
||||
"watch_history" => user.watched,
|
||||
"preferences" => user.preferences,
|
||||
}.to_json
|
||||
else
|
||||
env.response.content_type = "application/xml"
|
||||
env.response.headers["content-disposition"] = "attachment"
|
||||
export = XML.build do |xml|
|
||||
xml.element("opml", version: "1.1") do
|
||||
xml.element("body") do
|
||||
if format == "newpipe"
|
||||
title = "YouTube Subscriptions"
|
||||
else
|
||||
title = "Invidious Subscriptions"
|
||||
end
|
||||
|
||||
xml.element("outline", text: title, title: title) do
|
||||
subscriptions.each do |channel|
|
||||
if format == "newpipe"
|
||||
xmlUrl = "https://www.youtube.com/feeds/videos.xml?channel_id=#{channel.id}"
|
||||
else
|
||||
xmlUrl = "#{url}/feed/channel/#{channel.id}"
|
||||
end
|
||||
|
||||
xml.element("outline", text: channel.author, title: channel.author,
|
||||
"type": "rss", xmlUrl: xmlUrl)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
next export.gsub(%(<?xml version="1.0"?>\n), "")
|
||||
end
|
||||
end
|
||||
|
||||
templated "subscription_manager"
|
||||
end
|
||||
|
||||
get "/data_control" do |env|
|
||||
user = env.get? "user"
|
||||
referer = env.request.headers["referer"]?
|
||||
referer ||= "/"
|
||||
|
||||
if user
|
||||
user = user.as(User)
|
||||
|
||||
templated "data_control"
|
||||
else
|
||||
env.redirect referer
|
||||
end
|
||||
end
|
||||
|
||||
post "/data_control" do |env|
|
||||
user = env.get? "user"
|
||||
referer = env.request.headers["referer"]?
|
||||
referer ||= "/"
|
||||
|
||||
if user
|
||||
user = user.as(User)
|
||||
|
||||
HTTP::FormData.parse(env.request) do |part|
|
||||
body = part.body.gets_to_end
|
||||
if body.empty?
|
||||
next
|
||||
end
|
||||
|
||||
case part.name
|
||||
when "import_invidious"
|
||||
body = JSON.parse(body)
|
||||
body["subscriptions"].as_a.each do |ucid|
|
||||
ucid = ucid.as_s
|
||||
if !user.subscriptions.includes? ucid
|
||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", ucid, user.id)
|
||||
|
||||
begin
|
||||
client = make_client(YT_URL)
|
||||
get_channel(ucid, client, PG_DB, false, false)
|
||||
rescue ex
|
||||
next
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
body["watch_history"].as_a.each do |id|
|
||||
id = id.as_s
|
||||
if !user.watched.includes? id
|
||||
PG_DB.exec("UPDATE users SET watched = array_append(watched,$1) WHERE id = $2", id, user.id)
|
||||
end
|
||||
end
|
||||
|
||||
PG_DB.exec("UPDATE users SET preferences = $1 WHERE id = $2", body["preferences"].to_json, user.id)
|
||||
when "import_youtube"
|
||||
subscriptions = XML.parse(body)
|
||||
subscriptions.xpath_nodes(%q(//outline[@type="rss"])).each do |channel|
|
||||
ucid = channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
|
||||
|
||||
if !user.subscriptions.includes? ucid
|
||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", ucid, user.id)
|
||||
|
||||
begin
|
||||
client = make_client(YT_URL)
|
||||
get_channel(ucid, client, PG_DB, false, false)
|
||||
rescue ex
|
||||
next
|
||||
end
|
||||
end
|
||||
end
|
||||
when "import_newpipe_subscriptions"
|
||||
body = JSON.parse(body)
|
||||
body["subscriptions"].as_a.each do |channel|
|
||||
ucid = channel["url"].as_s.match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
|
||||
|
||||
if !user.subscriptions.includes? ucid
|
||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", ucid, user.id)
|
||||
|
||||
begin
|
||||
client = make_client(YT_URL)
|
||||
get_channel(ucid, client, PG_DB, false, false)
|
||||
rescue ex
|
||||
next
|
||||
end
|
||||
end
|
||||
end
|
||||
when "import_newpipe"
|
||||
Zip::Reader.open(body) do |file|
|
||||
file.each_entry do |entry|
|
||||
if entry.filename == "newpipe.db"
|
||||
# We do this because the SQLite driver cannot parse a database from an IO
|
||||
# Currently: channel URLs can **only** be subscriptions, and
|
||||
# video URLs can **only** be watch history, so this works okay for now.
|
||||
|
||||
db = entry.io.gets_to_end
|
||||
db.scan(/youtube\.com\/watch\?v\=(?<id>[a-zA-Z0-9_-]{11})/) do |md|
|
||||
if !user.watched.includes? md["id"]
|
||||
PG_DB.exec("UPDATE users SET watched = array_append(watched,$1) WHERE id = $2", md["id"], user.id)
|
||||
end
|
||||
end
|
||||
|
||||
db.scan(/youtube\.com\/channel\/(?<ucid>[a-zA-Z0-9_-]{22})/) do |md|
|
||||
ucid = md["ucid"]
|
||||
if !user.subscriptions.includes? ucid
|
||||
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", ucid, user.id)
|
||||
|
||||
begin
|
||||
client = make_client(YT_URL)
|
||||
get_channel(ucid, client, PG_DB, false, false)
|
||||
rescue ex
|
||||
next
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
env.redirect referer
|
||||
end
|
||||
|
||||
get "/subscription_ajax" do |env|
|
||||
user = env.get? "user"
|
||||
referer = env.request.headers["referer"]?
|
||||
|
|
|
|||
50
src/invidious/views/data_control.ecr
Normal file
50
src/invidious/views/data_control.ecr
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
<% content_for "header" do %>
|
||||
<title>Import and Export Data - Invidious</title>
|
||||
<% end %>
|
||||
|
||||
<div class="h-box">
|
||||
<form class="pure-form pure-form-aligned" enctype="multipart/form-data" action="/data_control" method="post">
|
||||
<fieldset>
|
||||
<legend>Import</legend>
|
||||
|
||||
<div class="pure-control-group">
|
||||
<label for="import_youtube">Import Invidious data</label>
|
||||
<input type="file" id="import_invidious" name="import_invidious">
|
||||
</div>
|
||||
|
||||
<div class="pure-control-group">
|
||||
<label for="import_youtube">Import <a target="_blank" style="color: #0366d6"
|
||||
href="https://support.google.com/youtube/answer/6224202?hl=en-GB">YouTube subscriptions</a></label>
|
||||
<input type="file" id="import_youtube" name="import_youtube">
|
||||
</div>
|
||||
|
||||
<div class="pure-control-group">
|
||||
<label for="import_newpipe_subscriptions">Import NewPipe subscriptions (.json)</label>
|
||||
<input type="file" id="import_newpipe_subscriptions" name="import_newpipe_subscriptions">
|
||||
</div>
|
||||
|
||||
<div class="pure-control-group">
|
||||
<label for="import_newpipe">Import NewPipe data (.zip)</label>
|
||||
<input type="file" id="import_newpipe" name="import_newpipe">
|
||||
</div>
|
||||
|
||||
<div class="pure-controls">
|
||||
<button type="submit" class="pure-button pure-button-primary">Import</button>
|
||||
</div>
|
||||
|
||||
<legend>Export</legend>
|
||||
|
||||
<div class="pure-control-group">
|
||||
<a href="/subscription_manager?action_takeout=1">Export subscriptions as OPML</a>
|
||||
</div>
|
||||
|
||||
<div class="pure-control-group">
|
||||
<a href="/subscription_manager?action_takeout=1&format=newpipe">Export subscriptions as OPML (NewPipe)</a>
|
||||
</div>
|
||||
|
||||
<div class="pure-control-group">
|
||||
<a href="/subscription_manager?action_takeout=1&format=json">Export data as JSON</a>
|
||||
</div>
|
||||
</fieldset>
|
||||
</form>
|
||||
</div>
|
||||
|
|
@ -101,7 +101,13 @@ function update_value(element) {
|
|||
<div class="pure-control-group">
|
||||
<label>
|
||||
<a href="/clear_watch_history">Clear watch history</a>
|
||||
</labe>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="pure-control-group">
|
||||
<label>
|
||||
<a href="/data_control">Import/Export data</a>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="pure-controls">
|
||||
|
|
|
|||
|
|
@ -2,7 +2,16 @@
|
|||
<title>Subscription manager - Invidious</title>
|
||||
<% end %>
|
||||
|
||||
<h1><%= subscriptions.size %> subscriptions</h1>
|
||||
<div class="pure-g h-box">
|
||||
<div class="pure-u-2-3">
|
||||
<h3><%= subscriptions.size %> subscriptions</h3>
|
||||
</div>
|
||||
<div class="pure-u-1-3" style="text-align:right;">
|
||||
<h3>
|
||||
<a href="/data_control">Import/Export</a>
|
||||
</h3>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<% subscriptions.each do |channel| %>
|
||||
<h3 class="h-box">
|
||||
|
|
|
|||
Loading…
Reference in a new issue