Skip to content

Commit 2bbd424

Browse files
SamantazFoxSamantazFox
committed
Move import logic to its own module
1 parent ef8dc72 commit 2bbd424

File tree

3 files changed

+203
-136
lines changed

3 files changed

+203
-136
lines changed

‎spec/invidious/user/imports_spec.cr‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ def csv_sample
2525
CSV
2626
end
2727

28-
Spectator.describe "Invidious::User::Imports" do
28+
Spectator.describe Invidious::User::Import do
2929
it "imports CSV" do
30-
subscriptions = parse_subscription_export_csv(csv_sample)
30+
subscriptions = Invidious::User::Import.parse_subscription_export_csv(csv_sample)
3131

3232
expect(subscriptions).to be_an(Array(String))
3333
expect(subscriptions.size).to eq(13)

‎src/invidious/routes/preferences.cr‎

Lines changed: 11 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -321,149 +321,27 @@ module Invidious::Routes::PreferencesRoute
321321
# TODO: Unify into single import based on content-type
322322
case part.name
323323
when "import_invidious"
324-
body = JSON.parse(body)
325-
326-
if body["subscriptions"]?
327-
user.subscriptions += body["subscriptions"].as_a.map(&.as_s)
328-
user.subscriptions.uniq!
329-
330-
user.subscriptions = get_batch_channels(user.subscriptions)
331-
332-
Invidious::Database::Users.update_subscriptions(user)
333-
end
334-
335-
if body["watch_history"]?
336-
user.watched += body["watch_history"].as_a.map(&.as_s)
337-
user.watched.uniq!
338-
Invidious::Database::Users.update_watch_history(user)
339-
end
340-
341-
if body["preferences"]?
342-
user.preferences = Preferences.from_json(body["preferences"].to_json)
343-
Invidious::Database::Users.update_preferences(user)
344-
end
345-
346-
if playlists = body["playlists"]?.try &.as_a?
347-
playlists.each do |item|
348-
title = item["title"]?.try &.as_s?.try &.delete("<>")
349-
description = item["description"]?.try &.as_s?.try &.delete("\r")
350-
privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
351-
352-
next if !title
353-
next if !description
354-
next if !privacy
355-
356-
playlist = create_playlist(title, privacy, user)
357-
Invidious::Database::Playlists.update_description(playlist.id, description)
358-
359-
videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
360-
raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500
361-
362-
video_id = video_id.try &.as_s?
363-
next if !video_id
364-
365-
begin
366-
video = get_video(video_id)
367-
rescue ex
368-
next
369-
end
370-
371-
playlist_video = PlaylistVideo.new({
372-
title: video.title,
373-
id: video.id,
374-
author: video.author,
375-
ucid: video.ucid,
376-
length_seconds: video.length_seconds,
377-
published: video.published,
378-
plid: playlist.id,
379-
live_now: video.live_now,
380-
index: Random::Secure.rand(0_i64..Int64::MAX),
381-
})
382-
383-
Invidious::Database::PlaylistVideos.insert(playlist_video)
384-
Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index)
385-
end
386-
end
387-
end
324+
Invidious::User::Import.from_invidious(user, body)
388325
when "import_youtube"
389326
filename = part.filename || ""
390-
extension = filename.split(".").last
391-
392-
if extension == "xml" || type == "application/xml" || type == "text/xml"
393-
subscriptions = XML.parse(body)
394-
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
395-
channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
396-
end
397-
elsif extension == "json" || type == "application/json"
398-
subscriptions = JSON.parse(body)
399-
user.subscriptions += subscriptions.as_a.compact_map do |entry|
400-
entry["snippet"]["resourceId"]["channelId"].as_s
401-
end
402-
elsif extension == "csv" || type == "text/csv"
403-
subscriptions = parse_subscription_export_csv(body)
404-
user.subscriptions += subscriptions
405-
else
327+
success = Invidious::User::Import.from_youtube(user, body, filename, type)
328+
329+
if !success
406330
haltf(env, status_code: 415,
407331
response: error_template(415, "Invalid subscription file uploaded")
408332
)
409333
end
410-
411-
user.subscriptions.uniq!
412-
user.subscriptions = get_batch_channels(user.subscriptions)
413-
414-
Invidious::Database::Users.update_subscriptions(user)
415334
when "import_freetube"
416-
user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md|
417-
md["channel_id"]
418-
end
419-
user.subscriptions.uniq!
420-
421-
user.subscriptions = get_batch_channels(user.subscriptions)
422-
423-
Invidious::Database::Users.update_subscriptions(user)
335+
Invidious::User::Import.from_freetube(user, body)
424336
when "import_newpipe_subscriptions"
425-
body = JSON.parse(body)
426-
user.subscriptions += body["subscriptions"].as_a.compact_map do |channel|
427-
if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/)
428-
next match["channel"]
429-
elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/)
430-
response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
431-
html = XML.parse_html(response.body)
432-
ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
433-
next ucid if ucid
434-
end
435-
436-
nil
437-
end
438-
user.subscriptions.uniq!
439-
440-
user.subscriptions = get_batch_channels(user.subscriptions)
441-
442-
Invidious::Database::Users.update_subscriptions(user)
337+
Invidious::User::Import.from_newpipe_subs(user, body)
443338
when "import_newpipe"
444-
Compress::Zip::Reader.open(IO::Memory.new(body)) do |file|
445-
file.each_entry do |entry|
446-
if entry.filename == "newpipe.db"
447-
tempfile = File.tempfile(".db")
448-
File.write(tempfile.path, entry.io.gets_to_end)
449-
db = DB.open("sqlite3://" + tempfile.path)
450-
451-
user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v="))
452-
user.watched.uniq!
339+
success = Invidious::User::Import.from_newpipe(user, body)
453340

454-
Invidious::Database::Users.update_watch_history(user)
455-
456-
user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/"))
457-
user.subscriptions.uniq!
458-
459-
user.subscriptions = get_batch_channels(user.subscriptions)
460-
461-
Invidious::Database::Users.update_subscriptions(user)
462-
463-
db.close
464-
tempfile.delete
465-
end
466-
end
341+
if !success
342+
haltf(env, status_code: 415,
343+
response: error_template(415, "Uploaded file is too large")
344+
)
467345
end
468346
else nil # Ignore
469347
end

‎src/invidious/user/imports.cr‎

Lines changed: 190 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,194 @@ struct Invidious::User
2929

3030
return subscriptions
3131
end
32-
end
32+
33+
# -------------------
34+
# Invidious
35+
# -------------------
36+
37+
# Import from another invidious account
38+
def from_invidious(user : User, body : String)
39+
data = JSON.parse(body)
40+
41+
if data["subscriptions"]?
42+
user.subscriptions += data["subscriptions"].as_a.map(&.as_s)
43+
user.subscriptions.uniq!
44+
user.subscriptions = get_batch_channels(user.subscriptions)
45+
46+
Invidious::Database::Users.update_subscriptions(user)
47+
end
48+
49+
if data["watch_history"]?
50+
user.watched += data["watch_history"].as_a.map(&.as_s)
51+
user.watched.uniq!
52+
Invidious::Database::Users.update_watch_history(user)
53+
end
54+
55+
if data["preferences"]?
56+
user.preferences = Preferences.from_json(data["preferences"].to_json)
57+
Invidious::Database::Users.update_preferences(user)
58+
end
59+
60+
if playlists = data["playlists"]?.try &.as_a?
61+
playlists.each do |item|
62+
title = item["title"]?.try &.as_s?.try &.delete("<>")
63+
description = item["description"]?.try &.as_s?.try &.delete("\r")
64+
privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
65+
66+
next if !title
67+
next if !description
68+
next if !privacy
69+
70+
playlist = create_playlist(title, privacy, user)
71+
Invidious::Database::Playlists.update_description(playlist.id, description)
72+
73+
videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
74+
raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500
75+
76+
video_id = video_id.try &.as_s?
77+
next if !video_id
78+
79+
begin
80+
video = get_video(video_id)
81+
rescue ex
82+
next
83+
end
84+
85+
playlist_video = PlaylistVideo.new({
86+
title: video.title,
87+
id: video.id,
88+
author: video.author,
89+
ucid: video.ucid,
90+
length_seconds: video.length_seconds,
91+
published: video.published,
92+
plid: playlist.id,
93+
live_now: video.live_now,
94+
index: Random::Secure.rand(0_i64..Int64::MAX),
95+
})
96+
97+
Invidious::Database::PlaylistVideos.insert(playlist_video)
98+
Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index)
99+
end
100+
end
101+
end
102+
end
103+
104+
# -------------------
105+
# Youtube
106+
# -------------------
107+
108+
# Import subscribed channels from Youtube
109+
# Returns success status
110+
def from_youtube(user : User, body : String, filename : String, type : String) : Bool
111+
extension = filename.split(".").last
112+
113+
if extension == "xml" || type == "application/xml" || type == "text/xml"
114+
subscriptions = XML.parse(body)
115+
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
116+
channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
117+
end
118+
elsif extension == "json" || type == "application/json"
119+
subscriptions = JSON.parse(body)
120+
user.subscriptions += subscriptions.as_a.compact_map do |entry|
121+
entry["snippet"]["resourceId"]["channelId"].as_s
122+
end
123+
elsif extension == "csv" || type == "text/csv"
124+
subscriptions = parse_subscription_export_csv(body)
125+
user.subscriptions += subscriptions
126+
else
127+
return false
128+
end
129+
130+
user.subscriptions.uniq!
131+
user.subscriptions = get_batch_channels(user.subscriptions)
132+
133+
Invidious::Database::Users.update_subscriptions(user)
134+
return true
135+
end
136+
137+
# -------------------
138+
# Freetube
139+
# -------------------
140+
141+
def from_freetube(user : User, body : String)
142+
matches = body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/)
143+
144+
user.subscriptions += matches.map(&.["channel_id"])
145+
user.subscriptions.uniq!
146+
user.subscriptions = get_batch_channels(user.subscriptions)
147+
148+
Invidious::Database::Users.update_subscriptions(user)
149+
end
150+
151+
# -------------------
152+
# Newpipe
153+
# -------------------
154+
155+
def from_newpipe_subs(user : User, body : String)
156+
data = JSON.parse(body)
157+
158+
user.subscriptions += data["subscriptions"].as_a.compact_map do |channel|
159+
if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/)
160+
next match["channel"]
161+
elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/)
162+
# Resolve URL using the API
163+
resolved_url = YoutubeAPI.resolve_url("https://www.youtube.com/user/#{match["user"]}")
164+
ucid = resolved_url.dig?("endpoint", "browseEndpoint", "browseId")
165+
next ucid.as_s if ucid
166+
end
167+
168+
nil
169+
end
170+
171+
user.subscriptions.uniq!
172+
user.subscriptions = get_batch_channels(user.subscriptions)
173+
174+
Invidious::Database::Users.update_subscriptions(user)
175+
end
176+
177+
def from_newpipe(user : User, body : String) : Bool
178+
io = IO::Memory.new(body)
179+
180+
Compress::Zip::File.open(io) do |file|
181+
file.entries.each do |entry|
182+
entry.open do |file_io|
183+
# Ensure max size of 4MB
184+
io_sized = IO::Sized.new(file_io, 0x400000)
185+
186+
next if entry.filename != "newpipe.db"
187+
188+
tempfile = File.tempfile(".db")
189+
190+
begin
191+
File.write(tempfile.path, io_sized.gets_to_end)
192+
rescue
193+
return false
194+
end
195+
196+
db = DB.open("sqlite3://" + tempfile.path)
197+
198+
user.watched += db.query_all("SELECT url FROM streams", as: String)
199+
.map(&.lchop("https://www.youtube.com/watch?v="))
200+
201+
user.watched.uniq!
202+
Invidious::Database::Users.update_watch_history(user)
203+
204+
user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
205+
.map(&.lchop("https://www.youtube.com/channel/"))
206+
207+
user.subscriptions.uniq!
208+
user.subscriptions = get_batch_channels(user.subscriptions)
209+
210+
Invidious::Database::Users.update_subscriptions(user)
211+
212+
db.close
213+
tempfile.delete
214+
end
215+
end
216+
end
217+
218+
# Success!
219+
return true
220+
end
221+
end # module
33222
end

0 commit comments

Comments
 (0)