Skip to content

Commit 15d156e

Browse files
yuudiyuudi
committed
fix: use chunk downloading
1 parent 89d51c8 commit 15d156e

File tree

2 files changed

+69
-259
lines changed

2 files changed

+69
-259
lines changed

‎assets/receive.js‎

Lines changed: 14 additions & 171 deletions
Original file line numberDiff line numberDiff line change
@@ -91,116 +91,6 @@ async function decrypt_file_name(key, name_encrypted, nonce, file_id) {
9191
return dec.decode(plain_filename_array);
9292
}
9393

94-
// async function decrypt_file_part(key, cipher, nonce, file_id, counter) {
95-
// let counter_array = new Uint8Array(new Uint32Array([counter]).buffer);
96-
// let file_id_array = new Uint8Array(
97-
// new Uint32Array([file_id * 2 + 1]).buffer
98-
// );
99-
// let CTR = new Uint8Array([
100-
// ...nonce,
101-
// ...file_id_array.reverse(),
102-
// ...counter_array.reverse(),
103-
// ]);
104-
// let plain = await crypto.subtle.decrypt(
105-
// {
106-
// name: "AES-CTR",
107-
// counter: CTR,
108-
// length: 128,
109-
// },
110-
// key,
111-
// cipher
112-
// );
113-
// return plain;
114-
// }
115-
116-
// // splits a ReadableStream into chunks of a given size
117-
// // code from https://gist.github.com/thomaskonrad/b8f30e3f18ea2f538bdf422203bdc473
118-
// class StreamSlicer {
119-
// constructor(chunkSize, processor, progress) {
120-
// this.chunkSize = chunkSize;
121-
// this.partialChunk = new Uint8Array(this.chunkSize);
122-
// this.offset = 0;
123-
// this.counter = 0;
124-
// this.processor = processor;
125-
// this.progress = progress;
126-
// }
127-
// async send(buf, writer) {
128-
// let data = await this.processor(buf, this.counter);
129-
// await writer.write(data);
130-
// this.counter += this.chunkSize;
131-
// this.progress(this.counter);
132-
// this.partialChunk = new Uint8Array(this.chunkSize);
133-
// this.offset = 0;
134-
// this.offset += this.chunkSize;
135-
// }
136-
// async transform(chunk, writer) {
137-
// let i = 0;
138-
// if (this.offset > 0) {
139-
// const len = Math.min(
140-
// chunk.byteLength,
141-
// this.chunkSize - this.offset
142-
// );
143-
// this.partialChunk.set(chunk.slice(0, len), this.offset);
144-
// this.offset += len;
145-
// i += len;
146-
// if (this.offset === this.chunkSize) {
147-
// await this.send(this.partialChunk, writer);
148-
// }
149-
// }
150-
// while (i < chunk.byteLength) {
151-
// const remainingBytes = chunk.byteLength - i;
152-
// if (remainingBytes >= this.chunkSize) {
153-
// const record = chunk.slice(i, i + this.chunkSize);
154-
// i += this.chunkSize;
155-
// await this.send(record, writer);
156-
// } else {
157-
// const end = chunk.slice(i, i + remainingBytes);
158-
// i += end.byteLength;
159-
// this.partialChunk.set(end);
160-
// this.offset = end.byteLength;
161-
// }
162-
// }
163-
// }
164-
// async flush(writer) {
165-
// // only for last chunk
166-
// if (this.offset > 0) {
167-
// let data = await this.processor(
168-
// this.partialChunk.slice(0, this.offset),
169-
// this.counter
170-
// );
171-
// await writer.write(data);
172-
// }
173-
// }
174-
// }
175-
176-
// async function download_file(file_info, writer, progress_callback) {
177-
// let response = await fetch(file_info.download_url);
178-
// let cipher_readable_stream = response.body.getReader();
179-
// let stream_slicer = new StreamSlicer(
180-
// 327680 /* 320KiB */,
181-
// async function (buf, counter) {
182-
// let data = await decrypt_file_part(
183-
// file_info.key,
184-
// buf,
185-
// file_info.nonce,
186-
// file_info.file_id,
187-
// counter / 16
188-
// );
189-
// return data;
190-
// // await writer.write(data);
191-
// },
192-
// progress_callback
193-
// );
194-
// while (true) {
195-
// let { done, value } = await cipher_readable_stream.read();
196-
// if (done) {
197-
// await stream_slicer.flush(writer);
198-
// return;
199-
// }
200-
// await stream_slicer.transform(value, writer);
201-
// }
202-
// }
203-
20494
(async function () {
20595
let file_list = document.getElementById("file-list");
20696
let notice_area = document.getElementById("notice");
@@ -270,54 +160,26 @@ async function decrypt_file_name(key, name_encrypted, nonce, file_id) {
270160
);
271161
let download_url = file_info["@microsoft.graph.downloadUrl"];
272162
await navigator.serviceWorker.controller.postMessage({
273-
file_path: file_info.name,
274-
download_url: download_url,
275-
key: key,
276-
nonce: nonce,
277-
filename: filename,
278-
file_size: file_info.size,
279-
file_id: file_id,
163+
request: "add_file",
164+
file_info: {
165+
file_path: file_info.name,
166+
download_url: download_url,
167+
key: key,
168+
nonce: nonce,
169+
filename: filename,
170+
file_size: file_info.size,
171+
file_id: file_id,
172+
},
280173
});
174+
setInterval(function () {
175+
// keep service work alive
176+
navigator.serviceWorker.controller.postMessage({ request: "ping" });
177+
}, 100);
281178
a.innerText = filename;
282179
// a.download = filename; // chrome will request to backend if use download attribute
283180
a.classList.add("link-like");
284181
let readable_size = humanFileSize(file_info.size, true, 2);
285182
let size_node = document.createTextNode(` (${readable_size}) `);
286-
// a.addEventListener("click", async function (e) {
287-
// e.preventDefault();
288-
// if (!("showSaveFilePicker" in window)) {
289-
// alert(
290-
// "your browser dose not support stream saver, please switch to Chrome/Edge or use CLI downloader"
291-
// );
292-
// return;
293-
// }
294-
// current_downloading += 1;
295-
// let file_handle = await window.showSaveFilePicker({
296-
// suggestedName: filename,
297-
// });
298-
// let file_writer = await file_handle.createWritable();
299-
// await download_file(
300-
// {
301-
// key: key,
302-
// nonce: nonce,
303-
// file_id: file_id,
304-
// file_size: file_info.size,
305-
// download_url: download_url,
306-
// },
307-
// file_writer,
308-
// function (downloaded) {
309-
// let readable_downloaded = humanFileSize(
310-
// downloaded,
311-
// true,
312-
// 2
313-
// );
314-
// size_node.nodeValue = ` (${readable_downloaded} / ${readable_size}) `;
315-
// }
316-
// );
317-
// file_writer.close();
318-
// size_node.nodeValue = ` (downloaded / ${readable_size}) `;
319-
// current_downloading -= 1;
320-
// });
321183
info.append(a);
322184
info.append(size_node);
323185
let nonce_offset_hex = (file_id * 2 + 1).toString(16).padStart(8, "0");
@@ -335,25 +197,6 @@ async function decrypt_file_name(key, name_encrypted, nonce, file_id) {
335197
notice_area.innerText = "";
336198
}, 2000);
337199
});
338-
// a.href = "/s/download/" + file_info.name;
339-
// a.download = filename;
340-
// await navigator.serviceWorker.controller.postMessage({
341-
// file_path: file_info.name,
342-
// download_url: download_url,
343-
// key: key,
344-
// nonce: nonce,
345-
// filename: filename,
346-
// file_size: file_info.size,
347-
// file_id: file_id,
348-
// });
349-
// let readable_size = humanFileSize(file_info.size, true, 2);
350-
// info.append(a);
351-
// info.append(document.createTextNode(` (${readable_size})`));
352-
// let cli_downloader = document.createElement("div");
353-
// let nonce_offset_hex = (Number(file_id) * 2 + 1)
354-
// .toString(16)
355-
// .padStart(8, "0");
356-
// cli_downloader.innerText = `wget "${download_url}" -O- | openssl enc -d -aes-256-ctr -K "${key_hex}" -iv "${nonce_hex}${nonce_offset_hex}00000000" -out "${filename}"`;
357200
info.append(cli_downloader);
358201
info.classList.add("file-item");
359202
file_list.append(info);

‎virtual-downloader.js‎

Lines changed: 55 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,3 @@
1-
// splits a ReadableStream into chunks of a given size
2-
// code from https://gist.github.com/thomaskonrad/b8f30e3f18ea2f538bdf422203bdc473
3-
class StreamSlicer {
4-
constructor(chunkSize, processor) {
5-
this.chunkSize = chunkSize;
6-
this.partialChunk = new Uint8Array(this.chunkSize);
7-
this.offset = 0;
8-
this.counter = 0;
9-
this.processor = processor;
10-
}
11-
async send(buf, controller) {
12-
let data = await this.processor(buf, this.counter);
13-
controller.enqueue(data);
14-
this.counter += this.chunkSize;
15-
this.partialChunk = new Uint8Array(this.chunkSize);
16-
this.offset = 0;
17-
this.offset += this.chunkSize;
18-
}
19-
async transform(chunk, controller) {
20-
let i = 0;
21-
if (this.offset > 0) {
22-
const len = Math.min(
23-
chunk.byteLength,
24-
this.chunkSize - this.offset
25-
);
26-
this.partialChunk.set(chunk.slice(0, len), this.offset);
27-
this.offset += len;
28-
i += len;
29-
if (this.offset === this.chunkSize) {
30-
await this.send(this.partialChunk, controller);
31-
}
32-
}
33-
while (i < chunk.byteLength) {
34-
const remainingBytes = chunk.byteLength - i;
35-
if (remainingBytes >= this.chunkSize) {
36-
const record = chunk.slice(i, i + this.chunkSize);
37-
i += this.chunkSize;
38-
await this.send(record, controller);
39-
} else {
40-
const end = chunk.slice(i, i + remainingBytes);
41-
i += end.byteLength;
42-
this.partialChunk.set(end);
43-
this.offset = end.byteLength;
44-
}
45-
}
46-
}
47-
async flush(controller) {
48-
// only for last chunk
49-
if (this.offset > 0) {
50-
let data = await this.processor(
51-
this.partialChunk.slice(0, this.offset),
52-
this.counter
53-
);
54-
controller.enqueue(data);
55-
}
56-
}
57-
}
58-
591
let FilesData = {};
602

613
async function decrypt_file_part(key, cipher, nonce, file_id, counter) {
@@ -89,45 +31,70 @@ self.addEventListener("activate", function (event) {
8931
});
9032

9133
self.addEventListener("message", function (event) {
92-
FilesData[event.data.file_path] = event.data;
34+
if (event.data.request === "add_file") {
35+
FilesData[event.data.file_info.file_path] = event.data.file_info;
36+
}
9337
});
9438

39+
async function try_fetch(input, init, tries = 3) {
40+
try {
41+
return await fetch(input, init);
42+
} catch (e) {
43+
if (tries > 0) {
44+
return try_fetch(input, init, tries - 1);
45+
}
46+
throw e;
47+
}
48+
}
49+
9550
function decrypt_file_stream(file_info) {
96-
let stream_slicer = new StreamSlicer(327680 /* 320KiB */, async function (
97-
buf,
98-
counter
99-
) {
100-
let data = await decrypt_file_part(
101-
file_info.key,
102-
buf,
103-
file_info.nonce,
104-
file_info.file_id,
105-
counter / 16
106-
);
107-
return new Uint8Array(data);
108-
});
10951
let decrypted_readable_stream = new ReadableStream({
11052
async start(controller) {
111-
let response = await fetch(file_info.download_url);
112-
this.cipher_readable_stream = response.body.getReader();
113-
while (true) {
114-
let { done, value } = await this.cipher_readable_stream.read();
115-
if (done) {
116-
await stream_slicer.flush(controller);
117-
controller.close();
118-
return;
53+
const chunk_size = 1310720;
54+
let chunk_number = Math.ceil(file_info.file_size / chunk_size);
55+
let fetched = 0;
56+
let fetch_queue = [];
57+
async function next_fetch() {
58+
if (fetched >= chunk_number) {
59+
return null;
11960
}
120-
await stream_slicer.transform(value, controller);
61+
let i = fetched;
62+
fetched += 1;
63+
let start = i * chunk_size;
64+
let end;
65+
if (i === chunk_number - 1) {
66+
end = file_info.file_size - 1;
67+
} else {
68+
end = start + chunk_size - 1;
69+
}
70+
let response = await try_fetch(file_info.download_url, {
71+
headers: { Range: `bytes=${start}-${end}` },
72+
});
73+
let data = await response.arrayBuffer();
74+
let plain = await decrypt_file_part(
75+
file_info.key,
76+
data,
77+
file_info.nonce,
78+
file_info.file_id,
79+
start / 16
80+
);
81+
return new Uint8Array(plain);
82+
}
83+
fetch_queue.push(next_fetch());
84+
setTimeout(function () {
85+
// 4 concurrent download
86+
fetch_queue.push(next_fetch());
87+
fetch_queue.push(next_fetch());
88+
fetch_queue.push(next_fetch());
89+
}, 1000);
90+
for (let j = 0; j < chunk_number; j++) {
91+
let chunk = await fetch_queue.shift();
92+
controller.enqueue(chunk);
93+
fetch_queue.push(next_fetch());
12194
}
122-
// controller.enqueue(
123-
// new Uint8Array([49, 50, 49, 50, 49, 50, 49, 50, 49, 50])
124-
// );
125-
// controller.close();
126-
// return;
95+
controller.close();
12796
},
128-
// async pull(controller) {},
12997
});
130-
// TransformStream();
13198
return new Response(decrypted_readable_stream, {
13299
headers: {
133100
"Content-Length": file_info.file_size,

0 commit comments

Comments
 (0)