Skip to content

Commit da6983e

Browse files
sdjddsdjdd
authored
feat(qiniu): shard upload (#650)
1 parent 0ef2180 commit da6983e

File tree

2 files changed

+297
-1
lines changed

2 files changed

+297
-1
lines changed

‎src/uploader/qiniu.js‎

Lines changed: 268 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,13 @@
11
const { getAdapter } = require('../adapter');
22
const debug = require('debug')('leancloud:qiniu');
3+
const ajax = require('../utils/ajax');
4+
const btoa = require('../utils/btoa');
35

4-
module.exports = function(uploadInfo, data, file, saveOptions = {}) {
6+
const SHARD_THRESHOLD = 1024 * 1024 * 64;
7+
8+
const CHUNK_SIZE = 1024 * 1024 * 16;
9+
10+
function upload(uploadInfo, data, file, saveOptions = {}) {
511
// Get the uptoken to upload files to qiniu.
612
const uptoken = uploadInfo.token;
713
const url = uploadInfo.upload_url || 'https://upload.qiniup.com';
@@ -52,4 +58,265 @@ module.exports = function(uploadInfo, data, file, saveOptions = {}) {
5258
throw error;
5359
}
5460
);
61+
}
62+
63+
function urlSafeBase64(string) {
64+
const base64 = btoa(unescape(encodeURIComponent(string)));
65+
let result = '';
66+
for (const ch of base64) {
67+
switch (ch) {
68+
case '+':
69+
result += '-';
70+
break;
71+
case '/':
72+
result += '_';
73+
break;
74+
default:
75+
result += ch;
76+
}
77+
}
78+
return result;
79+
}
80+
81+
class ShardUploader {
82+
constructor(uploadInfo, data, file, saveOptions) {
83+
this.data = data;
84+
this.file = file;
85+
this.size = undefined;
86+
this.offset = 0;
87+
this.uploadedChunks = 0;
88+
89+
const key = urlSafeBase64(uploadInfo.key);
90+
this.baseURL = `https://upload.qiniup.com/buckets/${uploadInfo.bucket}/objects/${key}/uploads`;
91+
this.upToken = 'UpToken ' + uploadInfo.token;
92+
93+
this.uploaded = 0;
94+
if (saveOptions && saveOptions.onprogress) {
95+
this.onProgress = ({ loaded }) => {
96+
loaded += this.uploadedChunks * CHUNK_SIZE;
97+
if (loaded <= this.uploaded) {
98+
return;
99+
}
100+
if (this.size) {
101+
saveOptions.onprogress({
102+
loaded,
103+
total: this.size,
104+
percent: (loaded / this.size) * 100,
105+
});
106+
} else {
107+
saveOptions.onprogress({ loaded });
108+
}
109+
this.uploaded = loaded;
110+
};
111+
}
112+
}
113+
114+
/**
115+
* @returns {Promise<string>}
116+
*/
117+
getUploadId() {
118+
return ajax({
119+
method: 'POST',
120+
url: this.baseURL,
121+
headers: {
122+
Authorization: this.upToken,
123+
},
124+
}).then(res => res.uploadId);
125+
}
126+
127+
getChunk() {
128+
throw new Error('Not implemented');
129+
}
130+
131+
/**
132+
* @param {string} uploadId
133+
* @param {number} partNumber
134+
* @param {any} data
135+
* @returns {Promise<{ partNumber: number; etag: string; }>}
136+
*/
137+
uploadPart(uploadId, partNumber, data) {
138+
return ajax({
139+
method: 'PUT',
140+
url: `${this.baseURL}/${uploadId}/${partNumber}`,
141+
headers: {
142+
Authorization: this.upToken,
143+
},
144+
data,
145+
onprogress: this.onProgress,
146+
}).then(({ etag }) => ({ partNumber, etag }));
147+
}
148+
149+
stopUpload(uploadId) {
150+
return ajax({
151+
method: 'DELETE',
152+
url: `${this.baseURL}/${uploadId}`,
153+
headers: {
154+
Authorization: this.upToken,
155+
},
156+
});
157+
}
158+
159+
upload() {
160+
const parts = [];
161+
return this.getUploadId().then(uploadId => {
162+
const uploadPart = () => {
163+
return Promise.resolve(this.getChunk())
164+
.then(chunk => {
165+
if (!chunk) {
166+
return;
167+
}
168+
const partNumber = parts.length + 1;
169+
return this.uploadPart(uploadId, partNumber, chunk).then(part => {
170+
parts.push(part);
171+
this.uploadedChunks++;
172+
return uploadPart();
173+
});
174+
})
175+
.catch(error =>
176+
this.stopUpload(uploadId).then(() => Promise.reject(error))
177+
);
178+
};
179+
180+
return uploadPart().then(() =>
181+
ajax({
182+
method: 'POST',
183+
url: `${this.baseURL}/${uploadId}`,
184+
headers: {
185+
Authorization: this.upToken,
186+
},
187+
data: {
188+
parts,
189+
fname: this.file.attributes.name,
190+
mimeType: this.file.attributes.mime_type,
191+
},
192+
})
193+
);
194+
});
195+
}
196+
}
197+
198+
class BlobUploader extends ShardUploader {
199+
constructor(uploadInfo, data, file, saveOptions) {
200+
super(uploadInfo, data, file, saveOptions);
201+
this.size = data.size;
202+
}
203+
204+
/**
205+
* @returns {Blob | null}
206+
*/
207+
getChunk() {
208+
if (this.offset >= this.size) {
209+
return null;
210+
}
211+
const chunk = this.data.slice(this.offset, this.offset + CHUNK_SIZE);
212+
this.offset += chunk.size;
213+
return chunk;
214+
}
215+
}
216+
217+
/* NODE-ONLY:start */
218+
class BufferUploader extends ShardUploader {
219+
constructor(uploadInfo, data, file, saveOptions) {
220+
super(uploadInfo, data, file, saveOptions);
221+
this.size = data.length;
222+
}
223+
224+
/**
225+
* @returns {Buffer | null}
226+
*/
227+
getChunk() {
228+
if (this.offset >= this.size) {
229+
return null;
230+
}
231+
const chunk = this.data.slice(this.offset, this.offset + CHUNK_SIZE);
232+
this.offset += chunk.length;
233+
return chunk;
234+
}
235+
}
236+
/* NODE-ONLY:end */
237+
238+
/* NODE-ONLY:start */
239+
class StreamUploader extends ShardUploader {
240+
/**
241+
* @param {number} [size]
242+
* @returns {Buffer | null}
243+
*/
244+
_read(size) {
245+
const chunk = this.data.read(size);
246+
if (chunk) {
247+
this.offset += chunk.length;
248+
}
249+
return chunk;
250+
}
251+
252+
/**
253+
* @returns {Buffer | null | Promise<Buffer | null>}
254+
*/
255+
getChunk() {
256+
if (this.data.readableLength >= CHUNK_SIZE) {
257+
return this._read(CHUNK_SIZE);
258+
}
259+
260+
if (this.data.readableEnded) {
261+
if (this.data.readable) {
262+
return this._read();
263+
}
264+
return null;
265+
}
266+
267+
return new Promise((resolve, reject) => {
268+
const onReadable = () => {
269+
const chunk = this._read(CHUNK_SIZE);
270+
if (chunk !== null) {
271+
resolve(chunk);
272+
removeListeners();
273+
}
274+
};
275+
276+
const onError = error => {
277+
reject(error);
278+
removeListeners();
279+
};
280+
281+
const removeListeners = () => {
282+
this.data.off('readable', onReadable);
283+
this.data.off('error', onError);
284+
};
285+
286+
this.data.on('readable', onReadable);
287+
this.data.on('error', onError);
288+
});
289+
}
290+
}
291+
/* NODE-ONLY:end */
292+
293+
function isBlob(data) {
294+
return typeof Blob !== 'undefined' && data instanceof Blob;
295+
}
296+
297+
/* NODE-ONLY:start */
298+
function isBuffer(data) {
299+
return typeof Buffer !== 'undefined' && Buffer.isBuffer(data);
300+
}
301+
/* NODE-ONLY:end */
302+
303+
/* NODE-ONLY:start */
304+
function isStream(data) {
305+
return typeof require === 'function' && data instanceof require('stream');
306+
}
307+
/* NODE-ONLY:end */
308+
309+
module.exports = function(uploadInfo, data, file, saveOptions = {}) {
310+
if (isBlob(data) && data.size >= SHARD_THRESHOLD) {
311+
return new BlobUploader(uploadInfo, data, file, saveOptions).upload();
312+
}
313+
/* NODE-ONLY:start */
314+
if (isBuffer(data) && data.length >= SHARD_THRESHOLD) {
315+
return new BufferUploader(uploadInfo, data, file, saveOptions).upload();
316+
}
317+
if (isStream(data)) {
318+
return new StreamUploader(uploadInfo, data, file, saveOptions).upload();
319+
}
320+
/* NODE-ONLY:end */
321+
return upload(uploadInfo, data, file, saveOptions);
55322
};

‎src/utils/btoa.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// base64 character set, plus padding character (=)
2+
const b64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';
3+
4+
module.exports = string => {
5+
let result = '';
6+
7+
for (let i = 0; i < string.length; ) {
8+
const a = string.charCodeAt(i++);
9+
const b = string.charCodeAt(i++);
10+
const c = string.charCodeAt(i++);
11+
if (a > 255 || b > 255 || c > 255) {
12+
throw new TypeError(
13+
'Failed to encode base64: The string to be encoded contains characters outside of the Latin1 range.'
14+
);
15+
}
16+
17+
const bitmap = (a << 16) | (b << 8) | c;
18+
result +=
19+
b64.charAt((bitmap >> 18) & 63) +
20+
b64.charAt((bitmap >> 12) & 63) +
21+
b64.charAt((bitmap >> 6) & 63) +
22+
b64.charAt(bitmap & 63);
23+
}
24+
25+
// To determine the final padding
26+
const rest = string.length % 3;
27+
// If there's need of padding, replace the last 'A's with equal signs
28+
return rest ? result.slice(0, rest - 3) + '==='.substring(rest) : result;
29+
};

0 commit comments

Comments
 (0)