Skip to content

Commit 851df13

Browse files
TorstenDittmannTorstenDittmann
authored
Merge pull request #459 from appwrite/feat-improve-file-upload-node
Feat: Improve node file upload
2 parents c3f7957 + bea3ff6 commit 851df13

File tree

7 files changed

+182
-66
lines changed

7 files changed

+182
-66
lines changed

‎composer.lock

Lines changed: 7 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/SDK/Language/Node.php

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public function getTypeName($type)
2626
case self::TYPE_ARRAY:
2727
return 'string[]';
2828
case self::TYPE_FILE:
29-
return 'string';
29+
return 'InputFile';
3030
break;
3131
}
3232

@@ -63,6 +63,12 @@ public function getFiles()
6363
'template' => 'node/lib/query.js.twig',
6464
'minify' => false,
6565
],
66+
[
67+
'scope' => 'default',
68+
'destination' => 'lib/inputFile.js',
69+
'template' => 'node/lib/inputFile.js.twig',
70+
'minify' => false,
71+
],
6672
[
6773
'scope' => 'default',
6874
'destination' => '/lib/service.js',

‎templates/node/index.js.twig

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const Client = require('./lib/client.js');
22
const Query = require('./lib/query.js');
3+
const InputFile = require('./lib/inputFile.js');
34
const {{spec.title | caseUcfirst}}Exception = require('./lib/exception.js');
45
{% for service in spec.services %}
56
const {{service.name | caseUcfirst}} = require('./lib/services/{{service.name | caseDash}}.js');
@@ -8,6 +9,7 @@ const {{service.name | caseUcfirst}} = require('./lib/services/{{service.name |
89
module.exports = {
910
Client,
1011
Query,
12+
InputFile,
1113
{{spec.title | caseUcfirst}}Exception,
1214
{% for service in spec.services %}
1315
{{service.name | caseUcfirst}},

‎templates/node/lib/client.js.twig

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,13 @@ class Client {
9393
let flatParams = Client.flatten(params);
9494

9595
for (const key in flatParams) {
96-
form.append(key, flatParams[key])
96+
const value = flatParams[key];
97+
98+
if(value && value.type && value.type === 'file') {
99+
form.append(key, value.file, { filename: value.filename });
100+
} else {
101+
form.append(key, flatParams[key]);
102+
}
97103
}
98104

99105
headers = {

‎templates/node/lib/inputFile.js.twig

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
const { Readable } = require('stream');
2+
const fs = require('fs');
3+
const { promisify } = require('util');
4+
5+
class InputFile {
6+
stream; // Content of file, readable stream
7+
size; // Total final size of the file content
8+
name; // File name
9+
10+
static fromPath = (filePath, name) => {
11+
const stream = fs.createReadStream(filePath);
12+
const { size } = fs.statSync(filePath);
13+
return new InputFile(stream, name, size);
14+
};
15+
16+
static fromBuffer = (buffer, name) => {
17+
const stream = Readable.from(buffer.toString());
18+
const size = Buffer.byteLength(buffer);
19+
return new InputFile(stream, name, size);
20+
};
21+
22+
static fromBlob = (blob, name) => {
23+
const buffer = blob.arrayBuffer();
24+
const stream = Readable.from(buffer.toString());
25+
const size = Buffer.byteLength(buffer);
26+
return new InputFile(stream, name);
27+
};
28+
29+
static fromStream = (stream, name, size) => {
30+
return new InputFile(stream, name, size);
31+
};
32+
33+
static fromPlainText = (content, name) => {
34+
const buffer = Buffer.from(content, "utf-8");
35+
const stream = Readable.from(buffer.toString());
36+
const size = Buffer.byteLength(buffer);
37+
return new InputFile(stream, name, size);
38+
};
39+
40+
constructor(stream, name, size) {
41+
this.stream = stream;
42+
this.name = name;
43+
this.size = size;
44+
}
45+
}
46+
47+
module.exports = InputFile;

‎templates/node/lib/services/service.js.twig

Lines changed: 106 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
const Service = require('../service.js');
22
const {{spec.title | caseUcfirst}}Exception = require('../exception.js');
3+
const InputFile = require('../inputFile.js');
34
const client = require('../client.js');
5+
const Stream = require('stream');
46
const { promisify } = require('util');
57
const fs = require('fs');
68

@@ -34,95 +36,146 @@ class {{ service.name | caseUcfirst }} extends Service {
3436
{% for parameter in method.parameters.query %}
3537

3638
if (typeof {{ parameter.name | caseCamel | escapeKeyword }} !== 'undefined') {
37-
payload['{{ parameter.name }}'] = {{ parameter.name | caseCamel | escapeKeyword }}{% if method.consumes[0] == "multipart/form-data" and ( parameter.type != "string" and parameter.type != "array" ) %}.toString(){% endif %};
39+
payload['{{ parameter.name }}'] = {{ parameter.name | caseCamel | escapeKeyword }}{% if method.consumes[0] == "multipart/form-data" and ( parameter.type != "string" and parameter.type != "array" and parameter.type != "file" ) %}.toString(){% endif %};
3840
}
3941
{% endfor %}
4042
{% for parameter in method.parameters.body %}
4143

4244
if (typeof {{ parameter.name | caseCamel | escapeKeyword }} !== 'undefined') {
43-
payload['{{ parameter.name }}'] = {{ parameter.name | caseCamel | escapeKeyword}}{% if method.consumes[0] == "multipart/form-data" and ( parameter.type != "string" and parameter.type != "array" ) %}.toString(){% endif %};
45+
payload['{{ parameter.name }}'] = {{ parameter.name | caseCamel | escapeKeyword}}{% if method.consumes[0] == "multipart/form-data" and ( parameter.type != "string" and parameter.type != "array" and parameter.type != "file" ) %}.toString(){% endif %};
4446
}
4547
{% endfor %}
4648

4749
{% if 'multipart/form-data' in method.consumes %}
4850
{% for parameter in method.parameters.all %}
4951
{% if parameter.type == 'file' %}
50-
const { size: size } = await promisify(fs.stat)({{ parameter.name | caseCamel | escapeKeyword }});
52+
const size = {{ parameter.name | caseCamel | escapeKeyword }}.size;
5153

52-
if (size <= client.CHUNK_SIZE) {
53-
payload['{{ parameter.name }}'] = fs.createReadStream({{ parameter.name | caseCamel | escapeKeyword }});
54-
55-
return await this.client.call('{{ method.method | caseLower }}', path, {
54+
const headers = {
5655
{% for parameter in method.parameters.header %}
57-
'{{ parameter.name }}': ${{ parameter.name | caseCamel | escapeKeyword }},
56+
'{{ parameter.name }}': ${{ parameter.name | caseCamel | escapeKeyword }},
5857
{% endfor %}
5958
{% for key, header in method.headers %}
60-
'{{ key }}': '{{ header }}',
59+
'{{ key }}': '{{ header }}',
6160
{% endfor %}
62-
}, payload{% if method.type == 'location' %}, 'arraybuffer'{% endif %});
63-
} else {
64-
let id = undefined;
65-
let response = undefined;
61+
};
6662

67-
let counter = 0;
68-
const totalCounters = Math.ceil(size / client.CHUNK_SIZE);
63+
let id = undefined;
64+
let response = undefined;
6965

70-
const headers = {
71-
{% for parameter in method.parameters.header %}
72-
'{{ parameter.name }}': ${{ parameter.name | caseCamel | escapeKeyword }},
73-
{% endfor %}
74-
{% for key, header in method.headers %}
75-
'{{ key }}': '{{ header }}',
76-
{% endfor %}
77-
};
66+
let chunksUploaded = 0;
7867

7968
{% for parameter in method.parameters.all %}
8069
{% if parameter.isUploadID %}
81-
if({{ parameter.name | caseCamel | escapeKeyword }} != 'unique()') {
82-
try {
83-
response = await this.client.call('get', path + '/' + {{ parameter.name }}, headers);
84-
counter = response.chunksUploaded;
85-
} catch(e) {
86-
}
70+
if({{ parameter.name | caseCamel | escapeKeyword }} != 'unique()') {
71+
try {
72+
response = await this.client.call('get', path + '/' + {{ parameter.name }}, headers);
73+
chunksUploaded = response.chunksUploaded;
74+
} catch(e) {
8775
}
76+
}
8877
{% endif %}
8978
{% endfor %}
9079

91-
for (counter; counter < totalCounters; counter++) {
92-
const start = (counter * client.CHUNK_SIZE);
93-
const end = Math.min((((counter * client.CHUNK_SIZE) + client.CHUNK_SIZE) - 1), size);
80+
let currentChunk = Buffer.from('');
81+
let currentChunkSize = 0;
82+
let currentChunkStart = 0;
83+
84+
const selfClient = this.client;
9485

86+
async function uploadChunk(lastUpload = false) {
87+
if(chunksUploaded - 1 >= currentChunkStart / client.CHUNK_SIZE) {
88+
return;
89+
}
90+
91+
const start = currentChunkStart;
92+
const end = Math.min(((start + client.CHUNK_SIZE) - 1), size);
93+
94+
if(!lastUpload || currentChunkStart !== 0) {
9595
headers['content-range'] = 'bytes ' + start + '-' + end + '/' + size;
96+
}
9697

97-
if (id) {
98-
headers['x-{{spec.title | caseLower }}-id'] = id;
99-
}
98+
if (id) {
99+
headers['x-{{spec.title | caseLower }}-id'] = id;
100+
}
101+
102+
const stream = Stream.Readable.from(currentChunk);
103+
payload['{{ parameter.name }}'] = { type: 'file', file: stream, filename: file.name };
104+
105+
response = await selfClient.call('{{ method.method | caseLower }}', path, headers, payload{% if method.type == 'location' %}, 'arraybuffer'{% endif %});
100106

101-
const stream = fs.createReadStream({{ parameter.name | caseCamel | escapeKeyword }}, {
102-
start,
103-
end
107+
if (!id) {
108+
id = response['$id'];
109+
}
110+
111+
if (onProgress !== null) {
112+
onProgress({
113+
$id: response['$id'],
114+
progress: Math.min((start+client.CHUNK_SIZE) * client.CHUNK_SIZE, size) / size * 100,
115+
sizeUploaded: end+1,
116+
chunksTotal: response['chunksTotal'],
117+
chunksUploaded: response['chunksUploaded']
104118
});
105-
payload['{{ parameter.name }}'] = stream;
119+
}
106120

107-
response = await this.client.call('{{ method.method | caseLower }}', path, headers, payload{% if method.type == 'location' %}, 'arraybuffer'{% endif %});
121+
currentChunkStart += client.CHUNK_SIZE;
122+
}
108123

109-
if (!id) {
110-
id = response['$id'];
124+
return await new Promise((resolve, reject) => {
125+
const writeStream = new Stream.Writable();
126+
writeStream._write = async (mainChunk, encoding, next) => {
127+
// Segment incoming chunk into up to 5MB chunks
128+
const mainChunkSize = Buffer.byteLength(mainChunk);
129+
const chunksCount = Math.ceil(mainChunkSize / client.CHUNK_SIZE);
130+
const chunks = [];
131+
132+
for(let i = 0; i < chunksCount; i++) {
133+
const chunk = mainChunk.slice(i * client.CHUNK_SIZE, client.CHUNK_SIZE);
134+
chunks.push(chunk);
111135
}
112-
113-
if (onProgress !== null) {
114-
onProgress({
115-
$id: response['$id'],
116-
progress: Math.min((counter+1) * client.CHUNK_SIZE, size) / size * 100,
117-
sizeUploaded: end+1,
118-
chunksTotal: response['chunksTotal'],
119-
chunksUploaded: response['chunksUploaded']
120-
});
136+
137+
for (const chunk of chunks) {
138+
const chunkSize = Buffer.byteLength(chunk);
139+
140+
if(chunkSize + currentChunkSize == client.CHUNK_SIZE) {
141+
// Upload chunk
142+
currentChunk = Buffer.concat([currentChunk, chunk]);
143+
await uploadChunk();
144+
currentChunk = Buffer.from('');
145+
currentChunkSize = 0;
146+
} else if(chunkSize + currentChunkSize > client.CHUNK_SIZE) {
147+
// Upload chunk, put rest into next chunk
148+
const bytesToUpload = client.CHUNK_SIZE - currentChunkSize;
149+
const newChunkSection = chunk.slice(0, bytesToUpload);
150+
currentChunk = Buffer.concat([currentChunk, newChunkSection]);
151+
currentChunkSize = Buffer.byteLength(currentChunk);
152+
await uploadChunk();
153+
currentChunk = chunk.slice(bytesToUpload, undefined);
154+
currentChunkSize = chunkSize - bytesToUpload;
155+
} else {
156+
// Append into current chunk
157+
currentChunk = Buffer.concat([currentChunk, chunk]);
158+
currentChunkSize = chunkSize + currentChunkSize;
159+
}
121160
}
161+
162+
next();
122163
}
123164

124-
return response;
125-
}
165+
writeStream.on("finish", async () => {
166+
if(currentChunkSize > 0) {
167+
await uploadChunk(true);
168+
}
169+
170+
resolve(response);
171+
});
172+
173+
writeStream.on("error", (err) => {
174+
reject(err);
175+
});
176+
177+
{{ parameter.name | caseCamel | escapeKeyword }}.stream.pipe(writeStream);
178+
});
126179
{% endif %}
127180
{% endfor %}
128181
{% else %}

‎tests/languages/node/test.js

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11

22
const appwrite = require('../../sdks/node/index');
3+
const InputFile = require('../../sdks/node/lib/inputFile');
34
const fs = require('fs');
45

56
async function start() {
@@ -53,10 +54,10 @@ async function start() {
5354
response = await general.redirect();
5455
console.log(response.result);
5556

56-
response = await general.upload('string', 123, ['string in array'], __dirname + '/../../resources/file.png');
57+
response = await general.upload('string', 123, ['string in array'], InputFile.fromPath(__dirname + '/../../resources/file.png', 'file.png'));
5758
console.log(response.result);
5859

59-
response = await general.upload('string', 123, ['string in array'], __dirname + '/../../resources/large_file.mp4');
60+
response = await general.upload('string', 123, ['string in array'], InputFile.fromPath(__dirname + '/../../resources/large_file.mp4', 'large_file.mp4'));
6061
console.log(response.result);
6162

6263
try {
@@ -80,4 +81,6 @@ async function start() {
8081
await general.empty();
8182
}
8283

83-
start();
84+
start().catch((err) => {
85+
console.log(err);
86+
});

0 commit comments

Comments
 (0)