diff --git a/templates/node/src/client.ts.twig b/templates/node/src/client.ts.twig index 7bfbb9b9d..684b491ae 100644 --- a/templates/node/src/client.ts.twig +++ b/templates/node/src/client.ts.twig @@ -67,6 +67,7 @@ function getUserAgent() { class Client { static CHUNK_SIZE = 1024 * 1024 * 5; + static MAX_CONCURRENCY = 6; config = { endpoint: '{{ spec.endpoint }}', @@ -211,38 +212,104 @@ class Client { return await this.call(method, url, headers, originalPayload); } - let start = 0; - let response = null; + const totalChunks = Math.ceil(file.size / Client.CHUNK_SIZE); - while (start < file.size) { - let end = start + Client.CHUNK_SIZE; // Prepare end for the next chunk - if (end >= file.size) { - end = file.size; // Adjust for the last chunk to include the last byte - } + const firstChunkStart = 0; + const firstChunkEnd = Math.min(Client.CHUNK_SIZE, file.size); + const firstChunk = file.slice(firstChunkStart, firstChunkEnd); - headers['content-range'] = `bytes ${start}-${end-1}/${file.size}`; - const chunk = file.slice(start, end); + const firstChunkHeaders = { ...headers }; + firstChunkHeaders['content-range'] = `bytes ${firstChunkStart}-${firstChunkEnd - 1}/${file.size}`; + + const firstPayload = { ...originalPayload }; + firstPayload[fileParam] = new File([firstChunk], file.name); + + const firstResponse = await this.call(method, url, firstChunkHeaders, firstPayload); + + if (!firstResponse?.$id) { + throw new Error('First chunk upload failed - no ID returned'); + } - let payload = { ...originalPayload }; - payload[fileParam] = new File([chunk], file.name); + let completedChunks = 1; + let totalUploaded = firstChunkEnd; + + if (onProgress && typeof onProgress === 'function') { + onProgress({ + $id: firstResponse.$id, + progress: Math.round((totalUploaded / file.size) * 100), + sizeUploaded: totalUploaded, + chunksTotal: totalChunks, + chunksUploaded: completedChunks + }); + } - response = await this.call(method, url, headers, payload); + if (totalChunks === 1) { + return firstResponse; + } - if (onProgress && typeof onProgress === 'function') { - onProgress({ - $id: response.$id, - progress: Math.round((end / file.size) * 100), - sizeUploaded: end, - chunksTotal: Math.ceil(file.size / Client.CHUNK_SIZE), - chunksUploaded: Math.ceil(end / Client.CHUNK_SIZE) - }); + let response = firstResponse; + + for (let chunkIndex = 1; chunkIndex < totalChunks; chunkIndex += Client.MAX_CONCURRENCY) { + const batchEnd = Math.min(chunkIndex + Client.MAX_CONCURRENCY, totalChunks); + + const batchPromises = []; + for (let i = chunkIndex; i < batchEnd; i++) { + const start = i * Client.CHUNK_SIZE; + const end = Math.min(start + Client.CHUNK_SIZE, file.size); + + batchPromises.push((async () => { + const chunk = file.slice(start, end); + const chunkHeaders = { ...headers }; + chunkHeaders['content-range'] = `bytes ${start}-${end - 1}/${file.size}`; + chunkHeaders['x-{{spec.title | caseLower}}-id'] = firstResponse.$id; + + const payload = { ...originalPayload }; + payload[fileParam] = new File([chunk], file.name); + + try { + const chunkResponse = await this.call(method, url, chunkHeaders, payload); + return { + success: true, + response: chunkResponse, + chunkInfo: { index: i, start, end }, + error: null + }; + } catch (error) { + return { + success: false, + response: null, + chunkInfo: { index: i, start, end }, + error + }; + } + })()); } - if (response && response.$id) { - headers['x-{{spec.title | caseLower }}-id'] = response.$id; + const batchResults = await Promise.all(batchPromises); + + const failures = batchResults.filter(result => !result.success); + if (failures.length > 0) { + const errorMessages = failures.map(f => `Chunk ${f.chunkInfo.index}: ${f.error}`); + throw new Error(`Chunk upload failures: ${errorMessages.join(', ')}`); } - start = end; + for (const result of batchResults) { + if (result.success) { + completedChunks++; + totalUploaded += (result.chunkInfo.end - result.chunkInfo.start); + response = result.response; + + if (onProgress && typeof onProgress === 'function') { + onProgress({ + $id: firstResponse.$id, + progress: Math.round((totalUploaded / file.size) * 100), + sizeUploaded: totalUploaded, + chunksTotal: totalChunks, + chunksUploaded: completedChunks + }); + } + } + } } return response; diff --git a/templates/web/src/client.ts.twig b/templates/web/src/client.ts.twig index 92b1e7b4b..c37d0cb66 100644 --- a/templates/web/src/client.ts.twig +++ b/templates/web/src/client.ts.twig @@ -296,6 +296,7 @@ class {{spec.title | caseUcfirst}}Exception extends Error { */ class Client { static CHUNK_SIZE = 1024 * 1024 * 5; + static MAX_CONCURRENCY = 6; /** * Holds configuration such as project. @@ -639,38 +640,104 @@ class Client { return await this.call(method, url, headers, originalPayload); } - let start = 0; - let response = null; + const totalChunks = Math.ceil(file.size / Client.CHUNK_SIZE); + + const firstChunkStart = 0; + const firstChunkEnd = Math.min(Client.CHUNK_SIZE, file.size); + const firstChunk = file.slice(firstChunkStart, firstChunkEnd); + + const firstChunkHeaders = { ...headers }; + firstChunkHeaders['content-range'] = `bytes ${firstChunkStart}-${firstChunkEnd - 1}/${file.size}`; + + const firstPayload = { ...originalPayload }; + firstPayload[fileParam] = new File([firstChunk], file.name); + + const firstResponse = await this.call(method, url, firstChunkHeaders, firstPayload); + + if (!firstResponse?.$id) { + throw new Error('First chunk upload failed - no ID returned'); + } - while (start < file.size) { - let end = start + Client.CHUNK_SIZE; // Prepare end for the next chunk - if (end >= file.size) { - end = file.size; // Adjust for the last chunk to include the last byte - } + let completedChunks = 1; + let totalUploaded = firstChunkEnd; + + if (onProgress && typeof onProgress === 'function') { + onProgress({ + $id: firstResponse.$id, + progress: Math.round((totalUploaded / file.size) * 100), + sizeUploaded: totalUploaded, + chunksTotal: totalChunks, + chunksUploaded: completedChunks + }); + } - headers['content-range'] = `bytes ${start}-${end-1}/${file.size}`; - const chunk = file.slice(start, end); + if (totalChunks === 1) { + return firstResponse; + } - let payload = { ...originalPayload }; - payload[fileParam] = new File([chunk], file.name); + let response = firstResponse; + + for (let chunkIndex = 1; chunkIndex < totalChunks; chunkIndex += Client.MAX_CONCURRENCY) { + const batchEnd = Math.min(chunkIndex + Client.MAX_CONCURRENCY, totalChunks); + + const batchPromises = []; + for (let i = chunkIndex; i < batchEnd; i++) { + const start = i * Client.CHUNK_SIZE; + const end = Math.min(start + Client.CHUNK_SIZE, file.size); + + batchPromises.push((async () => { + const chunk = file.slice(start, end); + const chunkHeaders = { ...headers }; + chunkHeaders['content-range'] = `bytes ${start}-${end - 1}/${file.size}`; + chunkHeaders['x-{{spec.title | caseLower}}-id'] = firstResponse.$id; + + const payload = { ...originalPayload }; + payload[fileParam] = new File([chunk], file.name); + + try { + const chunkResponse = await this.call(method, url, chunkHeaders, payload); + return { + success: true, + response: chunkResponse, + chunkInfo: { index: i, start, end }, + error: null + }; + } catch (error) { + return { + success: false, + response: null, + chunkInfo: { index: i, start, end }, + error + }; + } + })()); + } - response = await this.call(method, url, headers, payload); + const batchResults = await Promise.all(batchPromises); - if (onProgress && typeof onProgress === 'function') { - onProgress({ - $id: response.$id, - progress: Math.round((end / file.size) * 100), - sizeUploaded: end, - chunksTotal: Math.ceil(file.size / Client.CHUNK_SIZE), - chunksUploaded: Math.ceil(end / Client.CHUNK_SIZE) - }); + const failures = batchResults.filter(result => !result.success); + if (failures.length > 0) { + const errorMessages = failures.map(f => `Chunk ${f.chunkInfo.index}: ${f.error}`); + throw new Error(`Chunk upload failures: ${errorMessages.join(', ')}`); } - if (response && response.$id) { - headers['x-{{spec.title | caseLower }}-id'] = response.$id; + for (const result of batchResults) { + if (result.success) { + completedChunks++; + totalUploaded += (result.chunkInfo.end - result.chunkInfo.start); + response = result.response; + + if (onProgress && typeof onProgress === 'function') { + onProgress({ + $id: firstResponse.$id, + progress: Math.round((totalUploaded / file.size) * 100), + sizeUploaded: totalUploaded, + chunksTotal: totalChunks, + chunksUploaded: completedChunks + }); + } + } } - - start = end; } return response;