Prevent empty last part in progressive upload

Author: olivier-lando

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 All changes to this project will be documented in this file.
 
+## [1.0.9] - 2022-05-24
+- Progressive upload: prevent last part to be empty
+
 ## [1.0.8] - 2022-04-27
 - Create a AbstractUploader
 - Add origin header

README.md

@@ -276,10 +276,11 @@ Using delegated upload tokens for authentication is best options when uploading
 #### Common options
 
 
-| Option name | Mandatory | Type   | Description                                           |
-| ----------: | --------- | ------ | ----------------------------------------------------- |
-|     apiHost | no        | string | api.video host (default: ws.api.video)                |
-|     retries | no        | number | number of retries when an API call fails (default: 5) |
+|       Option name | Mandatory | Type    | Description                                                                                                                                      |
+| ----------------: | --------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+|           apiHost | no        | string  | api.video host (default: ws.api.video)                                                                                                           |
+|           retries | no        | number  | number of retries when an API call fails (default: 5)                                                                                            |
+| preventEmptyParts | no        | boolean | if true, the upload will succeed even if an empty Blob is passed to uploadLastPart(). This may alter performances a bit in some cases (default: false) |
 
 
 ### Example

dist/index.js

@@ -1,9 +1,12 @@
 import { AbstractUploader, CommonOptions, VideoUploadResponse, WithAccessToken, WithApiKey, WithUploadToken } from "./abstract-uploader";
-export interface ProgressiveUploaderOptionsWithUploadToken extends CommonOptions, WithUploadToken {
+export interface ProgressiveUploadCommonOptions {
+    preventEmptyParts?: boolean;
 }
-export interface ProgressiveUploaderOptionsWithAccessToken extends CommonOptions, WithAccessToken {
+export interface ProgressiveUploaderOptionsWithUploadToken extends ProgressiveUploadCommonOptions, CommonOptions, WithUploadToken {
 }
-export interface ProgressiveUploaderOptionsWithApiKey extends CommonOptions, WithApiKey {
+export interface ProgressiveUploaderOptionsWithAccessToken extends ProgressiveUploadCommonOptions, CommonOptions, WithAccessToken {
+}
+export interface ProgressiveUploaderOptionsWithApiKey extends ProgressiveUploadCommonOptions, CommonOptions, WithApiKey {
 }
 export interface ProgressiveUploadProgressEvent {
     uploadedBytes: number;
@@ -18,6 +21,7 @@ export declare class ProgressiveUploader extends AbstractUploader<ProgressivePro
     private currentPartBlobs;
     private currentPartBlobsSize;
     private queue;
+    private preventEmptyParts;
     constructor(options: ProgressiveUploaderOptionsWithAccessToken | ProgressiveUploaderOptionsWithUploadToken | ProgressiveUploaderOptionsWithApiKey);
     uploadPart(file: Blob): Promise<void>;
     uploadLastPart(file: Blob): Promise<VideoUploadResponse>;

dist/src/progressive-video-uploader.d.ts

@@ -1,6 +1,6 @@
 {
   "name": "@api.video/video-uploader",
-  "version": "1.0.8",
+  "version": "1.0.9",
   "description": "api.video video uploader",
   "repository": {
     "type": "git",

package-lock.json

@@ -1,10 +1,13 @@
 import { AbstractUploader, CommonOptions, MIN_CHUNK_SIZE, VideoUploadResponse, WithAccessToken, WithApiKey, WithUploadToken } from "./abstract-uploader";
 import { PromiseQueue } from "./promise-queue";
 
+export interface ProgressiveUploadCommonOptions {
+    preventEmptyParts?: boolean;
+}
 
-export interface ProgressiveUploaderOptionsWithUploadToken extends CommonOptions, WithUploadToken { }
-export interface ProgressiveUploaderOptionsWithAccessToken extends CommonOptions, WithAccessToken { }
-export interface ProgressiveUploaderOptionsWithApiKey extends CommonOptions, WithApiKey { }
+export interface ProgressiveUploaderOptionsWithUploadToken extends ProgressiveUploadCommonOptions, CommonOptions, WithUploadToken { }
+export interface ProgressiveUploaderOptionsWithAccessToken extends ProgressiveUploadCommonOptions, CommonOptions, WithAccessToken { }
+export interface ProgressiveUploaderOptionsWithApiKey extends ProgressiveUploadCommonOptions, CommonOptions, WithApiKey { }
 
 export interface ProgressiveUploadProgressEvent {
     uploadedBytes: number;
@@ -21,24 +24,40 @@ export class ProgressiveUploader extends AbstractUploader<ProgressiveProgressEve
     private currentPartBlobs: Blob[] = [];
     private currentPartBlobsSize = 0;
     private queue = new PromiseQueue();
+    private preventEmptyParts: boolean;
 
     constructor(options: ProgressiveUploaderOptionsWithAccessToken | ProgressiveUploaderOptionsWithUploadToken | ProgressiveUploaderOptionsWithApiKey) {
         super(options);
+        this.preventEmptyParts = options.preventEmptyParts || false;
     }
 
     public uploadPart(file: Blob): Promise<void> {
         this.currentPartBlobsSize += file.size;
         this.currentPartBlobs.push(file);
 
-        if (this.currentPartBlobsSize >= MIN_CHUNK_SIZE) {
-            return this.queue.add(() => {
-                const promise = this.upload(new Blob(this.currentPartBlobs)).then(res => {
-                    this.videoId = res.videoId;
-                });
-                this.currentPartNum++;
+        if ((this.preventEmptyParts && (this.currentPartBlobsSize - file.size >= MIN_CHUNK_SIZE))
+            || (!this.preventEmptyParts && (this.currentPartBlobsSize >= MIN_CHUNK_SIZE))) {
+
+            let toSend: any[];
+            if(this.preventEmptyParts) {
+                toSend = this.currentPartBlobs.slice(0, -1);
+                this.currentPartBlobs = this.currentPartBlobs.slice(-1);
+                this.currentPartBlobsSize = this.currentPartBlobs.length === 0 ? 0 : this.currentPartBlobs[0].size;
+            } else {
+                toSend = this.currentPartBlobs;
                 this.currentPartBlobs = [];
                 this.currentPartBlobsSize = 0;
-                return promise;
+            }
+
+            return this.queue.add(() => {
+                if (toSend.length > 0) {
+                    const promise = this.upload(new Blob(toSend)).then(res => {
+                        this.videoId = res.videoId;
+                    });
+                    this.currentPartNum++;
+                    return promise;
+                }
+                return new Promise(resolve => resolve());
             });
         }
         return Promise.resolve();

package.json

@@ -69,6 +69,36 @@ describe('Content-range', () => {
     });
 });
 
+describe('Prevent empty part', () => {
+    beforeEach(() => mock.setup());
+    afterEach(() => mock.teardown());
+
+    it('content-range headers are properly set', async () => {
+        const uploadToken = "the-upload-token";
+
+        const uploader = new ProgressiveUploader({uploadToken, preventEmptyParts: true});
+
+        const expectedRanges = [
+            'part 1/*',
+            'part 2/*',
+            'part 3/3',
+        ];
+
+        mock.post(`https://ws.api.video/upload?token=${uploadToken}`, (req, res) => {
+            expect(req.header("content-range")).to.be.eq(expectedRanges.shift());
+            return res.status(201).body("{}");
+        });
+
+        await uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename"));
+        await uploader.uploadPart(new File([new ArrayBuffer(5*1024*1024)], "filename"));
+        await uploader.uploadPart(new File([new ArrayBuffer(3*1024*1024)], "filename"));
+        await uploader.uploadLastPart(new Blob());
+
+        expect(expectedRanges).has.lengthOf(0);
+    });
+});
+
+
 describe('Access token auth', () => {
     beforeEach(() => mock.setup());
     afterEach(() => mock.teardown());