Merge pull request #16 from diffusionstudio/konstantin/fix/load-partial-video

implemented and tested partial video loading

Author: k9p5

package-lock.json

@@ -1,7 +1,7 @@
 {
   "name": "@diffusionstudio/core",
   "private": false,
-  "version": "1.0.0-rc.6",
+  "version": "1.0.0-rc.7",
   "type": "module",
   "description": "Build bleeding edge video processing applications",
   "files": [

package.json

@@ -15,8 +15,8 @@ import type { MimeType } from '../../types';
  * @param mimeType The mimetype to check
  * @returns A valid mimetype
  */
-export function parseMimeType(mimeType: string): MimeType {
-	if (!Object.keys(SUPPORTED_MIME_TYPES.MIXED).includes(mimeType)) {
+export function parseMimeType(mimeType?: string | null): MimeType {
+	if (!Object.keys(SUPPORTED_MIME_TYPES.MIXED).includes(mimeType ?? '')) {
 		throw new errors.ValidationError({
 			message: `${mimeType} is not an accepted mime type`,
 			code: 'invalid_mimetype',

src/clips/utils/index.ts

@@ -51,6 +51,7 @@ export class VideoClip extends VisualMixin(MediaClip<VideoClipProps>) {
 		this.element.controls = false;
 		this.element.playsInline = true;
 		this.element.style.display = 'hidden';
+		this.element.crossOrigin = "anonymous";
 
 		(this.textrues.html5.source as any).autoPlay = false;
 		(this.textrues.html5.source as any).loop = false;
@@ -140,7 +141,7 @@ export class VideoClip extends VisualMixin(MediaClip<VideoClipProps>) {
 
 	public async seek(time: Timestamp): Promise<void> {
 		if (this.track?.composition?.rendering) {
-			const buffer = this.decodeVideo();
+			const buffer = await this.decodeVideo();
 			return new Promise<void>((resolve) => {
 				buffer.onenqueue = () => resolve();
 			});
@@ -149,7 +150,7 @@ export class VideoClip extends VisualMixin(MediaClip<VideoClipProps>) {
 		return super.seek(time);
 	}
 
-	private decodeVideo() {
+	private async decodeVideo() {
 		this.buffer = new FrameBuffer();
 		this.worker = new DecodeWorker();
 
@@ -165,7 +166,7 @@ export class VideoClip extends VisualMixin(MediaClip<VideoClipProps>) {
 
 		this.worker.postMessage({
 			type: 'init',
-			file: this.source.file!,
+			file: await this.source.getFile(),
 			range: this.demuxRange,
 			fps: this.track?.composition?.fps ?? FPS_DEFAULT,
 		} satisfies InitMessageData);

src/clips/video/video.ts

@@ -7,8 +7,6 @@
 
 import { Source } from './source';
 import { documentToSvgImageUrl } from './html.utils';
-import { parseMimeType } from '../clips';
-import { IOError } from '../errors';
 
 import type { ClipType } from '../clips';
 
@@ -57,47 +55,26 @@ export class HtmlSource extends Source {
 		return this.objectURL;
 	}
 
-	public async from(input: File | string, init?: RequestInit | undefined): Promise<this> {
-		try {
-			this.state = 'LOADING';
-
-			if (input instanceof File) {
-				this.name = input.name;
-				this.mimeType = parseMimeType(input.type);
-				this.external = false;
-				this.file = input;
-			} else {
-				// case input is a request url
-				const res = await fetch(input, init);
-
-				if (!res?.ok) throw new IOError({
-					code: 'unexpectedIOError',
-					message: 'An unexpected error occurred while fetching the file',
-				});
-
-				const blob = await res.blob();
-				this.name = input.toString().split('/').at(-1) ?? '';
-				this.external = true;
-				this.file = new File([blob], this.name, { type: blob.type });
-				this.externalURL = input;
-				this.mimeType = parseMimeType(blob.type);
-			}
-
-			this.iframe.setAttribute('src', URL.createObjectURL(this.file));
-
-			await new Promise<void>((resolve, reject) => {
-				this.iframe.onload = () => resolve();
-				this.iframe.onerror = (e) => reject(e);
-			});
-
-			this.state = 'READY';
-			this.trigger('load', undefined);
-		} catch (e) {
-			this.state = 'ERROR';
-			throw e;
-		}
+	protected async loadUrl(url: string | URL | Request, init?: RequestInit) {
+		await super.loadUrl(url, init);
+
+		this.iframe.setAttribute('src', URL.createObjectURL(this.file!));
+
+		await new Promise<void>((resolve, reject) => {
+			this.iframe.onload = () => resolve();
+			this.iframe.onerror = (e) => reject(e);
+		});
+	}
+
+	protected async loadFile(file: File) {
+		await super.loadFile(file);
+
+		this.iframe.setAttribute('src', URL.createObjectURL(this.file!));
 
-		return this;
+		await new Promise<void>((resolve, reject) => {
+			this.iframe.onload = () => resolve();
+			this.iframe.onerror = (e) => reject(e);
+		});
 	}
 
 	/**

src/sources/html.ts

@@ -15,8 +15,6 @@ import { Timestamp } from '../models';
 import type { MimeType } from '../types';
 import type { ClipType } from '../clips';
 
-type Url = string | URL | Request;
-
 type Events = {
 	load: undefined;
 	update: undefined;
@@ -32,7 +30,7 @@ export class Source extends EventEmitterMixin<Events, typeof Serializer>(Seriali
 	 * Locally accessible blob address to the data
 	 */
 	@serializable()
-	public objectURL: string | undefined;
+	public objectURL?: string;
 
 	/**
 	 * Defines the default duration
@@ -111,36 +109,44 @@ export class Source extends EventEmitterMixin<Events, typeof Serializer>(Seriali
 		return this.file;
 	}
 
-	public async from(input: File | Url, init?: RequestInit | undefined): Promise<this> {
+	protected async loadFile(file: File) {
+		this.name = file.name;
+		this.mimeType = parseMimeType(file.type);
+		this.external = false;
+		this.file = file;
+	}
+
+	protected async loadUrl(url: string | URL | Request, init?: RequestInit) {
+		const res = await fetch(url, init);
+
+		if (!res?.ok) throw new IOError({
+			code: 'unexpectedIOError',
+			message: 'An unexpected error occurred while fetching the file',
+		});
+
+		const blob = await res.blob();
+		this.name = url.toString().split('/').at(-1) ?? '';
+		this.external = true;
+		this.file = new File([blob], this.name, { type: blob.type });
+		this.externalURL = url;
+		this.mimeType = parseMimeType(blob.type);
+	}
+
+	public async from(input: File | string | URL | Request, init?: RequestInit): Promise<this> {
 		try {
 			this.state = 'LOADING';
 
 			if (input instanceof File) {
-				this.name = input.name;
-				this.mimeType = parseMimeType(input.type);
-				this.external = false;
-				this.file = input;
+				await this.loadFile(input);
 			} else {
-				// case input is a request url
-				const res = await fetch(input, init);
-
-				if (!res?.ok) throw new IOError({
-					code: 'unexpectedIOError',
-					message: 'An unexpected error occurred while fetching the file',
-				});
-
-				const blob = await res.blob();
-				this.name = input.toString().split('/').at(-1) ?? '';
-				this.external = true;
-				this.file = new File([blob], this.name, { type: blob.type });
-				this.externalURL = input;
-				this.mimeType = parseMimeType(blob.type);
+				await this.loadUrl(input, init);
 			}
 
 			this.state = 'READY';
 			this.trigger('load', undefined);
 		} catch (e) {
 			this.state == 'ERROR';
+			this.trigger('error', new Error(String(e)));
 			throw e;
 		}
 
@@ -173,7 +179,7 @@ export class Source extends EventEmitterMixin<Events, typeof Serializer>(Seriali
 	/**
 	 * Downloads the file
 	 */
-	public async export(): Promise<void> {
+	public async download(): Promise<void> {
 		const file = await this.getFile();
 
 		downloadObject(file, this.name);
@@ -192,7 +198,7 @@ export class Source extends EventEmitterMixin<Events, typeof Serializer>(Seriali
 	 */
 	public static async from<T extends Source>(
 		this: new () => T,
-		input: File | Url,
+		input: File | string | URL | Request,
 		init?: RequestInit | undefined,
 		source = new this(),
 	): Promise<T> {

src/sources/source.ts

@@ -0,0 +1,74 @@
+/**
+ * Copyright (c) 2024 The Diffusion Studio Authors
+ *
+ * This Source Code Form is subject to the terms of the Mozilla 
+ * Public License, v. 2.0 that can be found in the LICENSE file.
+ */
+
+import { setFetchMockReturnValue } from '../../vitest.mocks';
+import { describe, expect, it, vi } from 'vitest';
+import { VideoSource } from './video';
+import { sleep } from '../utils';
+
+describe('The Video Source Object', () => {
+	it('should be createable from a http address', async () => {
+		const resetFetch = setFetchMockReturnValue({
+			ok: true,
+			blob: async () => {
+				await sleep(1);
+				return new Blob([], { type: 'video/mp4' });
+			},
+			headers: {
+				get(_: string) {
+					return 'video/mp4'
+				}
+			} as any
+		});
+
+		const source = new VideoSource();
+		const loadFn = vi.fn();
+		source.on('load', loadFn);
+
+		await source.from('https://external.url');
+
+		expect(source.name).toBe('external.url');
+		expect(source.mimeType).toBe('video/mp4');
+		expect(source.external).toBe(true);
+		expect(source.file).toBeUndefined();
+		expect(source.externalURL).toBe('https://external.url');
+		expect(source.objectURL).toBe('https://external.url');
+		expect(loadFn).toHaveBeenCalledTimes(1);
+
+		// file is being loaded in the background
+		await sleep(1);
+		expect(source.file).toBeInstanceOf(File);
+		expect(loadFn).toHaveBeenCalledTimes(2);
+
+		resetFetch();
+	});
+
+	it('should get a file after the asset has been fetched', async () => {
+		const resetFetch = setFetchMockReturnValue({
+			ok: true,
+			blob: async () => {
+				await sleep(1);
+				return new Blob([], { type: 'video/mp4' });
+			},
+			headers: {
+				get(_: string) {
+					return 'video/mp4'
+				}
+			} as any
+		});
+
+		const source = new VideoSource();
+		await source.from('https://external.mp4');
+
+		const file = await source.getFile();
+
+		expect(file).toBeInstanceOf(File);
+		expect(file.name).toBe('external.mp4');
+
+		resetFetch();
+	});
+});

src/sources/video.spec.ts

@@ -6,10 +6,46 @@
  */
 
 import { AudioSource } from './';
+import { parseMimeType } from '../clips';
+import { IOError, ValidationError } from '../errors';
+
 import type { ClipType } from '../clips';
 
 export class VideoSource extends AudioSource {
 	public readonly type: ClipType = 'video';
+	private downloadInProgress = true;
+
+	protected async loadUrl(url: string | URL | Request, init?: RequestInit | undefined) {
+		const res = await fetch(url, init);
+
+		if (!res?.ok) throw new IOError({
+			code: 'unexpectedIOError',
+			message: 'An unexpected error occurred while fetching the file',
+		});
+
+		this.name = url.toString().split('/').at(-1) ?? '';
+		this.external = true;
+		this.externalURL = url;
+		this.objectURL = String(url);
+		this.mimeType = parseMimeType(res.headers.get('Content-type'));
+
+		this.getBlob(res);
+	}
+
+	public async getFile(): Promise<File> {
+		if (!this.file && this.downloadInProgress) {
+			await new Promise(this.resolve('load'));
+		}
+
+		if (!this.file) {
+			throw new ValidationError({
+				code: 'fileNotAccessible',
+				message: "The desired file cannot be accessed",
+			});
+		}
+
+		return this.file;
+	}
 
 	public async thumbnail(): Promise<HTMLVideoElement> {
 		const video = document.createElement('video');
@@ -35,4 +71,19 @@ export class VideoSource extends AudioSource {
 		video.src = await this.createObjectURL();
 		return video;
 	}
+
+	private async getBlob(response: Response) {
+		try {
+			this.downloadInProgress = true;
+			const blob = await response.blob();
+
+			this.file = new File([blob], this.name, { type: blob.type });
+			this.trigger('load', undefined);
+		} catch (e) {
+			this.state == 'ERROR';
+			this.trigger('error', new Error(String(e)));
+		} finally {
+			this.downloadInProgress = false;
+		}
+	}
 }