Merge pull request #13 from diffusionstudio/konstantin/feature/opus-wasm-encoder

Konstantin/feature/opus wasm encoder

Author: k9p5

README.md

@@ -59,9 +59,9 @@ const composition = new core.Composition(); // 1920x1080
 await composition.add(video);  // convenience function for 
 await composition.add(text);   // clip -> track -> composition
 
-// export video using webcodecs at 25 FPS
-new core.WebcodecsEncoder(composition, { fps: 25 })
-  .export(); // use resolution = 2 to render at 4k 
+// render video using webcodecs at 25 FPS
+// use resolution: 2 to render at 4k 
+new core.Encoder(composition, { fps: 25 }).render();
 ```
 
 This may look familiar to some. That is because the API is heavily inspired by **Moviepy** and Swift UI. It models the structure of popular video editing applications such as Adobe Premiere or CapCut. The current state can be visualized as follows:
@@ -89,9 +89,9 @@ https://github.com/user-attachments/assets/7a943407-e916-4d9f-b46a-3163dbff44c3
 
 **Remotion** is a React-based video creation tool that transforms the entire DOM into videos. It's particularly suited for beginners, as web developers can start creating videos using the skills they already have.
 
-**Motion Canvas** uses a Canvas 2D implementation for rendering. It is intended as a standalone editor for creating production-quality animations. It features a unique imperative API that adds elements to the timeline procedurally, rather than relying on keyframes like traditional video editing tools. This makes Motion Canvas ideal for crafting detailed, animated videos.
+**Motion Canvas** is intended as a standalone editor for creating production-quality animations. It features a unique imperative API that adds elements to the timeline procedurally, rather than relying on keyframes like traditional video editing tools. This makes Motion Canvas ideal for crafting detailed, animated videos.
 
-In contrast, **Diffusion Studio** is not a framework with a visual editing interface but a video editing library that can be integrated into existing projects. It operates entirely on the client-side, eliminating the need for additional backend infrastructure. Diffusion Studio is also dedicated to supporting the latest rendering technologies, including WebGPU, WebGL, and WebCodecs. If a feature you need isn't available, you can easily extend it using [Pixi.js](https://github.com/pixijs/pixijs).
+In contrast, **Diffusion Studio** is not a framework with a visual editing interface but a **video editing library** that can be integrated into existing projects. It operates entirely on the **client-side**, eliminating the need for additional backend infrastructure. Diffusion Studio is also dedicated to supporting the latest rendering technologies, including WebGPU, WebGL, and WebCodecs. If a feature you need isn't available, you can **easily extend** it using [Pixi.js](https://github.com/pixijs/pixijs).
 
 ## Current features
 * **Video/Audio** trim and offset
@@ -134,50 +134,49 @@ I’m excited to be part of the next generation of video editing technology.
 ## Compatability
 
 ✅ Supported 
-⏰ Not yet supported 
-❌ Not planned 
-🔬 Not tested
+🧪 Experimental 
+❌ Not supported 
 
 ### Desktop
 
 | Browser           |    | Operating System  |    |
 | ----------------- | -- | ----------------- | -- |
 | Chrome            | ✅ | Windows           | ✅ |
 | Edge              | ✅ | Macos             | ✅ |
-| Firefox           | ⏰ | Linux             | ✅ |
-| Safari            | ⏰ |
-| Opera             | 🔬 |
+| Firefox           | 🧪 | Linux             | ✅ |
+| Safari            | ✅ |
+| Opera             | ✅ |
 | Brave             | ✅ |
-| Vivaldi           | 🔬 |
+| Vivaldi           | ✅ |
 
 
 ### Mobile
 
 | Browser           |    | Operating System  |    |
 | ----------------- | -- | ----------------- | -- |
-| Brave Android     | 🔬 | Android           | 🔬 |
-| Chrome Android    | 🔬 | iOS               | ⏰ |
-| Firefox Android   | ⏰ |
-| Opera Android     | 🔬 |
-| Safari iOS        | ⏰ |
+| Brave Android     | ✅ | Android           | ✅ |
+| Chrome Android    | ✅ | iOS               | ✅ |
+| Firefox Android   | 🧪 |
+| Opera Android     | ✅ |
+| Safari iOS        | ✅ |
 
 
 |             | Demultiplexing | Multiplexing |
 | ----------- | -------------- | -------------|
 | Mp4         | ✅             | ✅           |
-| Webm        | ✅             | ⏰           |
+| Webm        | ✅             | ❌           |
 | Mov         | ✅             | ❌           |
-| Mkv         | ⏰             | ❌           |
-| Avi         | ⏰             | ❌           |
+| Mkv         | ❌             | ❌           |
+| Avi         | ❌             | ❌           |
 
 |             | Decoding | Encoding          |
 | ----------- | -------- | ----------------- |
 | Avc1        | ✅       | ✅                |
-| Hevc        | ✅       | ⏰                |
-| VP9         | ✅       | ⏰                |
-| VP8         | ✅       | ⏰                |
-| AAC         | ✅       | ✅ (except Linux) |
+| AAC         | ✅       | ✅ (Chromium only)|
 | Opus        | ✅       | ✅                |
+| Wav         | ✅       | ✅                |
+| Hevc        | ✅       | ❌                |
+| VP9         | ✅       | ❌                |
+| VP8         | ✅       | ❌                |
 | Mp3         | ✅       | ❌                |
 | Ogg         | ✅       | ❌                |
-| Wav         | ✅       | N/A               |

package-lock.json

@@ -1,7 +1,7 @@
 {
   "name": "@diffusionstudio/core",
   "private": false,
-  "version": "1.0.0-rc.4",
+  "version": "1.0.0-rc.5",
   "type": "module",
   "description": "Build bleeding edge video processing applications",
   "files": [
@@ -61,21 +61,29 @@
     "url": "[email protected]:diffusionstudio/core.git"
   },
   "keywords": [
-    "edit videos",
-    "programmatic video editing",
-    "video editing",
+    "mp4",
+    "web",
+    "aac",
+    "h264",
+    "opus",
+    "edit",
+    "webgl",
+    "video",
+    "audio",
+    "record",
+    "canvas",
+    "tiktok",
+    "webgpu",
+    "encode",
+    "editor",
+    "decode",
+    "editing",
+    "youtube",
+    "recorder",
     "automate",
-    "web-based",
-    "browser-based",
-    "automation",
-    "ai video editor",
-    "video apps",
-    "video applications",
-    "create videos",
-    "automate workflows",
-    "automated video editing",
-    "free video editor",
-    "editor for youtube",
-    "online video editor"
+    "webcodecs",
+    "client-side",
+    "programmatic",
+    "browser-based"
   ]
 }

package.json

@@ -1,12 +1,12 @@
 import * as core from '../src';
-import { exportComposition } from './export';
+import { render } from './render';
 
 export function setupControls(composition: core.Composition) {
   const handlePlay = () => composition.play();
   const handlePause = () => composition.pause();
   const handleBack = () => composition.seek(0);
   const handleForward = () => composition.seek(composition.duration.frames);
-  const handleExport = () => exportComposition(composition);
+  const handleExport = () => render(composition);
 
   playButton.addEventListener('click', handlePlay);
   pauseButton.addEventListener('click', handlePause);

playground/controls.ts

@@ -104,24 +104,21 @@ await composition.add(
 );
 
 await composition.add(
-  new core.ImageClip(
-    await core.ImageSource.from('/dvd_logo.svg'),
-    {
-      stop: composition.duration,
-      x(this: core.TextClip, time: core.Timestamp) {
-        const width = typeof this.width == 'number' ? this.width : 0;
-        const range = composition.width - width;
-        const x = (time.seconds * 500) % (range * 2);
+  new core.ImageClip(await core.ImageSource.from('/dvd_logo.svg'), {
+    stop: composition.duration,
+    x(this: core.TextClip, time: core.Timestamp) {
+      const width = typeof this.width == 'number' ? this.width : 0;
+      const range = composition.width - width;
+      const x = (time.seconds * 500) % (range * 2);
 
-        return x > range ? range * 2 - x : x;
-      },
-      y(this: core.TextClip, time: core.Timestamp) {
-        const height = typeof this.height == 'number' ? this.height : 0;
-        const range = composition.height - height;
-        const y = (time.seconds * 200) % (range * 2);
+      return x > range ? range * 2 - x : x;
+    },
+    y(this: core.TextClip, time: core.Timestamp) {
+      const height = typeof this.height == 'number' ? this.height : 0;
+      const range = composition.height - height;
+      const y = (time.seconds * 200) % (range * 2);
 
-        return y > range ? range * 2 - y : y;
-      },
-    }
-  )
+      return y > range ? range * 2 - y : y;
+    },
+  })
 );

playground/main.ts

@@ -3,11 +3,11 @@ import * as core from '../src';
 
 let fps = 30;
 
-export async function exportComposition(composition: core.Composition) {
+export async function render(composition: core.Composition) {
   if (loader.style.display != 'none') return;
 
   try {
-    const encoder = new core.WebcodecsEncoder(composition, { debug: true, fps });
+    const encoder = new core.Encoder(composition, { debug: true, fps });
 
     encoder.on('render', (event) => {
       const { progress, total } = event.detail;
@@ -24,15 +24,17 @@ export async function exportComposition(composition: core.Composition) {
         },
       ],
     });
+
     loader.style.display = 'block';
-    await encoder.export(fileHandle);
+    await encoder.render(fileHandle);
   } catch (e) {
     if (e instanceof DOMException) {
+      console.log(e)
       // user canceled file picker
-    } else if (e instanceof core.ExportError) {
+    } else if (e instanceof core.EncoderError) {
       alert(e.message);
     } else {
-      alert(String(e))
+      alert(String(e));
     }
   } finally {
     loader.style.display = 'none';
@@ -52,3 +54,7 @@ fpsButton.addEventListener('click', () => {
 
   if (!Number.isNaN(value)) fps = value
 });
+
+if (!('showSaveFilePicker' in window)) {
+  Object.assign(window, { showSaveFilePicker: async () => undefined });
+}

playground/export.ts playground/render.tsplayground/export.ts renamed to playground/render.ts

@@ -47,7 +47,10 @@ export class VideoClip extends VisualMixin(MediaClip<VideoClipProps>) {
 	public constructor(source?: File | VideoSource, props: VideoClipProps = {}) {
 		super();
 
-		(this.textrues.html5.source as any).playsInline = true;
+		this.element.controls = false;
+		this.element.playsInline = true;
+		this.element.style.display = 'hidden';
+
 		(this.textrues.html5.source as any).autoPlay = false;
 		(this.textrues.html5.source as any).loop = false;
 		this.sprite.texture = this.textrues.html5;
@@ -96,11 +99,11 @@ export class VideoClip extends VisualMixin(MediaClip<VideoClipProps>) {
 	}
 
 	public async connect(track: Track<VideoClip>): Promise<void> {
+		super.connect(track);
+
 		// without seeking the first frame a black frame will be rendered
 		const frame = track.composition?.frame ?? 0;
 		await this.seek(Timestamp.fromFrames(frame));
-
-		super.connect(track);
 	}
 
 	@visualize

src/clips/video/video.ts

@@ -402,7 +402,7 @@ export class Composition extends EventEmitterMixin<CompositionEvents, typeof Ser
 		this.trigger('pause', this.frame);
 	}
 
-	public async audio(numberOfChannels = 2, sampleRate = 44100): Promise<AudioBuffer> {
+	public async audio(numberOfChannels = 2, sampleRate = 48000): Promise<AudioBuffer> {
 		const length = this.duration.seconds * sampleRate;
 		const context = new OfflineAudioContext({
 			sampleRate,
@@ -445,7 +445,7 @@ export class Composition extends EventEmitterMixin<CompositionEvents, typeof Ser
 					output.getChannelData(i).set(outputData);
 				}
 			} catch (e) {
-				console.warn('could not get channel data', e);
+				console.error('could not get channel data', e);
 			}
 
 			clip.source.audioBuffer = undefined;