Video processing improvements

Author: xenova

src/utils/video.js

@@ -1,4 +1,5 @@
 import { RawImage } from "./image.js";
+import { apis } from "../env.js";
 
 export class RawVideoFrame {
 
@@ -42,55 +43,52 @@ export class RawVideo {
 /**
  * Loads a video.
  *
- * @param {string|Blob|HTMLVideoElement} url The video to process.
+ * @param {string|Blob|HTMLVideoElement} src The video to process.
  * @param {Object} [options] Optional parameters.
  * @param {number} [options.num_frames=null] The number of frames to sample uniformly.
- * If provided, the video is seeked to the desired positions rather than processing every frame.
  * @param {number} [options.fps=null] The number of frames to sample per second.
- * If provided (and num_frames is null), the video is seeked at fixed time intervals.
  *
- * @returns {Promise<RawVideo>} The video
+ * @returns {Promise<RawVideo>} The loaded video.
  */
-export async function load_video(url, { num_frames = null, fps = null } = {}) {
+export async function load_video(src, { num_frames = null, fps = null } = {}) {
+    if (!apis.IS_BROWSER_ENV) {
+        throw new Error("`load_video` is currently only supported in browser environments.");
+    }
+
+    // TODO: Support efficiently loading all frames using the WebCodecs API.
+    // Specfically, https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder
+    if (num_frames == null && fps == null) {
+        throw new Error("Either num_frames or fps must be provided.");
+    }
+
     const frames = [];
 
-    const video = document.createElement('video');
-    if (typeof url === 'string') {
-        video.src = url;
-    } else if (url instanceof Blob) {
-        video.src = URL.createObjectURL(url);
-    } else if (url instanceof HTMLVideoElement) {
-        video.src = url.src;
+    const video = document.createElement("video");
+    video.crossOrigin = "anonymous";
+    video.muted = true; // mute to allow autoplay and seeking
+
+    if (typeof src === 'string') {
+        video.src = src;
+    } else if (src instanceof Blob) {
+        video.src = URL.createObjectURL(src);
+    } else if (src instanceof HTMLVideoElement) {
+        video.src = src.src;
     } else {
         throw new Error("Invalid URL or video element provided.");
     }
-    video.crossOrigin = "anonymous";
-    video.muted = true; // mute to allow autoplay
-
     // Wait for metadata to load to obtain duration
-    await new Promise((resolve) => {
-        video.onloadedmetadata = resolve;
-    });
+    await new Promise((resolve) => video.onloadedmetadata = resolve);
 
     if (video.seekable.start(0) === video.seekable.end(0)) {
         // Fallback: Download entire video if not seekable
         const response = await fetch(video.src);
         const blob = await response.blob();
         video.src = URL.createObjectURL(blob);
-
-        await new Promise((resolve) => {
-            video.onloadedmetadata = resolve;
-        });
+        await new Promise((resolve) => video.onloadedmetadata = resolve);
     }
 
     const duration = video.duration;
 
-    // Build an array of sample times based on num_frames or fps
-    let sampleTimes = [];
-    if (num_frames == null && fps == null) {
-        throw new Error("Either num_frames or fps must be provided.");
-    }
-
     let count, step;
     if (num_frames != null) {
         count = num_frames;
@@ -100,11 +98,13 @@ export async function load_video(url, { num_frames = null, fps = null } = {}) {
         count = Math.floor(duration / step);
     }
 
-    for (let i = 0; i < count; i++) {
+    // Build an array of sample times based on num_frames or fps
+    let sampleTimes = [];
+    for (let i = 0; i < count; ++i) {
         sampleTimes.push(num_frames === 1 ? duration / 2 : i * step);
     }
 
-    const canvas = document.createElement('canvas');
+    const canvas = document.createElement("canvas");
     canvas.width = video.videoWidth;
     canvas.height = video.videoHeight;
     const ctx = canvas.getContext("2d", { willReadFrequently: true });