feat: Migrate video filter pipeline to MediaPipe-based implementation

packages/react-sdk/src/components/Notification/DegradedPerformanceNotification.tsx

@@ -0,0 +1,47 @@
+import { PropsWithChildren } from 'react';
+import { Placement } from '@floating-ui/react';
+
+import { useCallStateHooks, useI18n } from '@stream-io/video-react-bindings';
+import { Notification } from './Notification';
+import { useBackgroundFilters } from '../BackgroundFilters';
+import { useLowFpsWarning } from '../../hooks/useLowFpsWarning';
+
+export type DegradedPerformanceNotificationProps = {
+  /**
+   * Text message displayed by the notification.
+   */
+  text?: string;
+  placement?: Placement;
+  className?: string;
+};
+
+export const DegradedPerformanceNotification = ({
+  children,
+  text,
+  placement,
+  className,
+}: PropsWithChildren<DegradedPerformanceNotificationProps>) => {
+  const { useCallStatsReport } = useCallStateHooks();
+  const callStatsReport = useCallStatsReport();
+  const { backgroundFilter } = useBackgroundFilters();
+
+  const showLowFpsWarning = useLowFpsWarning(callStatsReport?.publisherStats);
+
+  const { t } = useI18n();
+
+  const message =
+    text ??
+    t(
+      'Background filters performance is degraded. Consider disabling filters for better performance.',
+    );
+  return (
+    <Notification
+      isVisible={showLowFpsWarning && !!backgroundFilter}
+      placement={placement || 'top-start'}
+      message={message}
+      className={className}
+    >
+      {children}
+    </Notification>
+  );
+};

packages/react-sdk/src/components/Notification/Notification.tsx

@@ -1,6 +1,6 @@
 import { PropsWithChildren, ReactNode, useEffect } from 'react';
+import clsx from 'clsx';
 import { Placement } from '@floating-ui/react';
-
 import { useFloatingUIPreset } from '../../hooks';
 
 export type NotificationProps = {
@@ -9,6 +9,7 @@ export type NotificationProps = {
   visibilityTimeout?: number;
   resetIsVisible?: () => void;
   placement?: Placement;
+  className?: string;
   iconClassName?: string | null;
   close?: () => void;
 };
@@ -21,6 +22,7 @@ export const Notification = (props: PropsWithChildren<NotificationProps>) => {
     visibilityTimeout,
     resetIsVisible,
     placement = 'top',
+    className,
     iconClassName = 'str-video__notification__icon',
     close,
   } = props;
@@ -44,7 +46,7 @@ export const Notification = (props: PropsWithChildren<NotificationProps>) => {
     <div ref={refs.setReference}>
       {isVisible && (
         <div
-          className="str-video__notification"
+          className={clsx('str-video__notification', className)}
           ref={refs.setFloating}
           style={{
             position: strategy,

packages/react-sdk/src/components/Notification/index.ts

@@ -2,3 +2,4 @@ export * from './Notification';
 export * from './PermissionNotification';
 export * from './SpeakingWhileMutedNotification';
 export * from './RecordingInProgressNotification';
+export * from './DegradedPerformanceNotification';

packages/react-sdk/src/hooks/useLowFpsWarning.ts

@@ -0,0 +1,54 @@
+import { AggregatedStatsReport } from '@stream-io/video-client';
+import { useEffect, useRef, useState } from 'react';
+
+const ALPHA = 0.2;
+const FPS_WARNING_THRESHOLD_LOWER = 23;
+const FPS_WARNING_THRESHOLD_UPPER = 25;
+const DEFAULT_FPS = 30;
+const DEVIATION_LIMIT = 0.5;
+const OUTLIER_PERSISTENCE = 5;
+
+/**
+ * Monitors FPS and shows a warning when performance stays low.
+ *
+ * Smooths out quick spikes using an EMA, ignores brief outliers,
+ * and uses two thresholds to avoid flickering near the limit.
+ *
+ * @param stats - Aggregated call stats containing FPS data.
+ * @returns True when the smoothed FPS stays below the warning threshold.
+ */
+export function useLowFpsWarning(stats?: AggregatedStatsReport): boolean {
+  const [lowFps, setLowFps] = useState<boolean>(false);
+  const emaRef = useRef<number>(DEFAULT_FPS);
+  const outlierStreakRef = useRef<number>(0);
+
+  const { highestFramesPerSecond, timestamp } = stats ?? {};
+
+  useEffect(() => {
+    if (!highestFramesPerSecond) {
+      emaRef.current = DEFAULT_FPS;
+      outlierStreakRef.current = 0;
+      setLowFps(false);
+      return;
+    }
+
+    const prevEma = emaRef.current;
+    const deviation = Math.abs(highestFramesPerSecond - prevEma) / prevEma;
+
+    const isOutlier =
+      highestFramesPerSecond < prevEma && deviation > DEVIATION_LIMIT;
+    outlierStreakRef.current = isOutlier ? outlierStreakRef.current + 1 : 0;
+    if (isOutlier && outlierStreakRef.current < OUTLIER_PERSISTENCE) return;
+
+    emaRef.current = ALPHA * highestFramesPerSecond + (1 - ALPHA) * prevEma;
+
+    setLowFps((prev) => {
+      if (prev && emaRef.current > FPS_WARNING_THRESHOLD_UPPER) return false;
+      if (!prev && emaRef.current < FPS_WARNING_THRESHOLD_LOWER) return true;
+
+      return prev;
+    });
+  }, [highestFramesPerSecond, timestamp]);
+
+  return lowFps;
+}

packages/react-sdk/src/translations/en.json

@@ -11,6 +11,7 @@
   "Speakers": "Speakers",
   "Video": "Video",
   "You are muted. Unmute to speak.": "You are muted. Unmute to speak.",
+  "Background filters performance is degraded. Consider disabling filters for better performance.": "Background filters performance is degraded. Consider disabling filters for better performance.",
 
   "Live": "Live",
   "Livestream starts soon": "Livestream starts soon",

packages/styling/src/BackgroundFilters/BackgroundFilters-layout.scss

@@ -32,3 +32,11 @@
     height: auto;
   }
 }
+
+.str-video__background-filters__notifications {
+  position: fixed;
+  z-index: 3;
+  left: 20px;
+  right: 20px;
+  top: 60px;
+}

packages/video-filters-web/index.ts

@@ -2,3 +2,6 @@ export * from './src/compatibility';
 export * from './src/createRenderer';
 export { SegmentationLevel } from './src/segmentation';
 export * from './src/tflite';
+export * from './src/mediapipe';
+export * from './src/types';
+export * from './src/VirtualBackground';

packages/video-filters-web/package.json

@@ -19,6 +19,7 @@
   "files": [
     "dist",
     "tf",
+    "mediapipe",
     "src",
     "index.ts",
     "package.json",
@@ -27,12 +28,14 @@
     "CHANGELOG.md"
   ],
   "dependencies": {
+    "@mediapipe/tasks-vision": "^0.10.22-rc.20250304",
     "@stream-io/worker-timer": "^1.2.5",
     "wasm-feature-detect": "^1.8.0"
   },
   "devDependencies": {
     "@rollup/plugin-replace": "^6.0.2",
     "@rollup/plugin-typescript": "^12.1.4",
+    "@types/dom-mediacapture-transform": "^0.1.11",
     "@types/emscripten": "^1.41.2",
     "rimraf": "^6.0.1",
     "rollup": "^4.52.4",

packages/video-filters-web/src/FallbackGenerator.ts

@@ -0,0 +1,81 @@
+/**
+ * Type representing a video track generator that can be either the native
+ * MediaStreamTrackGenerator or the fallback implementation.
+ */
+export interface MediaStreamTrackGenerator extends MediaStreamTrack {
+  readonly writable: WritableStream;
+}
+
+/**
+ * Fallback video processor for browsers that do not support
+ * MediaStreamTrackGenerator.
+ *
+ * Produces a video MediaStreamTrack sourced from a canvas and exposes
+ * a WritableStream<VideoFrame> on track.writable for writing frames.
+ */
+class FallbackGenerator {
+  constructor({
+    kind,
+    signalTarget,
+  }: {
+    kind: 'video';
+    signalTarget?: MediaStreamTrack;
+  }) {
+    if (kind !== 'video') {
+      throw new Error('Only video tracks are supported');
+    }
+
+    const canvas = document.createElement('canvas');
+    const ctx = canvas.getContext('2d', { desynchronized: true });
+    if (!ctx) {
+      throw new Error('Failed to get 2D context from canvas');
+    }
+
+    const mediaStream = canvas.captureStream();
+    const track = mediaStream.getVideoTracks()[0] as MediaStreamVideoTrack & {
+      writable: WritableStream<VideoFrame>;
+    };
+
+    const height = signalTarget?.getSettings().height;
+    const width = signalTarget?.getSettings().width;
+    if (height && width) {
+      canvas.height = height;
+      canvas.width = width;
+    }
+
+    if (!track) {
+      throw new Error('Failed to create canvas track');
+    }
+
+    if (signalTarget) {
+      signalTarget.addEventListener('ended', () => {
+        track.stop();
+      });
+    }
+
+    track.writable = new WritableStream({
+      write: (frame: VideoFrame) => {
+        canvas.width = frame.displayWidth;
+        canvas.height = frame.displayHeight;
+
+        ctx.drawImage(frame, 0, 0, canvas.width, canvas.height);
+        frame.close();
+      },
+      abort: () => {
+        track.stop();
+      },
+      close: () => {
+        track.stop();
+      },
+    });
+
+    return track as MediaStreamTrackGenerator;
+  }
+}
+
+const TrackGenerator =
+  typeof MediaStreamTrackGenerator !== 'undefined'
+    ? MediaStreamTrackGenerator
+    : FallbackGenerator;
+
+export { TrackGenerator };

packages/video-filters-web/src/FallbackProcessor.ts

@@ -0,0 +1,95 @@
+import { WorkerTimer } from '@stream-io/worker-timer';
+
+/**
+ * Type representing a video track processor that can be either the native
+ * MediaStreamTrackProcessor or the fallback implementation.
+ */
+export interface MediaStreamTrackProcessor {
+  readable: ReadableStream;
+}
+
+/**
+ * Fallback video processor for browsers that do not support
+ * MediaStreamTrackProcessor.
+ *
+ * Takes a video track and produces a `ReadableStream<VideoFrame>` by drawing
+ * frames to an `OffscreenCanvas`.
+ */
+class FallbackProcessor implements MediaStreamTrackProcessor {
+  readonly readable: ReadableStream<VideoFrame>;
+  readonly timers: WorkerTimer;
+
+  constructor({ track }: { track: MediaStreamTrack }) {
+    if (!track) throw new Error('MediaStreamTrack is required');
+    if (track.kind !== 'video') {
+      throw new Error('MediaStreamTrack must be video');
+    }
+    let running = true;
+
+    const video = document.createElement('video');
+    video.muted = true;
+    video.playsInline = true;
+    video.crossOrigin = 'anonymous';
+    video.srcObject = new MediaStream([track]);
+
+    const canvas = new OffscreenCanvas(1, 1);
+    const ctx = canvas.getContext('2d');
+
+    if (!ctx) throw new Error('Failed to get 2D context from OffscreenCanvas');
+
+    let timestamp = 0;
+    const frameRate = track.getSettings().frameRate || 30;
+    let frameDuration = 1000 / frameRate;
+
+    const close = () => {
+      video.pause();
+      video.srcObject = null;
+      video.src = '';
+
+      this.timers.destroy();
+    };
+
+    this.timers = new WorkerTimer({ useWorker: true });
+    this.readable = new ReadableStream({
+      start: async () => {
+        await Promise.all([
+          video.play(),
+          new Promise((r) =>
+            video.addEventListener('loadeddata', r, { once: true }),
+          ),
+        ]);
+        frameDuration = 1000 / (track.getSettings().frameRate || 30);
+        timestamp = performance.now();
+      },
+      pull: async (controller) => {
+        if (!running) {
+          controller.close();
+          close();
+          return;
+        }
+        const delta = performance.now() - timestamp;
+        if (delta <= frameDuration) {
+          await new Promise((r: (value?: unknown) => void) =>
+            this.timers.setTimeout(r, frameDuration - delta),
+          );
+        }
+        timestamp = performance.now();
+        canvas.width = video.videoWidth;
+        canvas.height = video.videoHeight;
+        ctx.drawImage(video, 0, 0);
+        controller.enqueue(new VideoFrame(canvas, { timestamp }));
+      },
+      cancel: () => {
+        running = false;
+        close();
+      },
+    });
+  }
+}
+
+const TrackProcessor =
+  typeof MediaStreamTrackProcessor !== 'undefined'
+    ? MediaStreamTrackProcessor
+    : FallbackProcessor;
+
+export { TrackProcessor };