broadcast: mirrored prop (#592)

* broadcast: mirrored prop

* address comments

* update

* remove unused import

Author: gioelecerati

packages/core-web/src/broadcast.ts

@@ -13,6 +13,7 @@ import { isPictureInPictureSupported } from "./media/controls";
 import { getRTCPeerConnectionConstructor } from "./webrtc/shared";
 import {
   attachMediaStreamToPeerConnection,
+  createMirroredVideoTrack,
   createNewWHIP,
   getDisplayMedia,
   getDisplayMediaExists,
@@ -146,6 +147,14 @@ export type InitialBroadcastProps = {
    * Set to true to disable ICE gathering. This is useful for testing purposes.
    */
   noIceGathering?: boolean;
+
+  /**
+   * Whether the video stream should be mirrored (horizontally flipped).
+   *
+   * Set to true to broadcast a mirrored view.
+   * Defaults to `false`.
+   */
+  mirrored?: boolean;
 };
 
 export type BroadcastAriaText = {
@@ -326,6 +335,7 @@ export const createBroadcastStore = ({
             ingestUrl: ingestUrl ?? null,
             video: initialProps?.video ?? true,
             noIceGathering: initialProps?.noIceGathering ?? false,
+            mirrored: initialProps?.mirrored ?? false,
           },
 
           __device: device,
@@ -638,7 +648,11 @@ export const addBroadcastEventListeners = (
   }
 
   // add effects
-  const removeEffectsFromStore = addEffectsToStore(element, store, mediaStore);
+  const { destroy: destroyEffects } = addEffectsToStore(
+    element,
+    store,
+    mediaStore,
+  );
 
   const removeHydrationListener = store.persist.onFinishHydration(
     ({ mediaDeviceIds, audio, video }) => {
@@ -656,7 +670,7 @@ export const addBroadcastEventListeners = (
 
       mediaDevices?.removeEventListener?.("devicechange", onDeviceChange);
 
-      removeEffectsFromStore?.();
+      destroyEffects?.();
 
       element?.removeAttribute?.(MEDIA_BROADCAST_INITIALIZED_ATTRIBUTE);
     },
@@ -818,6 +832,7 @@ const addEffectsToStore = (
       requestedVideoDeviceId: state.__controls.requestedVideoInputDeviceId,
       initialAudioConfig: state.__initialProps.audio,
       initialVideoConfig: state.__initialProps.video,
+      mirrored: state.__initialProps.mirrored,
       previousMediaStream: state.mediaStream,
     }),
     async ({
@@ -830,6 +845,7 @@ const addEffectsToStore = (
       previousMediaStream,
       initialAudioConfig,
       initialVideoConfig,
+      mirrored,
     }) => {
       try {
         if (!mounted || !hydrated) {
@@ -871,7 +887,6 @@ const addEffectsToStore = (
                   ? {
                       ...(audioConstraints ? audioConstraints : {}),
                       deviceId: {
-                        // we pass ideal here, so we don't get overconstrained errors
                         ideal: requestedAudioDeviceId,
                       },
                     }
@@ -887,13 +902,14 @@ const addEffectsToStore = (
                   ? {
                       ...(videoConstraints ? videoConstraints : {}),
                       deviceId: {
-                        // we pass ideal here, so we don't get overconstrained errors
                         ideal: requestedVideoDeviceId,
                       },
+                      ...(mirrored ? { facingMode: "user" } : {}),
                     }
                   : video
                     ? {
                         ...(videoConstraints ? videoConstraints : {}),
+                        ...(mirrored ? { facingMode: "user" } : {}),
                       }
                     : false,
             }));
@@ -934,11 +950,38 @@ const addEffectsToStore = (
             allAudioTracks?.[0] ??
             previousMediaStream?.getAudioTracks?.()?.[0] ??
             null;
-          const mergedVideoTrack =
+
+          let mergedVideoTrack =
             allVideoTracks?.[0] ??
             previousMediaStream?.getVideoTracks?.()?.[0] ??
             null;
 
+          if (
+            mergedVideoTrack &&
+            mirrored &&
+            requestedVideoDeviceId !== "screen"
+          ) {
+            try {
+              const videoSettings = mergedVideoTrack.getSettings();
+              const isFrontFacing =
+                videoSettings.facingMode === "user" ||
+                !videoSettings.facingMode;
+
+              if (isFrontFacing) {
+                element.classList.add("livepeer-mirrored-video");
+                mergedVideoTrack = createMirroredVideoTrack(mergedVideoTrack);
+              } else {
+                element.classList.remove("livepeer-mirrored-video");
+              }
+            } catch (err) {
+              warn(
+                `Failed to apply video mirroring: ${(err as Error).message}`,
+              );
+            }
+          } else {
+            element.classList.remove("livepeer-mirrored-video");
+          }
+
           if (mergedAudioTrack) mergedMediaStream.addTrack(mergedAudioTrack);
           if (mergedVideoTrack) mergedMediaStream.addTrack(mergedVideoTrack);
 
@@ -1123,20 +1166,22 @@ const addEffectsToStore = (
     },
   );
 
-  return () => {
-    destroyAudioVideoEnabled?.();
-    destroyErrorCount?.();
-    destroyMapDeviceListToFriendly?.();
-    destroyMediaStream?.();
-    destroyMediaSyncError?.();
-    destroyMediaSyncMounted?.();
-    destroyPeerConnectionAndMediaStream?.();
-    destroyPictureInPictureSupportedMonitor?.();
-    destroyRequestUserMedia?.();
-    destroyUpdateDeviceList?.();
-    destroyWhip?.();
-
-    cleanupWhip?.();
-    cleanupMediaStream?.();
+  return {
+    destroy: () => {
+      destroyAudioVideoEnabled?.();
+      destroyErrorCount?.();
+      destroyMapDeviceListToFriendly?.();
+      destroyMediaStream?.();
+      destroyMediaSyncError?.();
+      destroyMediaSyncMounted?.();
+      destroyPeerConnectionAndMediaStream?.();
+      destroyPictureInPictureSupportedMonitor?.();
+      destroyRequestUserMedia?.();
+      destroyUpdateDeviceList?.();
+      destroyWhip?.();
+
+      cleanupWhip?.();
+      cleanupMediaStream?.();
+    },
   };
 };

packages/core-web/src/webrtc/whip.ts

@@ -6,6 +6,8 @@ import {
   negotiateConnectionWithClientOffer,
 } from "./shared";
 
+const STANDARD_FPS = 30;
+
 export const VIDEO_WEBRTC_INITIALIZED_ATTRIBUTE =
   "data-livepeer-video-whip-initialized";
 
@@ -265,3 +267,75 @@ export const getDisplayMedia = (options?: DisplayMediaStreamOptions) => {
 
   return navigator.mediaDevices.getDisplayMedia(options);
 };
+
+/**
+ * Creates a mirrored version of a video track using a canvas element.
+ * This function ensures the stream sent to the server is mirrored horizontally.
+ */
+export const createMirroredVideoTrack = (
+  originalTrack: MediaStreamTrack,
+): MediaStreamTrack => {
+  if (originalTrack.kind !== "video") {
+    warn("Cannot mirror non-video track");
+    return originalTrack;
+  }
+
+  try {
+    const settings = originalTrack.getSettings();
+    const width = settings.width || 640;
+    const height = settings.height || 480;
+
+    const canvas = document.createElement("canvas");
+    canvas.width = width;
+    canvas.height = height;
+
+    const ctx = canvas.getContext("2d");
+    if (!ctx) {
+      warn("Could not get canvas context for mirroring video");
+      return originalTrack;
+    }
+
+    const video = document.createElement("video");
+    video.srcObject = new MediaStream([originalTrack]);
+    video.autoplay = true;
+    video.muted = true;
+
+    video.onloadedmetadata = () => {
+      video
+        .play()
+        .catch((e) =>
+          warn(`Failed to play video in mirroring process: ${e.message}`),
+        );
+    };
+
+    const drawFrame = () => {
+      if (video.readyState >= 2) {
+        ctx.clearRect(0, 0, width, height);
+
+        ctx.save();
+        ctx.scale(-1, 1);
+        ctx.drawImage(video, -width, 0, width, height);
+        ctx.restore();
+      }
+
+      requestAnimationFrame(drawFrame);
+    };
+
+    drawFrame();
+
+    const mirroredStream = canvas.captureStream(STANDARD_FPS);
+
+    const mirroredTrack = mirroredStream.getVideoTracks()[0];
+
+    originalTrack.addEventListener("ended", () => {
+      mirroredTrack.stop();
+      video.pause();
+      video.srcObject = null;
+    });
+
+    return mirroredTrack;
+  } catch (err) {
+    warn(`Error creating mirrored track: ${(err as Error).message}`);
+    return originalTrack;
+  }
+};

packages/core/src/version.ts

@@ -1,5 +1,5 @@
-const core = "@livepeer/core@3.2.5";
-const react = "@livepeer/react@4.2.5";
+const core = "@livepeer/core@3.2.8";
+const react = "@livepeer/react@4.2.10";
 
 export const version = {
   core,