WebRTC improvements (#4603)

Author: jpggvilaca

ui/src/features/inference/stream/stream.tsx

@@ -12,31 +12,26 @@ const useSetTargetSizeBasedOnVideo = (
 ) => {
     useEffect(() => {
         const video = videoRef.current;
+        if (!video) return;
 
-        const onsize = video?.addEventListener('loadedmetadata', (event) => {
-            const target = event.currentTarget as HTMLVideoElement;
-
-            if (target.videoWidth && target.videoHeight) {
-                setSize({ width: target.videoWidth, height: target.videoHeight });
+        const onLoaded = () => {
+            if (video.videoWidth && video.videoHeight) {
+                setSize({ width: video.videoWidth, height: video.videoHeight });
             }
-        });
-
-        const onresize = video?.addEventListener('resize', (event) => {
-            const target = event.currentTarget as HTMLVideoElement;
+        };
 
-            if (target.videoWidth && target.videoHeight) {
-                setSize({ width: target.videoWidth, height: target.videoHeight });
+        const resizeObserver = new ResizeObserver(() => {
+            if (video.videoWidth && video.videoHeight) {
+                setSize({ width: video.videoWidth, height: video.videoHeight });
             }
         });
 
-        return () => {
-            if (onsize) {
-                video?.removeEventListener('loadedmetadata', onsize);
-            }
+        video.addEventListener('loadedmetadata', onLoaded);
+        resizeObserver.observe(video);
 
-            if (onresize) {
-                video?.removeEventListener('resize', onresize);
-            }
+        return () => {
+            video.removeEventListener('loadedmetadata', onLoaded);
+            resizeObserver.disconnect();
         };
     }, [setSize, videoRef]);
 };
@@ -56,7 +51,7 @@ const useStreamToVideo = () => {
         }
 
         const receivers = peerConnection.getReceivers() ?? [];
-        const stream = new MediaStream(receivers.map((receiver) => receiver.track));
+        const stream = new MediaStream(receivers.map((receiver) => receiver.track).filter(Boolean));
 
         if (videoOutput && videoOutput.srcObject !== stream) {
             videoOutput.srcObject = stream;

ui/src/features/inference/stream/web-rtc-connection.ts

@@ -1,6 +1,8 @@
 // Copyright (C) 2025 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 
+import { v4 as uuid } from 'uuid';
+
 import { fetchClient } from '../../../api/client';
 
 export type WebRTCConnectionStatus = 'idle' | 'connecting' | 'connected' | 'disconnected' | 'failed';
@@ -37,8 +39,7 @@ export class WebRTCConnection {
     private timeoutId?: ReturnType<typeof setTimeout>;
 
     constructor() {
-        // TODO: replace with uuid
-        this.webrtcId = Math.random().toString(36).substring(7);
+        this.webrtcId = uuid();
     }
 
     public getStatus(): WebRTCConnectionStatus {
@@ -80,9 +81,11 @@ export class WebRTCConnection {
         } catch (err) {
             clearTimeout(this.timeoutId);
             console.error('Error setting up WebRTC:', err);
+
             this.emit({ type: 'error', error: err as Error });
             this.updateStatus('failed');
-            this.stop();
+
+            await this.stop();
         }
 
         if (this.peerConnection) {
@@ -109,21 +112,26 @@ export class WebRTCConnection {
     }
 
     private async waitForIceGathering(): Promise<void> {
-        await new Promise<void>((resolve) => {
-            if (!this.peerConnection || this.peerConnection.iceGatheringState === 'complete') {
-                resolve();
-                return;
-            }
-
-            const checkState = () => {
-                if (this.peerConnection && this.peerConnection.iceGatheringState === 'complete') {
-                    this.peerConnection.removeEventListener('icegatheringstatechange', checkState);
+        await Promise.race([
+            new Promise<void>((resolve) => {
+                if (!this.peerConnection || this.peerConnection.iceGatheringState === 'complete') {
                     resolve();
+                    return;
                 }
-            };
 
-            this.peerConnection?.addEventListener('icegatheringstatechange', checkState);
-        });
+                const checkState = () => {
+                    if (this.peerConnection && this.peerConnection.iceGatheringState === 'complete') {
+                        this.peerConnection.removeEventListener('icegatheringstatechange', checkState);
+                        resolve();
+                    }
+                };
+
+                this.peerConnection?.addEventListener('icegatheringstatechange', checkState);
+            }),
+            new Promise<void>((_, reject) =>
+                setTimeout(() => reject(new Error('ICE gathering timed out')), CONNECTION_TIMEOUT)
+            ),
+        ]);
     }
 
     private async sendOffer(): Promise<SessionData | undefined> {

ui/tests/fixtures.ts

@@ -21,8 +21,10 @@ const test = testBase.extend<Fixtures>({
                 return response(200).json([]);
             }),
             http.post('/api/webrtc/offer', ({ response }) => {
-                // Schema is empty, so we return an empty object
-                return response(200).json({} as never);
+                return response(200).json({
+                    type: 'answer',
+                    sdp: 'v=0\r\no=- 0 0 IN IP4 127.0.0.1\r\n',
+                } as never);
             }),
             http.post('/api/input_hook', ({ response }) => {
                 // Schema is empty, so we return an empty object