Expose track-level upstream & downstream stats (#103)

Sample app is updated to surface bandwidth stats and time to first render.

Also fixed dynacast for FireFox.

Author: davidzhao

example/sample.ts

@@ -1,9 +1,11 @@
 import {
   DataPacket_Kind, LocalParticipant,
+  LocalTrack,
   MediaDeviceFailure,
   Participant,
   ParticipantEvent,
-  RemoteParticipant, Room, RoomConnectOptions, RoomEvent,
+  RemoteAudioTrack,
+  RemoteParticipant, RemoteVideoTrack, Room, RoomConnectOptions, RoomEvent,
   RoomOptions, RoomState, setLogLevel, Track, TrackPublication,
   VideoCaptureOptions, VideoPresets,
 } from '../src/index';
@@ -16,8 +18,8 @@ const state = {
   encoder: new TextEncoder(),
   decoder: new TextDecoder(),
   defaultDevices: new Map<MediaDeviceKind, string>(),
+  bitrateInterval: undefined as any,
 };
-
 let currentRoom: Room | undefined;
 
 // handles actions from the HTML
@@ -56,6 +58,8 @@ const appActions = {
       await room.localParticipant.enableCameraAndMicrophone();
       updateButtonsForPublishState();
     }
+
+    state.bitrateInterval = setInterval(renderBitrate, 1000);
   },
 
   connectToRoom: async (
@@ -202,6 +206,9 @@ const appActions = {
     if (currentRoom) {
       currentRoom.disconnect();
     }
+    if (state.bitrateInterval) {
+      clearInterval(state.bitrateInterval);
+    }
   },
 
   disconnectSignal: () => {
@@ -342,7 +349,11 @@ function renderParticipant(participant: Participant, remove: boolean = false) {
       <div class="info-bar">
         <div id="name-${participant.sid}" class="name">
         </div>
-        <div id="size-${participant.sid}" class="size">
+        <div style="text-align: center;">
+          <span id="size-${participant.sid}" class="size">
+          </span>
+          <span id="bitrate-${participant.sid}" class="bitrate">
+          </span>
         </div>
         <div class="right">
           <span id="signal-${participant.sid}"></span>
@@ -387,11 +398,18 @@ function renderParticipant(participant: Participant, remove: boolean = false) {
 
   const cameraEnabled = cameraPub && cameraPub.isSubscribed && !cameraPub.isMuted;
   if (cameraEnabled) {
-    cameraPub?.videoTrack?.attach(videoElm);
     if (participant instanceof LocalParticipant) {
       // flip
       videoElm.style.transform = 'scale(-1, 1)';
+    } else if (!cameraPub?.videoTrack?.attachedElements.includes(videoElm)) {
+      const startTime = Date.now();
+      // measure time to render
+      videoElm.addEventListener('loadeddata', () => {
+        const elapsed = Date.now() - startTime;
+        appendLog(`RemoteVideoTrack ${cameraPub?.trackSid} rendered in ${elapsed}ms`);
+      });
     }
+    cameraPub?.videoTrack?.attach(videoElm);
   } else if (cameraPub?.videoTrack) {
     // detach manually whenever possible
     cameraPub.videoTrack?.detach(videoElm);
@@ -465,6 +483,32 @@ function renderScreenShare() {
   }
 }
 
+function renderBitrate() {
+  if (!currentRoom || currentRoom.state !== RoomState.Connected) {
+    return;
+  }
+  const participants: Participant[] = [...currentRoom.participants.values()];
+  participants.push(currentRoom.localParticipant);
+
+  for (const p of participants) {
+    const elm = $(`bitrate-${p.sid}`);
+    let totalBitrate = 0;
+    for (const t of p.tracks.values()) {
+      if (t.track instanceof RemoteAudioTrack || t.track instanceof RemoteVideoTrack
+        || t.track instanceof LocalTrack) {
+        totalBitrate += t.track.currentBitrate;
+      }
+    }
+    let displayText = '';
+    if (totalBitrate > 0) {
+      displayText = `${Math.round(totalBitrate / 1024).toLocaleString()} kbps`;
+    }
+    if (elm) {
+      elm.innerHTML = displayText;
+    }
+  }
+}
+
 function updateVideoSize(element: HTMLVideoElement, target: HTMLElement) {
   target.innerHTML = `(${element.videoWidth}x${element.videoHeight})`;
 }

src/room/participant/LocalParticipant.ts

@@ -416,6 +416,8 @@ export default class LocalParticipant extends Participant {
     const disableLayerPause = this.roomOptions?.expDisableLayerPause ?? false;
     if (track instanceof LocalVideoTrack && !disableLayerPause) {
       track.startMonitor(this.engine.client);
+    } else if (track instanceof LocalAudioTrack) {
+      track.startMonitor();
     }
 
     if (opts.videoCodec) {

src/room/participant/RemoteParticipant.ts

@@ -127,6 +127,8 @@ export default class RemoteParticipant extends Participant {
     track.source = publication.source;
     // keep publication's muted status
     track.isMuted = publication.isMuted;
+    track.receiver = receiver;
+    track.startMonitor();
 
     // when media track is ended, fire the event
     mediaTrack.onended = () => {

src/room/participant/publishUtils.test.ts

@@ -41,7 +41,7 @@ describe('computeVideoEncodings', () => {
     const encodings = computeVideoEncodings(false, 640, 480, {
       simulcast: false,
     });
-    expect(encodings).toBeUndefined();
+    expect(encodings).toEqual([{}]);
   });
 
   it('respects client defined bitrate', () => {

src/room/participant/publishUtils.ts

@@ -60,17 +60,17 @@ export function computeVideoEncodings(
   width?: number,
   height?: number,
   options?: TrackPublishOptions,
-): RTCRtpEncodingParameters[] | undefined {
+): RTCRtpEncodingParameters[] {
   let videoEncoding: VideoEncoding | undefined = options?.videoEncoding;
   if (isScreenShare) {
     videoEncoding = options?.screenShareEncoding;
   }
   const useSimulcast = !isScreenShare && options?.simulcast;
 
   if ((!videoEncoding && !useSimulcast) || !width || !height) {
-    // don't set encoding when we are not simulcasting and user isn't restricting
-    // encoding parameters
-    return;
+    // when we aren't simulcasting, will need to return a single encoding without
+    // capping bandwidth. we always require a encoding for dynacast
+    return [{}];
   }
 
   if (!videoEncoding) {

src/room/stats.ts

@@ -5,6 +5,9 @@ interface SenderStats {
   /** number of packets sent */
   packetsSent?: number;
 
+  /** number of bytes sent */
+  bytesSent?: number;
+
   /** jitter as perceived by remote */
   jitter?: number;
 
@@ -16,6 +19,8 @@ interface SenderStats {
 
   /** ID of the outbound stream */
   streamId?: string;
+
+  timestamp: number;
 }
 
 export interface AudioSenderStats extends SenderStats {
@@ -45,8 +50,6 @@ export interface VideoSenderStats extends SenderStats {
   qualityLimitationResolutionChanges: number;
 
   retransmittedPacketsSent: number;
-
-  timestamp: number;
 }
 
 interface ReceiverStats {
@@ -58,7 +61,29 @@ interface ReceiverStats {
   /** number of packets sent */
   packetsReceived?: number;
 
+  bytesReceived?: number;
+
   streamId?: string;
+
+  jitter?: number;
+
+  timestamp: number;
+}
+
+export interface AudioReceiverStats extends ReceiverStats {
+  type: 'audio';
+
+  concealedSamples?: number;
+
+  concealmentEvents?: number;
+
+  silentConcealedSamples?: number;
+
+  silentConcealmentEvents?: number;
+
+  totalAudioEnergy?: number;
+
+  totalSamplesDuration?: number;
 }
 
 export interface VideoReceiverStats extends ReceiverStats {
@@ -70,13 +95,36 @@ export interface VideoReceiverStats extends ReceiverStats {
 
   framesReceived: number;
 
-  frameWidth: number;
+  frameWidth?: number;
 
-  frameHeight: number;
+  frameHeight?: number;
 
-  firCount: number;
+  firCount?: number;
 
-  pliCount: number;
+  pliCount?: number;
 
-  nackCount: number;
+  nackCount?: number;
+}
+
+export function computeBitrate<T extends ReceiverStats | SenderStats>(
+  currentStats: T,
+  prevStats?: T,
+): number {
+  if (!prevStats) {
+    return 0;
+  }
+  let bytesNow: number | undefined;
+  let bytesPrev: number | undefined;
+  if ('bytesReceived' in currentStats) {
+    bytesNow = (currentStats as ReceiverStats).bytesReceived;
+    bytesPrev = (prevStats as ReceiverStats).bytesReceived;
+  } else if ('bytesSent' in currentStats) {
+    bytesNow = (currentStats as SenderStats).bytesSent;
+    bytesPrev = (prevStats as SenderStats).bytesSent;
+  }
+  if (bytesNow === undefined || bytesPrev === undefined
+    || currentStats.timestamp === undefined || prevStats.timestamp === undefined) {
+    return 0;
+  }
+  return ((bytesNow - bytesPrev) * 8 * 1000) / (currentStats.timestamp - prevStats.timestamp);
 }

src/room/track/LocalAudioTrack.ts

@@ -1,4 +1,5 @@
 import log from '../../logger';
+import { AudioSenderStats, computeBitrate, monitorFrequency } from '../stats';
 import LocalTrack from './LocalTrack';
 import { AudioCaptureOptions } from './options';
 import { Track } from './Track';
@@ -10,7 +11,7 @@ export default class LocalAudioTrack extends LocalTrack {
   /** @internal */
   stopOnMute: boolean = false;
 
-  /** @internal */
+  private prevStats?: AudioSenderStats;
 
   constructor(
     mediaTrack: MediaStreamTrack,
@@ -59,4 +60,53 @@ export default class LocalAudioTrack extends LocalTrack {
     }
     await this.restart(constraints);
   }
+
+  /* @internal */
+  startMonitor() {
+    setTimeout(() => {
+      this.monitorSender();
+    }, monitorFrequency);
+  }
+
+  private monitorSender = async () => {
+    if (!this.sender) {
+      this._currentBitrate = 0;
+      return;
+    }
+    const stats = await this.getSenderStats();
+
+    if (stats && this.prevStats) {
+      this._currentBitrate = computeBitrate(stats, this.prevStats);
+    }
+
+    this.prevStats = stats;
+    setTimeout(() => {
+      this.monitorSender();
+    }, monitorFrequency);
+  };
+
+  async getSenderStats(): Promise<AudioSenderStats | undefined> {
+    if (!this.sender) {
+      return undefined;
+    }
+
+    const stats = await this.sender.getStats();
+    let audioStats: AudioSenderStats | undefined;
+    stats.forEach((v) => {
+      if (v.type === 'outbound-rtp') {
+        audioStats = {
+          type: 'audio',
+          streamId: v.id,
+          packetsSent: v.packetsSent,
+          packetsLost: v.packetsLost,
+          bytesSent: v.bytesSent,
+          timestamp: v.timestamp,
+          roundTripTime: v.roundTripTime,
+          jitter: v.jitter,
+        };
+      }
+    });
+
+    return audioStats;
+  }
 }

src/room/track/LocalTrack.ts

@@ -10,6 +10,8 @@ export default class LocalTrack extends Track {
 
   protected constraints: MediaTrackConstraints;
 
+  protected _currentBitrate: number = 0;
+
   protected constructor(
     mediaTrack: MediaStreamTrack, kind: Track.Kind, constraints?: MediaTrackConstraints,
   ) {
@@ -37,6 +39,11 @@ export default class LocalTrack extends Track {
     return undefined;
   }
 
+  /** current send bits per second */
+  get currentBitrate(): number {
+    return this._currentBitrate;
+  }
+
   /**
    * @returns DeviceID of the device that is currently being used for this track
    */