User Timestamp Parsing

examples/demo/demo.ts

@@ -21,6 +21,7 @@
   ParticipantEvent,
   RemoteParticipant,
   RemoteTrackPublication,
+  RemoteVideoTrack,
   Room,
   RoomEvent,
   ScreenSharePresets,
@@ -34,10 +35,12 @@
   isLocalTrack,
   isRemoteParticipant,
   isRemoteTrack,
+  isVideoTrack,
   setLogLevel,
   supportsAV1,
   supportsVP9,
 } from '../../src/index';
+import { TrackEvent } from '../../src/room/events';
 import { isSVCCodec, sleep, supportsH265 } from '../../src/room/utils';
 
 setLogLevel(LogLevel.debug);
@@ -238,6 +241,43 @@
         appendLog('subscribed to track', pub.trackSid, participant.identity);
         renderParticipant(participant);
         renderScreenShare(room);
+
+        // Display the user timestamp that matches the frame currently on screen.
+        // Publish & Subscribe timestamps update every frame; latency updates at ~2 Hz
+        // so it's readable (matching the Rust subscriber's approach).
+        if (track instanceof RemoteVideoTrack) {
+          let lastLatencyUpdate = 0;
+          let cachedUserTimestampUs: number | undefined;
+          let cachedLatencyStr = '';
+
+          track.on(TrackEvent.TimeSyncUpdate, ({ rtpTimestamp }) => {
+            const timestampUs = track.lookupUserTimestamp(rtpTimestamp);
+            if (timestampUs !== undefined) {
+              cachedUserTimestampUs = timestampUs;
+            }
+            if (cachedUserTimestampUs === undefined) return;
+
+            const now = Date.now();
+
+            if (now - lastLatencyUpdate >= 500) {
+              const nowUs = now * 1000;
+              const latencyMs = (nowUs - cachedUserTimestampUs) / 1000;
+              cachedLatencyStr = `${latencyMs.toFixed(1)}ms`;
+              lastLatencyUpdate = now;
+            }
+
+            const container = getParticipantsAreaElement();
+            const tsElm = container.querySelector(`#user-ts-${participant.identity}`);
+            if (tsElm) {
+              const pubStr = new Date(cachedUserTimestampUs / 1000).toISOString().substring(11, 23);
+              const subStr = new Date(now).toISOString().substring(11, 23);
+              tsElm.innerHTML =
+                `Publish:&nbsp;&nbsp;&nbsp;${pubStr}<br>` +
+                `Subscribe:&nbsp;${subStr}<br>` +
+                `Latency:&nbsp;&nbsp;&nbsp;${cachedLatencyStr}`;
+            }
+          });
+        }
       })
       .on(RoomEvent.TrackUnsubscribed, (_, pub, participant) => {
         appendLog('unsubscribed from track', pub.trackSid);
@@ -811,6 +851,8 @@
           <span id="e2ee-${identity}" class="e2ee-on"></span>
         </div>
       </div>
+      <div id="user-ts-${identity}" class="user-ts-overlay">
+      </div>
       ${
         !isLocalParticipant(participant)
           ? `<div class="volume-control">

examples/demo/styles.css

@@ -164,3 +164,17 @@
   position: absolute;
   z-index: 4;
 }
+
+.participant .user-ts-overlay {
+  position: absolute;
+  bottom: 28px;
+  left: 0;
+  z-index: 5;
+  font-family: monospace;
+  font-size: 0.65em;
+  color: #eee;
+  background: rgba(0, 0, 0, 0.5);
+  padding: 3px 6px;
+  line-height: 1.4;
+  border-radius: 0 3px 0 0;
+}

src/e2ee/E2eeManager.ts

@@ -8,6 +8,7 @@
 import { DeviceUnsupportedError } from '../room/errors';
 import { EngineEvent, ParticipantEvent, RoomEvent } from '../room/events';
 import type RemoteTrack from '../room/track/RemoteTrack';
+import RemoteVideoTrack from '../room/track/RemoteVideoTrack';
 import type { Track } from '../room/track/Track';
 import type { VideoCodec } from '../room/track/options';
 import { mimeTypeToVideoCodecString } from '../room/track/utils';
@@ -221,6 +222,9 @@
           encryptFuture.resolve(data as EncryptDataResponseMessage['data']);
         }
         break;
+      case 'userTimestamp':
+        this.handleUserTimestamp(data.trackId, data.participantIdentity, data.timestampUs, data.rtpTimestamp);
+        break;
       default:
         break;
     }
@@ -231,6 +235,22 @@
     this.emit(EncryptionEvent.EncryptionError, ev.error, undefined);
   };
 
+  private handleUserTimestamp(trackId: string, participantIdentity: string, timestampUs: number, rtpTimestamp?: number) {
+    if (!this.room) {
+      return;
+    }
+    const participant = this.room.getParticipantByIdentity(participantIdentity);
+    if (!participant) {
+      return;
+    }
+    for (const pub of participant.trackPublications.values()) {
+      if (pub.track && pub.track.mediaStreamID === trackId && pub.track instanceof RemoteVideoTrack) {
+        pub.track.setUserTimestamp(timestampUs, rtpTimestamp);
+        return;
+      }
+    }
+  }
+
   public setupEngine(engine: RTCEngine) {
     engine.on(EngineEvent.RTPVideoMapUpdate, (rtpMap) => {
       this.postRTPMap(rtpMap);

src/e2ee/types.ts

@@ -149,6 +149,16 @@ export interface EncryptDataResponseMessage extends BaseMessage {
   };
 }
 
+export interface UserTimestampMessage extends BaseMessage {
+  kind: 'userTimestamp';
+  data: {
+    trackId: string;
+    participantIdentity: string;
+    timestampUs: number;
+    rtpTimestamp?: number;
+  };
+}
+
 export type E2EEWorkerMessage =
   | InitMessage
   | SetKeyMessage
@@ -165,7 +175,8 @@ export type E2EEWorkerMessage =
   | DecryptDataRequestMessage
   | DecryptDataResponseMessage
   | EncryptDataRequestMessage
-  | EncryptDataResponseMessage;
+  | EncryptDataResponseMessage
+  | UserTimestampMessage;
 
 export type KeySet = { material: CryptoKey; encryptionKey: CryptoKey };
 

src/e2ee/worker/FrameCryptor.ts

@@ -9,6 +9,8 @@ import { CryptorError, CryptorErrorReason } from '../errors';
 import { type CryptorCallbacks, CryptorEvent } from '../events';
 import type { DecodeRatchetOptions, KeyProviderOptions, KeySet, RatchetResult } from '../types';
 import { deriveKeys, isVideoFrame, needsRbspUnescaping, parseRbsp, writeRbsp } from '../utils';
+import { stripUserTimestampFromEncodedFrame } from '../../user_timestamp/UserTimestampTransformer';
+import type { UserTimestampMessage } from '../types';
 import type { ParticipantKeyHandler } from './ParticipantKeyHandler';
 import { processNALUsForEncryption } from './naluUtils';
 import { identifySifPayload } from './sifPayload';
@@ -412,6 +414,31 @@ export class FrameCryptor extends BaseFrameCryptor {
     encodedFrame: RTCEncodedVideoFrame | RTCEncodedAudioFrame,
     controller: TransformStreamDefaultController,
   ) {
+    // Always attempt to strip LKTS user timestamp trailer before any e2ee
+    // processing. On the send side, the trailer is appended *after* encryption,
+    // so it must be removed *before* decryption.
+    if (isVideoFrame(encodedFrame) && encodedFrame.data.byteLength > 0) {
+      try {
+        const userTsResult = stripUserTimestampFromEncodedFrame(
+          encodedFrame as RTCEncodedVideoFrame,
+        );
+        if (userTsResult !== undefined && this.trackId && this.participantIdentity) {
+          const msg: UserTimestampMessage = {
+            kind: 'userTimestamp',
+            data: {
+              trackId: this.trackId,
+              participantIdentity: this.participantIdentity,
+              timestampUs: userTsResult.timestampUs,
+              rtpTimestamp: userTsResult.rtpTimestamp,
+            },
+          };
+          postMessage(msg);
+        }
+      } catch {
+        // Best-effort: never break media pipeline if timestamp parsing fails.
+      }
+    }
+
     if (
       !this.isEnabled() ||
       // skip for decryption for empty dtx frames

src/index.ts

@@ -151,3 +151,12 @@ export type {
 };
 
 export { LocalTrackRecorder } from './room/track/record';
+
+export {
+  type UserTimestampInfo,
+  type UserTimestampWithRtp,
+  USER_TS_MAGIC,
+  USER_TS_TRAILER_SIZE,
+  extractUserTimestampTrailer,
+  stripUserTimestampFromEncodedFrame,
+} from './user_timestamp';

src/room/RTCEngine.ts

@@ -79,6 +79,7 @@ import type { TrackPublishOptions, VideoCodec } from './track/options';
 import { getTrackPublicationInfo } from './track/utils';
 import type { LoggerOptions } from './types';
 import {
+  isChromiumBased,
   isVideoCodec,
   isVideoTrack,
   isWeb,
@@ -655,9 +656,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
   private makeRTCConfiguration(serverResponse: JoinResponse | ReconnectResponse): RTCConfiguration {
     const rtcConfig = { ...this.rtcConfig };
 
-    if (this.signalOpts?.e2eeEnabled) {
-      this.log.debug('E2EE - setting up transports with insertable streams', this.logContext);
-      //  this makes sure that no data is sent before the transforms are ready
+    if (this.signalOpts?.e2eeEnabled || isChromiumBased()) {
+      this.log.debug('setting up transports with insertable streams', this.logContext);
       // @ts-ignore
       rtcConfig.encodedInsertableStreams = true;
     }

src/room/Room.ts

@@ -36,13 +36,13 @@
 import { EncryptionEvent } from '../e2ee';
 import { type BaseE2EEManager, E2EEManager } from '../e2ee/E2eeManager';
 import log, { LoggerNames, getLogger } from '../logger';
 import type {
   InternalRoomConnectOptions,
   InternalRoomOptions,
   RoomConnectOptions,
   RoomOptions,
 } from '../options';
 import { getBrowser } from '../utils/browserParser';
 import { BackOffStrategy } from './BackOffStrategy';
 import DeviceManager from './DeviceManager';
 import RTCEngine from './RTCEngine';
@@ -74,7 +74,12 @@
 import LocalVideoTrack from './track/LocalVideoTrack';
 import type RemoteTrack from './track/RemoteTrack';
 import RemoteTrackPublication from './track/RemoteTrackPublication';
+import RemoteVideoTrack from './track/RemoteVideoTrack';
 import { Track } from './track/Track';
+import { stripUserTimestampFromEncodedFrame } from '../user_timestamp/UserTimestampTransformer';
+//@ts-ignore
+import UserTimestampWorker from '../user_timestamp/userTimestamp.worker?worker';
+import { isScriptTransformSupported } from '../e2ee/utils';
 import type { TrackPublication } from './track/TrackPublication';
 import type { TrackProcessor } from './track/processor/types';
 import type { AdaptiveStreamSettings } from './track/types';
@@ -93,6 +98,7 @@
   getDisconnectReasonFromConnectionError,
   getEmptyAudioStreamTrack,
   isBrowserSupported,
+  isChromiumBased,
   isCloud,
   isLocalAudioTrack,
   isLocalParticipant,
@@ -2177,6 +2183,13 @@
             track.on(TrackEvent.VideoPlaybackFailed, this.handleVideoPlaybackFailed);
             track.on(TrackEvent.VideoPlaybackStarted, this.handleVideoPlaybackStarted);
           }
+          // When e2ee is NOT enabled, set up a standalone insertable-streams
+          // transform to extract LKTS user timestamp trailers from inbound
+          // video frames. When e2ee IS enabled, the FrameCryptor worker
+          // handles this before decryption.
+          if (!this.hasE2EESetup && track instanceof RemoteVideoTrack && track.receiver) {
+            this.setupUserTimestampTransform(track);
+          }
           this.emit(RoomEvent.TrackSubscribed, track, publication, participant);
         },
       )
@@ -2591,6 +2604,69 @@
     }
   }
 
+  /**
+   * Sets up an insertable-streams transform on a remote video track's receiver
+   * to strip LKTS user timestamp trailers. Used only when e2ee is NOT enabled
+   * (when e2ee is enabled, the FrameCryptor worker handles this).
+   *
+   * Uses RTCRtpScriptTransform on browsers that support it (non-Chromium),
+   * and falls back to createEncodedStreams (legacy insertable streams) on
+   * Chromium where encodedInsertableStreams is enabled on the PeerConnection.
+   */
+  private setupUserTimestampTransform(track: RemoteVideoTrack) {
+    const receiver = track.receiver;
+    if (!receiver) {
+      return;
+    }
+
+    try {
+      if (isScriptTransformSupported() && !isChromiumBased()) {
+        const worker = new UserTimestampWorker();
+        worker.onmessage = (ev: MessageEvent) => {
+          if (ev.data?.kind === 'userTimestamp' && typeof ev.data.timestampUs === 'number') {
+            track.setUserTimestamp(ev.data.timestampUs, ev.data.rtpTimestamp);
+          }
+        };
+        // @ts-ignore
+        receiver.transform = new RTCRtpScriptTransform(worker, { kind: 'decode' });
+        return;
+      }
+
+      // @ts-ignore
+      if (typeof receiver.createEncodedStreams === 'function') {
+        // @ts-ignore
+        const { readable, writable } = receiver.createEncodedStreams();
+
+        const transformStream = new TransformStream<RTCEncodedVideoFrame, RTCEncodedVideoFrame>({
+          transform: (encodedFrame, controller) => {
+            try {
+              const result = stripUserTimestampFromEncodedFrame(encodedFrame);
+              if (result !== undefined) {
+                track.setUserTimestamp(result.timestampUs, result.rtpTimestamp);
+              }
+            } catch {
+              // Best-effort: never break the media pipeline if parsing fails.
+            }
+            controller.enqueue(encodedFrame);
+          },
+        });
+
+        readable
+          .pipeThrough(transformStream)
+          .pipeTo(writable)
+          .catch(() => {});
+        return;
+      }
+
+      this.log.debug(
+        'user timestamp transform: no supported API available',
+        this.logContext,
+      );
+    } catch {
+      // If anything goes wrong (e.g., unsupported environment), just skip.
+    }
+  }
+
   // /** @internal */
   emit<E extends keyof RoomEventCallbacks>(
     event: E,