refactor: audio timestamp

Author: yuki-uchida

.vscode/settings.json

@@ -3,6 +3,12 @@
   "[javascript]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
+  "[typescript]": {"editor.defaultFormatter": "esbenp.prettier-vscode"},
+  "[typescriptreact]": {"editor.defaultFormatter": "esbenp.prettier-vscode"},
+  "[javascriptreact]": {"editor.defaultFormatter": "esbenp.prettier-vscode"},
+  "[json]": {"editor.defaultFormatter": "esbenp.prettier-vscode"},
+  "[css]": {"editor.defaultFormatter": "esbenp.prettier-vscode"},
+  "[markdown]": {"editor.defaultFormatter": "esbenp.prettier-vscode"},
   "[rust]": {
     "editor.defaultFormatter": "rust-lang.rust-analyzer",
     "editor.inlayHints.enabled": "on"

README.md

@@ -8,7 +8,6 @@ Being Deployed to Github Pages.
 
 - https://nttcom.github.io/moq-wasm/
 
-
 ## Implementation
 
 Supported version: draft-ietf-moq-transport-10
@@ -108,6 +107,7 @@ cargo run -p moqt-server-sample -- --log <Log Level>
 ### Run moqt-client-sample
 
 ```shell
+cd js && npm install
 make client
 ```
 

js/examples/call/src/media/mediaSubscriber.ts

@@ -22,7 +22,6 @@ interface AudioSubscriptionContext {
   worker: Worker
   writer: WritableStreamDefaultWriter<AudioData>
   stream: MediaStream
-  baseTimestamp: number | null
 }
 
 export class MediaSubscriber {
@@ -50,18 +49,16 @@ export class MediaSubscriber {
         | { type: 'frame'; frame: VideoFrame }
         | { type: 'bitrate'; kbps: number }
         | { type: 'latency'; media: 'video'; ms: number }
-        | { frame: VideoFrame }
-      if ('type' in data && data.type === 'bitrate') {
+      if (data.type === 'bitrate') {
         this.handlers.onRemoteVideoBitrate?.(userId, data.kbps)
         return
       }
-      if ('type' in data && data.type === 'latency') {
+      if (data.type === 'latency') {
         this.handlers.onRemoteVideoLatency?.(userId, data.ms)
         return
       }
-      const frame = 'type' in data ? data.frame : data.frame
       await writer.ready
-      await writer.write(frame)
+      await writer.write(data.frame)
     }
 
     const stream = new MediaStream([generator])
@@ -86,29 +83,24 @@ export class MediaSubscriber {
       userId,
       worker,
       writer,
-      stream: new MediaStream([generator]),
-      baseTimestamp: null
+      stream: new MediaStream([generator])
     }
     worker.onmessage = async (event: MessageEvent) => {
       const data = event.data as
         | { type: 'audioData'; audioData: AudioData }
         | { type: 'bitrate'; kbps: number }
         | { type: 'latency'; media: 'audio'; ms: number }
-        | { audioData: AudioData }
-      if ('type' in data && data.type === 'bitrate') {
+      if (data.type === 'bitrate') {
         this.handlers.onRemoteAudioBitrate?.(userId, data.kbps)
         return
       }
-      if ('type' in data && data.type === 'latency') {
+      if (data.type === 'latency') {
         this.handlers.onRemoteAudioLatency?.(userId, data.ms)
         return
       }
-      const audioData = 'type' in data ? data.audioData : data.audioData
-      const retimedAudioData = this.retimeAudioData(audioData, context)
-      console.debug(audioData.timestamp, '->', retimedAudioData.timestamp)
+      const audioData = data.audioData
       await writer.ready
-      await writer.write(retimedAudioData)
-      audioData.close()
+      await writer.write(audioData)
     }
 
     this.audioContexts.set(trackAlias, context)
@@ -138,51 +130,4 @@ export class MediaSubscriber {
       [payload.buffer]
     )
   }
-
-  private retimeAudioData(audioData: AudioData, context: AudioSubscriptionContext): AudioData {
-    const originalTimestamp = audioData.timestamp
-    if (context.baseTimestamp === null) {
-      context.baseTimestamp = originalTimestamp
-    }
-    const playbackTimestamp = originalTimestamp - context.baseTimestamp
-    return this.cloneAudioDataWithTimestamp(audioData, playbackTimestamp)
-  }
-
-  private cloneAudioDataWithTimestamp(audioData: AudioData, timestamp: number): AudioData {
-    const options: AudioDataCopyToOptions = {
-      planeIndex: 0,
-      frameCount: audioData.numberOfFrames
-    }
-    const byteLength = audioData.allocationSize(options)
-    const format = audioData.format as AudioSampleFormat
-    const buffer = new ArrayBuffer(byteLength)
-    const view = this.createAudioBufferView(format, buffer)
-    audioData.copyTo(view, options)
-    return new AudioData({
-      format,
-      sampleRate: audioData.sampleRate,
-      numberOfFrames: audioData.numberOfFrames,
-      numberOfChannels: audioData.numberOfChannels,
-      timestamp,
-      data: buffer
-    })
-  }
-
-  private createAudioBufferView(format: AudioSampleFormat, buffer: ArrayBuffer): ArrayBufferView {
-    switch (format) {
-      case 'u8':
-      case 'u8-planar':
-        return new Uint8Array(buffer)
-      case 's16':
-      case 's16-planar':
-        return new Int16Array(buffer)
-      case 's32':
-      case 's32-planar':
-        return new Int32Array(buffer)
-      case 'f32':
-      case 'f32-planar':
-      default:
-        return new Float32Array(buffer)
-    }
-  }
 }