Skip to content

Commit 23521db

Browse files
yuki-uchidayuki-uchida
committed
impl: AudioJitterBuffer Mode
1 parent 185d1b0 commit 23521db

File tree

13 files changed

+285
-64
lines changed

13 files changed

+285
-64
lines changed

‎js/examples/call/src/components/CallRoom.tsx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ export function CallRoom({ session, onLeave }: CallRoomProps) {
3333
toggleMicrophone,
3434
videoJitterConfigs,
3535
setVideoJitterBufferConfig,
36+
audioJitterConfigs,
37+
setAudioJitterBufferConfig,
3638
videoCodecOptions,
3739
videoResolutionOptions,
3840
videoBitrateOptions,
@@ -152,6 +154,8 @@ export function CallRoom({ session, onLeave }: CallRoomProps) {
152154
remoteMedia={remoteMedia}
153155
videoJitterConfigs={videoJitterConfigs}
154156
onChangeVideoJitterConfig={setVideoJitterBufferConfig}
157+
audioJitterConfigs={audioJitterConfigs}
158+
onChangeAudioJitterConfig={setAudioJitterBufferConfig}
155159
onToggleCamera={handleToggleCamera}
156160
onToggleScreenShare={async () => {
157161
const enabled = await toggleScreenShare()

‎js/examples/call/src/components/JitterBufferControls.tsx‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,26 @@
11
import type { VideoJitterBufferMode } from '../../../../utils/media/videoJitterBuffer'
2-
import { DEFAULT_VIDEO_JITTER_CONFIG, type VideoJitterConfig } from '../types/jitterBuffer'
2+
import type { AudioJitterBufferMode } from '../../../../utils/media/audioJitterBuffer'
3+
import {
4+
DEFAULT_VIDEO_JITTER_CONFIG,
5+
DEFAULT_AUDIO_JITTER_CONFIG,
6+
type VideoJitterConfig,
7+
type AudioJitterConfig
8+
} from '../types/jitterBuffer'
39

4-
const MODES: VideoJitterBufferMode[] = ['buffered', 'normal', 'correctly', 'fast']
10+
const VIDEO_MODES: VideoJitterBufferMode[] = ['buffered', 'normal', 'correctly', 'fast']
11+
const AUDIO_MODES: AudioJitterBufferMode[] = ['ordered', 'latest']
512

6-
interface JitterBufferControlsProps {
13+
interface VideoJitterBufferControlsProps {
714
value?: VideoJitterConfig
815
onChange: (value: Partial<VideoJitterConfig>) => void
916
}
1017

11-
export function JitterBufferControls({ value, onChange }: JitterBufferControlsProps) {
18+
interface AudioJitterBufferControlsProps {
19+
value?: AudioJitterConfig
20+
onChange: (value: Partial<AudioJitterConfig>) => void
21+
}
22+
23+
export function VideoJitterBufferControls({ value, onChange }: VideoJitterBufferControlsProps) {
1224
const config = value ?? DEFAULT_VIDEO_JITTER_CONFIG
1325

1426
const handleModeChange = (mode: VideoJitterBufferMode) => onChange({ mode })
@@ -26,15 +38,15 @@ export function JitterBufferControls({ value, onChange }: JitterBufferControlsPr
2638

2739
return (
2840
<div className="space-y-2 rounded-lg border border-white/10 bg-white/5 p-3">
29-
<div className="text-xs font-semibold uppercase tracking-wide text-blue-100">Jitter Buffer</div>
41+
<div className="text-xs font-semibold uppercase tracking-wide text-blue-100">Video Jitter Buffer</div>
3042
<label className="flex items-center justify-between gap-3 text-sm">
3143
<span className="text-blue-50">Mode</span>
3244
<select
3345
value={config.mode}
3446
onChange={(e) => handleModeChange(e.target.value as VideoJitterBufferMode)}
3547
className="w-36 rounded-md border border-white/10 bg-white/10 px-2 py-1 text-sm text-white outline-none focus:border-blue-300 focus:ring-1 focus:ring-blue-300"
3648
>
37-
{MODES.map((mode) => (
49+
{VIDEO_MODES.map((mode) => (
3850
<option key={mode} value={mode} className="bg-slate-900 text-white">
3951
{mode}
4052
</option>
@@ -68,3 +80,29 @@ export function JitterBufferControls({ value, onChange }: JitterBufferControlsPr
6880
</div>
6981
)
7082
}
83+
84+
export function AudioJitterBufferControls({ value, onChange }: AudioJitterBufferControlsProps) {
85+
const config = value ?? DEFAULT_AUDIO_JITTER_CONFIG
86+
87+
const handleModeChange = (mode: AudioJitterBufferMode) => onChange({ mode })
88+
89+
return (
90+
<div className="space-y-2 rounded-lg border border-white/10 bg-white/5 p-3">
91+
<div className="text-xs font-semibold uppercase tracking-wide text-blue-100">Audio Jitter Buffer</div>
92+
<label className="flex items-center justify-between gap-3 text-sm">
93+
<span className="text-blue-50">Mode</span>
94+
<select
95+
value={config.mode}
96+
onChange={(e) => handleModeChange(e.target.value as AudioJitterBufferMode)}
97+
className="w-36 rounded-md border border-white/10 bg-white/10 px-2 py-1 text-sm text-white outline-none focus:border-blue-300 focus:ring-1 focus:ring-blue-300"
98+
>
99+
{AUDIO_MODES.map((mode) => (
100+
<option key={mode} value={mode} className="bg-slate-900 text-white">
101+
{mode}
102+
</option>
103+
))}
104+
</select>
105+
</label>
106+
</div>
107+
)
108+
}

‎js/examples/call/src/components/MemberGrid.tsx‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ import { LocalMember, RemoteMember } from '../types/member'
22
import { RemoteMediaStreams } from '../types/media'
33
import { MediaStreamVideo, MediaStreamAudio } from './MediaStreamElements'
44
import { ReactNode, useState } from 'react'
5-
import type { VideoJitterConfig } from '../types/jitterBuffer'
6-
import { JitterBufferControls } from './JitterBufferControls'
5+
import type { VideoJitterConfig, AudioJitterConfig } from '../types/jitterBuffer'
6+
import { VideoJitterBufferControls, AudioJitterBufferControls } from './JitterBufferControls'
77
import { Mic, MicOff, Monitor, Settings2, Video, VideoOff } from 'lucide-react'
88
import { DeviceSelector } from './DeviceSelector'
99
import type { VideoEncodingSettings } from '../types/videoEncoding'
@@ -18,6 +18,8 @@ interface MemberGridProps {
1818
remoteMedia: Map<string, RemoteMediaStreams>
1919
videoJitterConfigs: Map<string, VideoJitterConfig>
2020
onChangeVideoJitterConfig: (userId: string, config: Partial<VideoJitterConfig>) => void
21+
audioJitterConfigs: Map<string, AudioJitterConfig>
22+
onChangeAudioJitterConfig: (userId: string, config: Partial<AudioJitterConfig>) => void
2123
onToggleCamera: () => void
2224
onToggleScreenShare: () => void
2325
onToggleMicrophone: () => void
@@ -61,6 +63,8 @@ export function MemberGrid({
6163
remoteMedia,
6264
videoJitterConfigs,
6365
onChangeVideoJitterConfig,
66+
audioJitterConfigs,
67+
onChangeAudioJitterConfig,
6468
onToggleCamera,
6569
onToggleScreenShare,
6670
onToggleMicrophone,
@@ -269,10 +273,14 @@ export function MemberGrid({
269273
<h4 className="text-lg font-semibold text-white">
270274
Jitter Buffer ({findMemberName(remoteMembers, jitterModalTarget)})
271275
</h4>
272-
<JitterBufferControls
276+
<VideoJitterBufferControls
273277
value={videoJitterConfigs.get(jitterModalTarget)}
274278
onChange={(config) => onChangeVideoJitterConfig(jitterModalTarget, config)}
275279
/>
280+
<AudioJitterBufferControls
281+
value={audioJitterConfigs.get(jitterModalTarget)}
282+
onChange={(config) => onChangeAudioJitterConfig(jitterModalTarget, config)}
283+
/>
276284
</div>
277285
</DeviceModal>
278286
)}

‎js/examples/call/src/hooks/useCallMedia.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,14 @@
11
import { useCallback, useEffect, useState } from 'react'
22
import { LocalSession } from '../session/localSession'
33
import { RemoteMediaStreams } from '../types/media'
4-
import { DEFAULT_VIDEO_JITTER_CONFIG, normalizeVideoJitterConfig, type VideoJitterConfig } from '../types/jitterBuffer'
4+
import {
5+
DEFAULT_VIDEO_JITTER_CONFIG,
6+
normalizeVideoJitterConfig,
7+
type VideoJitterConfig,
8+
DEFAULT_AUDIO_JITTER_CONFIG,
9+
normalizeAudioJitterConfig,
10+
type AudioJitterConfig
11+
} from '../types/jitterBuffer'
512
import {
613
DEFAULT_VIDEO_ENCODING_SETTINGS,
714
VIDEO_BITRATE_OPTIONS,
@@ -33,6 +40,8 @@ interface UseCallMediaResult {
3340
toggleMicrophone: () => Promise<boolean>
3441
videoJitterConfigs: Map<string, VideoJitterConfig>
3542
setVideoJitterBufferConfig: (userId: string, config: Partial<VideoJitterConfig>) => void
43+
audioJitterConfigs: Map<string, AudioJitterConfig>
44+
setAudioJitterBufferConfig: (userId: string, config: Partial<AudioJitterConfig>) => void
3645
videoCodecOptions: typeof VIDEO_CODEC_OPTIONS
3746
videoResolutionOptions: typeof VIDEO_RESOLUTION_OPTIONS
3847
videoBitrateOptions: typeof VIDEO_BITRATE_OPTIONS
@@ -67,6 +76,7 @@ export function useCallMedia(session: LocalSession | null): UseCallMediaResult {
6776
const [localAudioBitrate, setLocalAudioBitrate] = useState<number | null>(null)
6877
const [remoteMedia, setRemoteMedia] = useState<Map<string, RemoteMediaStreams>>(new Map())
6978
const [videoJitterConfigs, setVideoJitterConfigs] = useState<Map<string, VideoJitterConfig>>(new Map())
79+
const [audioJitterConfigs, setAudioJitterConfigs] = useState<Map<string, AudioJitterConfig>>(new Map())
7080
const [videoDevices, setVideoDevices] = useState<MediaDeviceInfo[]>([])
7181
const [audioDevices, setAudioDevices] = useState<MediaDeviceInfo[]>([])
7282
const [selectedVideoDeviceId, setSelectedVideoDeviceId] = useState<string | null>(null)
@@ -112,6 +122,7 @@ export function useCallMedia(session: LocalSession | null): UseCallMediaResult {
112122
setMicrophoneEnabled(false)
113123
setScreenShareEnabled(false)
114124
setVideoJitterConfigs(new Map())
125+
setAudioJitterConfigs(new Map())
115126
setVideoEncoderError(null)
116127
setAudioEncoderError(null)
117128
return
@@ -225,6 +236,7 @@ export function useCallMedia(session: LocalSession | null): UseCallMediaResult {
225236
setCameraEnabled(session.localMember.publishedTracks.video)
226237
setMicrophoneEnabled(session.localMember.publishedTracks.audio)
227238
setVideoJitterConfigs(new Map())
239+
setAudioJitterConfigs(new Map())
228240
}
229241
}, [session])
230242

@@ -352,6 +364,24 @@ export function useCallMedia(session: LocalSession | null): UseCallMediaResult {
352364
[session]
353365
)
354366

367+
const setAudioJitterBufferConfig = useCallback(
368+
(userId: string, config: Partial<AudioJitterConfig>) => {
369+
if (!session) {
370+
return
371+
}
372+
const controller = session.getMediaController()
373+
setAudioJitterConfigs((prev) => {
374+
const current = prev.get(userId) ?? DEFAULT_AUDIO_JITTER_CONFIG
375+
const next = normalizeAudioJitterConfig({ ...current, ...config })
376+
const updated = new Map(prev)
377+
updated.set(userId, next)
378+
controller.setAudioJitterBufferConfig(userId, next)
379+
return updated
380+
})
381+
},
382+
[session]
383+
)
384+
355385
const selectVideoDevice = useCallback(
356386
async (deviceId: string) => {
357387
setSelectedVideoDeviceId(deviceId)
@@ -472,6 +502,8 @@ export function useCallMedia(session: LocalSession | null): UseCallMediaResult {
472502
toggleMicrophone,
473503
videoJitterConfigs,
474504
setVideoJitterBufferConfig,
505+
audioJitterConfigs,
506+
setAudioJitterBufferConfig,
475507
videoCodecOptions: VIDEO_CODEC_OPTIONS,
476508
videoResolutionOptions: VIDEO_RESOLUTION_OPTIONS,
477509
videoBitrateOptions: VIDEO_BITRATE_OPTIONS,

‎js/examples/call/src/media/callMediaController.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { MoqtClientWrapper } from '@moqt/moqtClient'
22
import { MediaPublisher } from './mediaPublisher'
33
import { MediaSubscriber } from './mediaSubscriber'
4-
import type { VideoJitterConfig } from '../types/jitterBuffer'
4+
import type { VideoJitterConfig, AudioJitterConfig } from '../types/jitterBuffer'
55
import type { VideoEncodingSettings } from '../types/videoEncoding'
66
import type { AudioEncodingSettings } from '../types/audioEncoding'
77

@@ -98,6 +98,10 @@ export class CallMediaController {
9898
this.subscriber.setVideoJitterBufferConfig(userId, config)
9999
}
100100

101+
setAudioJitterBufferConfig(userId: string, config: AudioJitterConfig): void {
102+
this.subscriber.setAudioJitterBufferConfig(userId, config)
103+
}
104+
101105
async setVideoEncodingSettings(settings: VideoEncodingSettings, deviceId?: string, restartIfActive: boolean = false) {
102106
await this.publisher.setVideoEncodingSettings(settings, deviceId, restartIfActive)
103107
}

‎js/examples/call/src/media/mediaSubscriber.ts‎

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,15 @@
11
import { MoqtClientWrapper } from '@moqt/moqtClient'
22
import { SubgroupStreamObjectMessage } from '../../../../pkg/moqt_client_sample'
33
import type { VideoJitterBufferMode } from '../../../../utils/media/videoJitterBuffer'
4-
import { DEFAULT_VIDEO_JITTER_CONFIG, normalizeVideoJitterConfig, type VideoJitterConfig } from '../types/jitterBuffer'
4+
import type { AudioJitterBufferMode } from '../../../../utils/media/audioJitterBuffer'
5+
import {
6+
DEFAULT_VIDEO_JITTER_CONFIG,
7+
DEFAULT_AUDIO_JITTER_CONFIG,
8+
normalizeVideoJitterConfig,
9+
normalizeAudioJitterConfig,
10+
type VideoJitterConfig,
11+
type AudioJitterConfig
12+
} from '../types/jitterBuffer'
513

614
interface MediaSubscriberHandlers {
715
onRemoteVideoStream?: (userId: string, stream: MediaStream) => void
@@ -34,6 +42,7 @@ export class MediaSubscriber {
3442
private readonly videoContexts = new Map<bigint, VideoSubscriptionContext>()
3543
private readonly audioContexts = new Map<bigint, AudioSubscriptionContext>()
3644
private readonly videoJitterConfigByUserId = new Map<string, VideoJitterConfig>()
45+
private readonly audioJitterConfigByUserId = new Map<string, AudioJitterConfig>()
3746
private readonly videoCodecByTrackAlias = new Map<bigint, string>()
3847
private readonly videoSizeByTrackAlias = new Map<bigint, { width: number; height: number }>()
3948

@@ -73,13 +82,12 @@ export class MediaSubscriber {
7382
}
7483
if (data.type === 'decoderConfig') {
7584
this.videoCodecByTrackAlias.set(trackAlias, data.codec)
76-
if (typeof data.width === 'number' || typeof data.height === 'number') {
77-
this.handlers.onRemoteVideoConfig?.(userId, {
78-
codec: data.codec,
79-
width: data.width,
80-
height: data.height
81-
})
82-
}
85+
const lastSize = this.videoSizeByTrackAlias.get(trackAlias)
86+
this.handlers.onRemoteVideoConfig?.(userId, {
87+
codec: data.codec,
88+
width: data.width ?? lastSize?.width,
89+
height: data.height ?? lastSize?.height
90+
})
8391
return
8492
}
8593
if (data.type === 'frame') {
@@ -156,6 +164,10 @@ export class MediaSubscriber {
156164

157165
this.audioContexts.set(trackAlias, context)
158166
this.handlers.onRemoteAudioStream?.(userId, context.stream)
167+
const config = this.audioJitterConfigByUserId.get(userId)
168+
if (config) {
169+
worker.postMessage({ type: 'config', config })
170+
}
159171

160172
this.client.setOnSubgroupObjectHandler(trackAlias, (groupId, message) =>
161173
this.forwardToWorker(worker, trackAlias, groupId, message)
@@ -192,9 +204,25 @@ export class MediaSubscriber {
192204
}
193205
}
194206

207+
setAudioJitterBufferConfig(userId: string, config: AudioJitterConfig): void {
208+
const sanitized = this.sanitizeAudioConfig(config)
209+
this.audioJitterConfigByUserId.set(userId, sanitized)
210+
for (const context of this.audioContexts.values()) {
211+
if (context.userId === userId) {
212+
context.worker.postMessage({ type: 'config', config: sanitized })
213+
}
214+
}
215+
}
216+
195217
private sanitizeConfig(config: VideoJitterConfig): VideoJitterConfig {
196218
const normalized = normalizeVideoJitterConfig(config)
197219
const mode: VideoJitterBufferMode = normalized.mode ?? DEFAULT_VIDEO_JITTER_CONFIG.mode
198220
return { ...normalized, mode }
199221
}
222+
223+
private sanitizeAudioConfig(config: AudioJitterConfig): AudioJitterConfig {
224+
const normalized = normalizeAudioJitterConfig(config)
225+
const mode: AudioJitterBufferMode = normalized.mode ?? DEFAULT_AUDIO_JITTER_CONFIG.mode
226+
return { ...normalized, mode }
227+
}
200228
}

‎js/examples/call/src/types/jitterBuffer.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { VideoJitterBufferMode } from '../../../../utils/media/videoJitterBuffer'
2+
import type { AudioJitterBufferMode } from '../../../../utils/media/audioJitterBuffer'
23

34
export type VideoJitterConfig = {
45
mode: VideoJitterBufferMode
@@ -7,7 +8,7 @@ export type VideoJitterConfig = {
78
}
89

910
export const DEFAULT_VIDEO_JITTER_CONFIG: VideoJitterConfig = {
10-
mode: 'buffered',
11+
mode: 'fast',
1112
minDelayMs: 250,
1213
bufferedAheadFrames: 5
1314
}
@@ -26,3 +27,16 @@ export function normalizeVideoJitterConfig(config: Partial<VideoJitterConfig>):
2627
)
2728
return { mode, minDelayMs, bufferedAheadFrames }
2829
}
30+
31+
export type AudioJitterConfig = {
32+
mode: AudioJitterBufferMode
33+
}
34+
35+
export const DEFAULT_AUDIO_JITTER_CONFIG: AudioJitterConfig = {
36+
mode: 'latest'
37+
}
38+
39+
export function normalizeAudioJitterConfig(config: Partial<AudioJitterConfig>): AudioJitterConfig {
40+
const mode = config.mode ?? DEFAULT_AUDIO_JITTER_CONFIG.mode
41+
return { mode }
42+
}

0 commit comments

Comments
 (0)