Smooth all spatial audio motion with velocity-driven tweening

GMCP delivers movement as whole-metre position snaps, so sources and the
listener teleported around the head; near a source one strafe leapt ~115
degrees in a single audio frame. Interpolate everything instead:

- VectorTweener: keyed, single-rAF-loop glides; duration from the server's
  velocity (clamped 80-600ms), mid-flight retargeting, nlerp mode for
  orientation axes, injectable clock/scheduler for tests.
- Client.Spatial handler tweens listener/entity positions, nlerps forward
  vectors over 150ms; scene snapshots and enter/leave still snap.
- Media sound positions tween per sound; LiveKit panners snap on attach and
  ramp on sync; FOA encode/level gains and ambisonic distance gain ramp via
  setTargetAtTime (30ms tau) instead of stepping.
- spatialStore.patchEntity supports per-frame partial updates.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016CyEqCPCmxQYQHamSH1m2A

Author: ctoth

src/audio/AmbisonicRenderer.ts

@@ -1,6 +1,8 @@
 import type { Cacophony, AudioNode as CacophonyAudioNode, Playback } from 'cacophony';
 import Omnitone, { type FOARenderer } from 'omnitone/build/omnitone.min.esm.js';
 
+import { smoothParamTo } from './audioParamSmoothing';
+
 const IDENTITY_ROTATION = new Float32Array([1, 0, 0, 0, 1, 0, 0, 0, 1]);
 type AmbisonicInputMode = 'stereo-upmix' | 'foa-passthrough';
 
@@ -69,7 +71,8 @@ export class AmbisonicRenderer {
       return;
     }
     const clamped = Number.isFinite(gain) ? Math.min(1, Math.max(0, gain)) : 1;
-    this.distanceGain.gain.value = clamped;
+    const currentTime = (this.cacophony.context as unknown as BaseAudioContext).currentTime;
+    smoothParamTo(this.distanceGain.gain, clamped, currentTime);
   }
 
   cleanup(): void {

src/audio/LiveKitSpatialAudioBridge.test.ts

@@ -1,5 +1,6 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
 import { LiveKitSpatialAudioBridge } from "./LiveKitSpatialAudioBridge";
+import { SPATIAL_PARAM_TAU_S } from "./audioParamSmoothing";
 import { SPATIAL_DISTANCE_MODEL } from "./distanceModel";
 
 const MockMediaStream = vi.fn();
@@ -26,6 +27,10 @@ function createAudioParam(value = 0) {
       value = nextValue;
       return undefined as never;
     }),
+    setTargetAtTime: vi.fn((nextValue: number) => {
+      value = nextValue;
+      return undefined as never;
+    }),
   };
 }
 
@@ -160,7 +165,7 @@ describe("LiveKitSpatialAudioBridge", () => {
     expect(cacophony.resume).toHaveBeenCalledOnce();
   });
 
-  it("updates an existing panner position from the spatial lookup", () => {
+  it("snaps the initial panner position, then ramps position updates from the spatial lookup", () => {
     const remoteTrack = track(1);
     const { cacophony, panner } = createCacophony();
     const positions: Record<string, [number, number, number]> = {
@@ -169,12 +174,16 @@ describe("LiveKitSpatialAudioBridge", () => {
     const bridge = new LiveKitSpatialAudioBridge(cacophony, (participantId) => positions[participantId]);
 
     bridge.attachParticipantTrack("player-2", remoteTrack);
+
+    expect(panner.positionX.setValueAtTime).toHaveBeenCalledWith(1, 7);
+    expect(panner.positionX.setTargetAtTime).not.toHaveBeenCalled();
+
     positions["player-2"] = [4, 5, 6];
     bridge.syncParticipant("player-2");
 
-    expect(panner.positionX.setValueAtTime).toHaveBeenLastCalledWith(4, 7);
-    expect(panner.positionY.setValueAtTime).toHaveBeenLastCalledWith(5, 7);
-    expect(panner.positionZ.setValueAtTime).toHaveBeenLastCalledWith(6, 7);
+    expect(panner.positionX.setTargetAtTime).toHaveBeenLastCalledWith(4, 7, SPATIAL_PARAM_TAU_S);
+    expect(panner.positionY.setTargetAtTime).toHaveBeenLastCalledWith(5, 7, SPATIAL_PARAM_TAU_S);
+    expect(panner.positionZ.setTargetAtTime).toHaveBeenLastCalledWith(6, 7, SPATIAL_PARAM_TAU_S);
   });
 
   it("primes Chromium decode with a muted media element and tears it down on detach", () => {

src/audio/LiveKitSpatialAudioBridge.ts

@@ -7,6 +7,7 @@ import type {
   Position,
 } from "cacophony";
 
+import { smoothParamTo } from "./audioParamSmoothing";
 import { SPATIAL_DISTANCE_MODEL } from "./distanceModel";
 
 export type SpatialPositionLookup = (participantId: string) => Position | null | undefined;
@@ -67,7 +68,7 @@ export class LiveKitSpatialAudioBridge {
     panner.connect(outputGain);
     outputGain.connect(this.cacophony.globalGainNode);
 
-    this.applyPosition(panner, this.positionFor(participantId));
+    this.applyPosition(panner, this.positionFor(participantId), { snap: true });
     this.entries.set(participantId, {
       downmixNodes: nodes,
       outputGain,
@@ -180,11 +181,30 @@ export class LiveKitSpatialAudioBridge {
     return 2;
   }
 
-  private applyPosition(panner: CacophonyPannerNode, [x, y, z]: Position): void {
+  /**
+   * Aim the participant's panner. The first placement (attach) snaps so a new
+   * voice does not audibly fly in from the origin; subsequent syncs ramp with a
+   * short time constant to de-zipper the per-frame steps the position tweener
+   * delivers through the spatial store.
+   */
+  private applyPosition(
+    panner: CacophonyPannerNode,
+    [x, y, z]: Position,
+    options?: { snap?: boolean },
+  ): void {
     const time = this.cacophony.context.currentTime;
-    panner.positionX.setValueAtTime(x, time);
-    panner.positionY.setValueAtTime(y, time);
-    panner.positionZ.setValueAtTime(z, time);
+    const axes = [
+      [panner.positionX, x],
+      [panner.positionY, y],
+      [panner.positionZ, z],
+    ] as const;
+    for (const [param, value] of axes) {
+      if (options?.snap) {
+        param.setValueAtTime(value, time);
+      } else {
+        smoothParamTo(param, value, time);
+      }
+    }
   }
 
   private disconnectEntry(entry: SpatialAudioEntry): void {

src/audio/MediaService.ts

@@ -10,6 +10,7 @@ import { usePreferences } from '../stores/preferencesStore';
 import { AmbisonicRenderer } from './AmbisonicRenderer';
 import { PositionalFoaRenderer } from './PositionalFoaRenderer';
 import { distanceBetween, inverseDistanceGain, SPATIAL_DISTANCE_MODEL } from './distanceModel';
+import { VectorTweener } from './vectorTween';
 import { EffectChain } from './effects/EffectChain';
 import { MediaEffects } from './effects/MediaEffects';
 import type { EffectSpec } from './effects/types';
@@ -146,6 +147,8 @@ export interface ExtendedSound extends Sound {
 
 interface MediaServiceOptions {
   manageFocus?: boolean;
+  /** Override the sound-position tweener (tests inject a manually-clocked one). */
+  motion?: VectorTweener;
 }
 
 export class MediaService {
@@ -154,6 +157,8 @@ export class MediaService {
   defaultUrl = '';
 
   private readonly cleanedSounds = new WeakSet<ExtendedSound>();
+  /** Glides server-sent sound positions (keyed by sound) instead of snapping. */
+  private readonly motion: VectorTweener;
   private readonly effects: MediaEffects;
   private readonly mediaSession = new MediaSessionController();
   private readonly preloadedSoundKeys = new Set<string>();
@@ -168,6 +173,7 @@ export class MediaService {
     this.cacophony = cacophony;
     this.effects = new MediaEffects(this.cacophony);
     this.manageFocus = options.manageFocus ?? true;
+    this.motion = options.motion ?? new VectorTweener();
 
     this.setGlobalVolume(usePreferences.getState().sound.volume);
     if (this.manageFocus && typeof window !== 'undefined') {
@@ -644,6 +650,7 @@ export class MediaService {
   }
 
   private releaseSound(sound: ExtendedSound, key?: string): void {
+    this.motion.cancel(sound);
     if (sound === this.currentMusic) {
       this.currentMusic = undefined;
       this.mediaSession.clear();
@@ -879,10 +886,14 @@ export class MediaService {
     }
 
     if (data.position?.length) {
-      sound.mediaPosition = [data.position[0], data.position[1], data.position[2]];
-      sound.position = sound.mediaPosition;
-      this.updateAmbisonicDistance(sound as ExtendedSound);
-      this.updatePositionalSpatial(sound as ExtendedSound);
+      const target: Position = [data.position[0], data.position[1], data.position[2]];
+      // First placement snaps; later updates glide from the current position.
+      this.motion.tween(sound, sound.mediaPosition, target, (value) => {
+        sound.mediaPosition = [value[0], value[1], value[2]];
+        sound.position = sound.mediaPosition;
+        this.updateAmbisonicDistance(sound);
+        this.updatePositionalSpatial(sound);
+      });
     }
 
     if (data.start !== undefined) {

src/audio/PositionalFoaRenderer.ts

@@ -1,6 +1,7 @@
 import type { Cacophony, AudioNode as CacophonyAudioNode, Playback } from 'cacophony';
 import { encodeMonoToFoaSN3D, type FoaDecoder } from 'cacophony';
 
+import { smoothParamTo } from './audioParamSmoothing';
 import { sourceBearing, type Vec3 } from './foaBearing';
 
 /** One encoder bank: four ACN [W,Y,Z,X] gains fed by one signal, aimed at the
@@ -148,13 +149,17 @@ export class PositionalFoaRenderer {
     this.setBearing(azimuth, elevation);
   }
 
-  /** Set every bank's encode gains from an azimuth (CCW, +left) and elevation (+up). */
+  /** Set every bank's encode gains from an azimuth (CCW, +left) and elevation (+up).
+   *  Writes are smoothed (short exponential ramps) so a bearing change glides
+   *  instead of clicking — encode gains are pure crossfade weights, so ramping
+   *  them is safe. */
   setBearing(azimuthRad: number, elevationRad: number): void {
+    const currentTime = this.context.currentTime;
     for (const bank of this.banks) {
       const coeffs = encodeMonoToFoaSN3D(1, azimuthRad + bank.azOffset, elevationRad); // [W,Y,Z,X]
       for (let i = 0; i < 4; i++) {
         const v = coeffs[i];
-        bank.gains[i].gain.value = Number.isFinite(v) ? v : i === 0 ? 1 : 0;
+        smoothParamTo(bank.gains[i].gain, Number.isFinite(v) ? v : i === 0 ? 1 : 0, currentTime);
       }
     }
   }
@@ -173,7 +178,7 @@ export class PositionalFoaRenderer {
 
   private applyLevel(): void {
     if (this.levelGain) {
-      this.levelGain.gain.value = this.makeup * this.distanceAttenuation;
+      smoothParamTo(this.levelGain.gain, this.makeup * this.distanceAttenuation, this.context.currentTime);
     }
   }
 

src/audio/audioParamSmoothing.ts

@@ -0,0 +1,36 @@
+/**
+ * Shared de-zipper for spatial AudioParam writes. Position tweening delivers
+ * per-frame steps; smoothing each write with a short exponential ramp
+ * (setTargetAtTime) removes the residual staircase and the clicks that raw
+ * `param.value =` assignments produce on gain and panner params.
+ */
+
+/** Time constant (s): ~63% of the way per tau, settled within ~3·tau (≈90ms). */
+export const SPATIAL_PARAM_TAU_S = 0.03;
+
+/**
+ * Structural subset of AudioParam accepted here, so tests (and cacophony's
+ * wrapper types) can pass plain `{ value }` objects.
+ */
+export interface SmoothableParam {
+  value: number;
+  setTargetAtTime?(value: number, startTime: number, timeConstant: number): unknown;
+}
+
+/**
+ * Ramp `param` toward `value` with an exponential approach starting at
+ * `currentTime`. Falls back to a direct assignment when the param cannot
+ * schedule (mock nodes, detached contexts).
+ */
+export function smoothParamTo(
+  param: SmoothableParam,
+  value: number,
+  currentTime: number | undefined,
+  tauS: number = SPATIAL_PARAM_TAU_S,
+): void {
+  if (typeof param.setTargetAtTime === 'function' && Number.isFinite(currentTime)) {
+    param.setTargetAtTime(value, currentTime as number, tauS);
+  } else {
+    param.value = value;
+  }
+}