fix(realtime-session): preserve audio format and session config fields on agent update

Author: myaroshefsky

packages/agents-realtime/src/realtimeSession.ts

@@ -183,6 +183,13 @@ export class RealtimeSession<
   #shouldIncludeAudioData: boolean;
   #interruptedByGuardrail: Record<string, boolean> = {};
   #audioStarted = false;
+  // Keeps track of the last full session config we sent (camelCase keys) so that
+  // subsequent updates (e.g. during agent handoffs) preserve properties that are
+  // not explicitly recalculated here (such as inputAudioFormat, outputAudioFormat,
+  // modalities, speed, toolChoice, turnDetection, etc.). Without this, updating
+  // the agent would drop audio format overrides (e.g. g711_ulaw) and revert to
+  // transport defaults causing issues for integrations like Twilio.
+  #lastSessionConfig: Partial<RealtimeSessionConfig> | null = null;
 
   constructor(
     public readonly initialAgent:
@@ -313,14 +320,34 @@ export class RealtimeSession<
       );
     }
 
-    return {
+    // Start from any previously-sent config (so we preserve values like audio formats)
+    // and the original options.config provided by the user. Preference order:
+    // 1. Last session config we sent (#lastSessionConfig)
+    // 2. Original options.config
+    // 3. Additional config passed into this invocation (explicit overrides)
+    // Finally we overwrite dynamic fields (instructions, voice, model, tools, tracing)
+    // to ensure they always reflect the current agent & runtime state.
+    const base: Partial<RealtimeSessionConfig> = {
+      ...(this.#lastSessionConfig ?? {}),
+      ...(this.options.config ?? {}),
+      ...(additionalConfig ?? {}),
+    };
+
+    // Note: Certain fields cannot be updated after the session begins, such as voice and model
+    const fullConfig: Partial<RealtimeSessionConfig> = {
+      ...base,
       instructions,
       voice: this.#currentAgent.voice,
       model: this.options.model,
       tools: this.#currentTools,
       tracing: tracingConfig,
-      ...additionalConfig,
     };
+
+    // Update our cache so subsequent updates inherit the full set including any
+    // dynamic fields we just overwrote.
+    this.#lastSessionConfig = fullConfig;
+
+    return fullConfig;
   }
 
   async updateAgent(newAgent: RealtimeAgent<TBaseContext>) {
@@ -689,6 +716,9 @@ export class RealtimeSession<
       url: options.url,
       initialSessionConfig: await this.#getSessionConfig(this.options.config),
     });
+    // Ensure the cached lastSessionConfig includes everything passed as the initial session config
+    // (the call above already set it via #getSessionConfig but in case additional overrides were
+    // passed directly here in the future we could merge them). For now it's a no-op.
 
     this.#history = [];
     this.emit('history_updated', this.#history);

packages/agents-realtime/test/realtimeSession.test.ts

@@ -300,4 +300,20 @@ describe('RealtimeSession', () => {
     } as any);
     expect(startEvents).toBe(2);
   });
+
+  it('preserves custom audio formats across updateAgent', async () => {
+    const t = new FakeTransport();
+    const agent = new RealtimeAgent({ name: 'Orig', handoffs: [] });
+    const s = new RealtimeSession(agent, {
+      transport: t,
+      config: { inputAudioFormat: 'g711_ulaw', outputAudioFormat: 'g711_ulaw' },
+    });
+    await s.connect({ apiKey: 'test' });
+    const newAgent = new RealtimeAgent({ name: 'Next', handoffs: [] });
+    await s.updateAgent(newAgent);
+    // Find the last updateSessionConfig call
+    const last = t.updateSessionConfigCalls.at(-1)!;
+    expect(last.inputAudioFormat).toBe('g711_ulaw');
+    expect(last.outputAudioFormat).toBe('g711_ulaw');
+  });
 });