fix: session config breaks tool registration - cannot use tools with audio customization

- Fix _getMergedSessionConfig to only include tools field when tools are present
- Prevent sending empty/undefined tools array which disables tool calls
- Add comprehensive tests covering the issue scenarios
- Resolves mutual exclusivity between session config and tool functionality

Fixes #339

Author: AjayKumbham

.changeset/fix-session-config-tool-registration.md

@@ -0,0 +1,11 @@
+---
+'@openai/agents-realtime': patch
+---
+
+Fix session config breaking tool registration
+
+When initializing a RealtimeSession with any config object, tool registration was being broken because empty or undefined tools arrays were being sent to the OpenAI API, which disabled tool calls. This fix ensures that the tools field is only included in session updates when tools are actually present, preventing the API from disabling tool functionality when session configuration is provided.
+
+The issue occurred because the `_getMergedSessionConfig` method would include `tools: undefined` in the session data when no tools were explicitly provided in the config, which the OpenAI API interpreted as a request to disable all tools. Now, the tools field is only included when tools are actually available and non-empty.
+
+This allows users to customize audio settings (voice, turn detection, noise reduction, etc.) while maintaining tool functionality, resolving the mutual exclusivity between session configuration and tool registration.

packages/agents-realtime/src/openaiRealtimeBase.ts

@@ -387,7 +387,7 @@ export abstract class OpenAIRealtimeBase
   }
 
   protected _getMergedSessionConfig(config: Partial<RealtimeSessionConfig>) {
-    const sessionData = {
+    const sessionData: any = {
       instructions: config.instructions,
       model:
         config.model ??
@@ -414,15 +414,20 @@ export abstract class OpenAIRealtimeBase
         DEFAULT_OPENAI_REALTIME_SESSION_CONFIG.turnDetection,
       tool_choice:
         config.toolChoice ?? DEFAULT_OPENAI_REALTIME_SESSION_CONFIG.toolChoice,
-      tools: config.tools?.map((tool) => ({
-        ...tool,
-        strict: undefined,
-      })),
       // We don't set tracing here to make sure that we don't try to override it on every
       // session.update as it might lead to errors
       ...(config.providerData ?? {}),
     };
 
+    // Only include tools if they are explicitly provided and not empty
+    // This prevents sending an empty tools array which would disable tool calls
+    if (config.tools && config.tools.length > 0) {
+      sessionData.tools = config.tools.map((tool) => ({
+        ...tool,
+        strict: undefined,
+      }));
+    }
+
     return sessionData;
   }
 

packages/agents-realtime/test/realtimeSession.test.ts

@@ -318,84 +318,219 @@ describe('RealtimeSession', () => {
     expect(last.outputAudioFormat).toBe('g711_ulaw');
   });
 
-it('defaults item status to completed for done output items without status', async () => {
-  class TestTransport extends OpenAIRealtimeBase {
-    status: 'connected' | 'disconnected' | 'connecting' | 'disconnecting' =
-      'connected';
-    connect = vi.fn(async () => {});
-    sendEvent = vi.fn();
-    mute = vi.fn();
-    close = vi.fn();
-    interrupt = vi.fn();
-    get muted() {
-      return false;
+  it('defaults item status to completed for done output items without status', async () => {
+    class TestTransport extends OpenAIRealtimeBase {
+      status: 'connected' | 'disconnected' | 'connecting' | 'disconnecting' =
+        'connected';
+      connect = vi.fn(async () => { });
+      sendEvent = vi.fn();
+      mute = vi.fn();
+      close = vi.fn();
+      interrupt = vi.fn();
+      get muted() {
+        return false;
+      }
     }
-  }
-  const transport = new TestTransport();
-  const agent = new RealtimeAgent({ name: 'A', handoffs: [] });
-  const session = new RealtimeSession(agent, { transport });
-  await session.connect({ apiKey: 'test' });
-  const historyEvents: RealtimeItem[][] = [];
-  session.on('history_updated', (h) => historyEvents.push([...h]));
-  (transport as any)._onMessage({
-    data: JSON.stringify({
-      type: 'response.output_item.done',
-      event_id: 'e',
-      item: {
-        id: 'm1',
-        type: 'message',
-        role: 'assistant',
-        content: [{ type: 'text', text: 'hi' }],
-      },
-      output_index: 0,
-      response_id: 'r1',
-    }),
+    const transport = new TestTransport();
+    const agent = new RealtimeAgent({ name: 'A', handoffs: [] });
+    const session = new RealtimeSession(agent, { transport });
+    await session.connect({ apiKey: 'test' });
+    const historyEvents: RealtimeItem[][] = [];
+    session.on('history_updated', (h) => historyEvents.push([...h]));
+    (transport as any)._onMessage({
+      data: JSON.stringify({
+        type: 'response.output_item.done',
+        event_id: 'e',
+        item: {
+          id: 'm1',
+          type: 'message',
+          role: 'assistant',
+          content: [{ type: 'text', text: 'hi' }],
+        },
+        output_index: 0,
+        response_id: 'r1',
+      }),
+    });
+    const latest = historyEvents.at(-1)!;
+    const msg = latest.find(
+      (i): i is Extract<RealtimeItem, { type: 'message'; role: 'assistant' }> =>
+        i.type === 'message' && i.role === 'assistant' && (i as any).itemId === 'm1'
+    );
+    expect(msg).toBeDefined();
+    expect(msg!.status).toBe('completed');
   });
-  const latest = historyEvents.at(-1)!;
-  const msg = latest.find(
-    (i): i is Extract<RealtimeItem, { type: 'message'; role: 'assistant' }> =>
-      i.type === 'message' && i.role === 'assistant' && (i as any).itemId === 'm1'
-  );
-  expect(msg).toBeDefined();
-  expect(msg!.status).toBe('completed');
-});
-
-it('preserves explicit completed status on done', async () => {
-  class TestTransport extends OpenAIRealtimeBase {
-    status: 'connected' | 'disconnected' | 'connecting' | 'disconnecting' = 'connected';
-    connect = vi.fn(async () => {});
-    sendEvent = vi.fn(); mute = vi.fn(); close = vi.fn(); interrupt = vi.fn();
-    get muted() { return false; }
-  }
-  const transport = new TestTransport();
-  const session = new RealtimeSession(new RealtimeAgent({ name: 'A', handoffs: [] }), { transport });
-  await session.connect({ apiKey: 'test' });
-
-  const historyEvents: RealtimeItem[][] = [];
-  session.on('history_updated', (h) => historyEvents.push([...h]));
-
-  (transport as any)._onMessage({
-    data: JSON.stringify({
-      type: 'response.output_item.done',
-      event_id: 'e',
-      item: {
-        id: 'm2',
-        type: 'message',
-        role: 'assistant',
-        status: 'completed',
-        content: [{ type: 'text', text: 'hi again' }],
-      },
-      output_index: 0,
-      response_id: 'r2',
-    }),
+
+  it('preserves explicit completed status on done', async () => {
+    class TestTransport extends OpenAIRealtimeBase {
+      status: 'connected' | 'disconnected' | 'connecting' | 'disconnecting' = 'connected';
+      connect = vi.fn(async () => { });
+      sendEvent = vi.fn(); mute = vi.fn(); close = vi.fn(); interrupt = vi.fn();
+      get muted() { return false; }
+    }
+    const transport = new TestTransport();
+    const session = new RealtimeSession(new RealtimeAgent({ name: 'A', handoffs: [] }), { transport });
+    await session.connect({ apiKey: 'test' });
+
+    const historyEvents: RealtimeItem[][] = [];
+    session.on('history_updated', (h) => historyEvents.push([...h]));
+
+    (transport as any)._onMessage({
+      data: JSON.stringify({
+        type: 'response.output_item.done',
+        event_id: 'e',
+        item: {
+          id: 'm2',
+          type: 'message',
+          role: 'assistant',
+          status: 'completed',
+          content: [{ type: 'text', text: 'hi again' }],
+        },
+        output_index: 0,
+        response_id: 'r2',
+      }),
+    });
+
+    const latest = historyEvents.at(-1)!;
+    const msg = latest.find(
+      (i): i is Extract<RealtimeItem, { type: 'message'; role: 'assistant' }> =>
+        i.type === 'message' && i.role === 'assistant' && (i as any).itemId === 'm2'
+    );
+    expect(msg).toBeDefined();
+    expect(msg!.status).toBe('completed'); // ensure we didn't overwrite server status
+  });
+
+  it('includes tools in session config when session config is provided', async () => {
+    const transport = new FakeTransport();
+    const agent = new RealtimeAgent({
+      name: 'TestAgent',
+      handoffs: [],
+      tools: [TEST_TOOL]
+    });
+
+    // Test with session config - tools should still be included
+    const session = new RealtimeSession(agent, {
+      transport,
+      config: {
+        voice: 'alloy',
+        turnDetection: { type: 'server_vad' }
+      }
+    });
+
+    await session.connect({ apiKey: 'test' });
+
+    // Check that the initial session config includes tools
+    const connectCall = transport.connectCalls[0];
+    expect(connectCall?.initialSessionConfig?.tools).toBeDefined();
+    expect(connectCall?.initialSessionConfig?.tools).toHaveLength(1);
+    expect(connectCall?.initialSessionConfig?.tools?.[0]?.name).toBe('test');
+
+    // Check that voice config is also preserved
+    expect(connectCall?.initialSessionConfig?.voice).toBe('alloy');
+    expect(connectCall?.initialSessionConfig?.turnDetection).toEqual({ type: 'server_vad' });
   });
 
-  const latest = historyEvents.at(-1)!;
-  const msg = latest.find(
-    (i): i is Extract<RealtimeItem, { type: 'message'; role: 'assistant' }> =>
-      i.type === 'message' && i.role === 'assistant' && (i as any).itemId === 'm2'
-  );
-  expect(msg).toBeDefined();
-  expect(msg!.status).toBe('completed'); // ensure we didn't overwrite server status
-});
+  it('includes tools in session config when no session config is provided', async () => {
+    const transport = new FakeTransport();
+    const agent = new RealtimeAgent({
+      name: 'TestAgent',
+      handoffs: [],
+      tools: [TEST_TOOL]
+    });
+
+    // Test without session config - tools should be included
+    const session = new RealtimeSession(agent, { transport });
+
+    await session.connect({ apiKey: 'test' });
+
+    // Check that the initial session config includes tools
+    const connectCall = transport.connectCalls[0];
+    expect(connectCall?.initialSessionConfig?.tools).toBeDefined();
+    expect(connectCall?.initialSessionConfig?.tools).toHaveLength(1);
+    expect(connectCall?.initialSessionConfig?.tools?.[0]?.name).toBe('test');
+  });
+
+  it('preserves tools when updateSessionConfig is called', async () => {
+    const transport = new FakeTransport();
+    const agent = new RealtimeAgent({
+      name: 'TestAgent',
+      handoffs: [],
+      tools: [TEST_TOOL]
+    });
+
+    const session = new RealtimeSession(agent, {
+      transport,
+      config: {
+        voice: 'alloy'
+      }
+    });
+
+    await session.connect({ apiKey: 'test' });
+
+    // Check that updateSessionConfig calls include tools
+    expect(transport.updateSessionConfigCalls.length).toBeGreaterThan(0);
+    const lastUpdateCall = transport.updateSessionConfigCalls[transport.updateSessionConfigCalls.length - 1];
+    expect(lastUpdateCall.tools).toBeDefined();
+    expect(lastUpdateCall.tools).toHaveLength(1);
+    expect(lastUpdateCall.tools?.[0]?.name).toBe('test');
+  });
+
+  it('does not include tools field when no tools are provided', async () => {
+    const transport = new FakeTransport();
+    const agent = new RealtimeAgent({
+      name: 'TestAgent',
+      handoffs: [],
+      tools: [] // No tools
+    });
+
+    const session = new RealtimeSession(agent, {
+      transport,
+      config: {
+        voice: 'alloy'
+      }
+    });
+
+    await session.connect({ apiKey: 'test' });
+
+    // Check that updateSessionConfig calls do not include tools field
+    expect(transport.updateSessionConfigCalls.length).toBeGreaterThan(0);
+    const lastUpdateCall = transport.updateSessionConfigCalls[transport.updateSessionConfigCalls.length - 1];
+    expect(lastUpdateCall.hasOwnProperty('tools')).toBe(false);
+  });
+
+  it('reproduces the original issue - tools work with config provided', async () => {
+    const transport = new FakeTransport();
+    const agent = new RealtimeAgent({
+      name: 'TestAgent',
+      handoffs: [],
+      tools: [TEST_TOOL]
+    });
+
+    // This is the scenario from the issue - session with config that includes voice setting
+    const session = new RealtimeSession(agent, {
+      transport,
+      config: {
+        voice: 'alloy', // Even just voice setting should not break tools
+        turnDetection: { type: 'server_vad' }
+      }
+    });
+
+    await session.connect({ apiKey: 'test' });
+
+    // Verify that tools are included in both initial and update configs
+    const connectCall = transport.connectCalls[0];
+    expect(connectCall?.initialSessionConfig?.tools).toBeDefined();
+    expect(connectCall?.initialSessionConfig?.tools).toHaveLength(1);
+    expect(connectCall?.initialSessionConfig?.tools?.[0]?.name).toBe('test');
+
+    // Verify that subsequent updates also include tools
+    expect(transport.updateSessionConfigCalls.length).toBeGreaterThan(0);
+    const lastUpdateCall = transport.updateSessionConfigCalls[transport.updateSessionConfigCalls.length - 1];
+    expect(lastUpdateCall.tools).toBeDefined();
+    expect(lastUpdateCall.tools).toHaveLength(1);
+    expect(lastUpdateCall.tools?.[0]?.name).toBe('test');
+
+    // Verify that voice config is preserved
+    expect(connectCall?.initialSessionConfig?.voice).toBe('alloy');
+    expect(lastUpdateCall.voice).toBe('alloy');
+  });
 });