Merge branch 'main' into add-custom-fetch-mcp-streamable-http

alonhar · web-flow · commit 02a3ca455e7e · 2025-08-11T09:55:56.000+03:00
diff --git a/.changeset/angry-cooks-wait.md b/.changeset/angry-cooks-wait.md
@@ -0,0 +1,7 @@
+---
+"@openai/agents-core": patch
+"@openai/agents-extensions": patch
+"@openai/agents-openai": patch
+---
+
+Fix #233 - eliminate confusion with "input_text" type items with role: "assistant"
diff --git a/.changeset/brave-swans-smile.md b/.changeset/brave-swans-smile.md
@@ -0,0 +1,5 @@
+---
+'@openai/agents-openai': patch
+---
+
+Pass through strict flag for function tools when using completion
diff --git a/.changeset/cool-cloths-fry.md b/.changeset/cool-cloths-fry.md
@@ -0,0 +1,6 @@
+---
+'@openai/agents-openai': patch
+'@openai/agents-core': patch
+---
+
+fix: support snake_case usage fields from OpenAI responses
diff --git a/.changeset/preserve-audio-formats-on-agent-update.md b/.changeset/preserve-audio-formats-on-agent-update.md
@@ -0,0 +1,5 @@
+---
+'@openai/agents-realtime': patch
+---
+
+fix(realtime-session): preserve audio format & other session config fields on agent update
diff --git a/docs/src/content/docs/extensions/ai-sdk.mdx b/docs/src/content/docs/extensions/ai-sdk.mdx
@@ -30,7 +30,7 @@ of supported models that can be brought into the Agents SDK through this adapter
 2. Choose your desired model package from the [Vercel's AI SDK](https://sdk.vercel.ai/docs/models/overview) and install it:
 
    ```bash
-   npm install @ai-sdk/openai
+   npm install @ai-sdk/openai@"^1.0.0"
    ```
 
 3. Import the adapter and model to connect to your agent:
@@ -48,6 +48,11 @@ of supported models that can be brought into the Agents SDK through this adapter
 
 </Steps>
 
+<Aside type="caution">
+  Vercel's AI SDK has recently migrated to v2, however openai agent extensions is not yet compatible with v2.
+  Therefore v1 versions of Vercel's AI SDK has to be installed.
+</Aside>
+
 ## Example
 
 <Code lang="typescript" code={aiSdkSetupExample} title="AI SDK Setup" />
diff --git a/packages/agents-core/src/types/protocol.ts b/packages/agents-core/src/types/protocol.ts
@@ -241,7 +241,6 @@ export type ComputerAction = z.infer<typeof computerActions>;
 export const AssistantContent = z.discriminatedUnion('type', [
   OutputText,
   Refusal,
-  InputText,
   AudioContent,
   ImageContent,
 ]);
diff --git a/packages/agents-core/src/usage.ts b/packages/agents-core/src/usage.ts
@@ -1,5 +1,15 @@
 import { UsageData } from './types/protocol';
 
+type UsageInput = Partial<
+  UsageData & {
+    input_tokens: number;
+    output_tokens: number;
+    total_tokens: number;
+    input_tokens_details: object;
+    output_tokens_details: object;
+  }
+> & { requests?: number };
+
 /**
  * Tracks token usage and request counts for an agent run.
  */
@@ -34,7 +44,7 @@ export class Usage {
    */
   public outputTokensDetails: Array<Record<string, number>> = [];
 
-  constructor(input?: Partial<UsageData> & { requests?: number }) {
+  constructor(input?: UsageInput) {
     if (typeof input === 'undefined') {
       this.requests = 0;
       this.inputTokens = 0;
@@ -44,14 +54,18 @@ export class Usage {
       this.outputTokensDetails = [];
     } else {
       this.requests = input?.requests ?? 1;
-      this.inputTokens = input?.inputTokens ?? 0;
-      this.outputTokens = input?.outputTokens ?? 0;
-      this.totalTokens = input?.totalTokens ?? 0;
-      this.inputTokensDetails = input?.inputTokensDetails
-        ? [input.inputTokensDetails]
+      this.inputTokens = input?.inputTokens ?? input?.input_tokens ?? 0;
+      this.outputTokens = input?.outputTokens ?? input?.output_tokens ?? 0;
+      this.totalTokens = input?.totalTokens ?? input?.total_tokens ?? 0;
+      const inputTokensDetails =
+        input?.inputTokensDetails ?? input?.input_tokens_details;
+      this.inputTokensDetails = inputTokensDetails
+        ? [inputTokensDetails as Record<string, number>]
         : [];
-      this.outputTokensDetails = input?.outputTokensDetails
-        ? [input.outputTokensDetails]
+      const outputTokensDetails =
+        input?.outputTokensDetails ?? input?.output_tokens_details;
+      this.outputTokensDetails = outputTokensDetails
+        ? [outputTokensDetails as Record<string, number>]
         : [];
     }
   }
diff --git a/packages/agents-core/test/usage.test.ts b/packages/agents-core/test/usage.test.ts
@@ -26,6 +26,24 @@ describe('Usage', () => {
     expect(usage.totalTokens).toBe(15);
   });
 
+  it('falls back to snake_case fields', () => {
+    const usage = new Usage({
+      requests: 2,
+      input_tokens: 7,
+      output_tokens: 3,
+      total_tokens: 10,
+      input_tokens_details: { foo: 1 },
+      output_tokens_details: { bar: 2 },
+    });
+
+    expect(usage.requests).toBe(2);
+    expect(usage.inputTokens).toBe(7);
+    expect(usage.outputTokens).toBe(3);
+    expect(usage.totalTokens).toBe(10);
+    expect(usage.inputTokensDetails).toEqual([{ foo: 1 }]);
+    expect(usage.outputTokensDetails).toEqual([{ bar: 2 }]);
+  });
+
   it('adds other Usage instances correctly', () => {
     const usageA = new Usage({
       inputTokens: 1,
diff --git a/packages/agents-extensions/src/aiSdk.ts b/packages/agents-extensions/src/aiSdk.ts
@@ -116,29 +116,16 @@ export function itemsToLanguageV1Messages(
         messages.push({
           role,
           content: content
-            .filter((c) => c.type === 'input_text' || c.type === 'output_text')
+            .filter((c) => c.type === 'output_text')
             .map((c) => {
               const { providerData: contentProviderData } = c;
-              if (c.type === 'output_text') {
-                return {
-                  type: 'text',
-                  text: c.text,
-                  providerMetadata: {
-                    ...(contentProviderData ?? {}),
-                  },
-                };
-              }
-              if (c.type === 'input_text') {
-                return {
-                  type: 'text',
-                  text: c.text,
-                  providerMetadata: {
-                    ...(contentProviderData ?? {}),
-                  },
-                };
-              }
-              const exhaustiveCheck = c satisfies never;
-              throw new UserError(`Unknown content type: ${exhaustiveCheck}`);
+              return {
+                type: 'text',
+                text: c.text,
+                providerMetadata: {
+                  ...(contentProviderData ?? {}),
+                },
+              };
             }),
           providerMetadata: {
             ...(providerData ?? {}),
diff --git a/packages/agents-openai/src/openaiChatCompletionsConverter.ts b/packages/agents-openai/src/openaiChatCompletionsConverter.ts
@@ -37,7 +37,7 @@ export function extractAllAssistantContent(
   }
   const out: ChatCompletionAssistantMessageParam['content'] = [];
   for (const c of content) {
-    if (c.type === 'output_text' || c.type === 'input_text') {
+    if (c.type === 'output_text') {
       out.push({
         type: 'text',
         text: c.text,
@@ -277,6 +277,7 @@ export function toolToOpenAI(tool: SerializedTool): ChatCompletionTool {
         name: tool.name,
         description: tool.description || '',
         parameters: tool.parameters,
+        strict: tool.strict,
       },
     };
   }
diff --git a/packages/agents-openai/test/openaiChatCompletionsConverter.test.ts b/packages/agents-openai/test/openaiChatCompletionsConverter.test.ts
@@ -253,6 +253,7 @@ describe('tool helpers', () => {
           required: [],
           additionalProperties: false,
         },
+        strict: true,
       },
     });
   });
diff --git a/packages/agents-openai/test/openaiChatCompletionsModel.test.ts b/packages/agents-openai/test/openaiChatCompletionsModel.test.ts
@@ -79,6 +79,32 @@ describe('OpenAIChatCompletionsModel', () => {
     ]);
   });
 
+  it('parses usage tokens from snake_case fields', async () => {
+    const client = new FakeClient();
+    const response = {
+      id: 'r',
+      choices: [{ message: { content: 'hi' } }],
+      usage: { prompt_tokens: 11, completion_tokens: 7, total_tokens: 18 },
+    } as any;
+    client.chat.completions.create.mockResolvedValue(response);
+
+    const model = new OpenAIChatCompletionsModel(client as any, 'gpt');
+    const req: any = {
+      input: 'u',
+      modelSettings: {},
+      tools: [],
+      outputType: 'text',
+      handoffs: [],
+      tracing: false,
+    };
+
+    const result = await withTrace('t', () => model.getResponse(req));
+
+    expect(result.usage.inputTokens).toBe(11);
+    expect(result.usage.outputTokens).toBe(7);
+    expect(result.usage.totalTokens).toBe(18);
+  });
+
   it('outputs message when content is empty string', async () => {
     const client = new FakeClient();
     const response = {
diff --git a/packages/agents-realtime/src/realtimeSession.ts b/packages/agents-realtime/src/realtimeSession.ts
@@ -183,6 +183,13 @@ export class RealtimeSession<
   #shouldIncludeAudioData: boolean;
   #interruptedByGuardrail: Record<string, boolean> = {};
   #audioStarted = false;
+  // Keeps track of the last full session config we sent (camelCase keys) so that
+  // subsequent updates (e.g. during agent handoffs) preserve properties that are
+  // not explicitly recalculated here (such as inputAudioFormat, outputAudioFormat,
+  // modalities, speed, toolChoice, turnDetection, etc.). Without this, updating
+  // the agent would drop audio format overrides (e.g. g711_ulaw) and revert to
+  // transport defaults causing issues for integrations like Twilio.
+  #lastSessionConfig: Partial<RealtimeSessionConfig> | null = null;
 
   constructor(
     public readonly initialAgent:
@@ -313,14 +320,34 @@ export class RealtimeSession<
       );
     }
 
-    return {
+    // Start from any previously-sent config (so we preserve values like audio formats)
+    // and the original options.config provided by the user. Preference order:
+    // 1. Last session config we sent (#lastSessionConfig)
+    // 2. Original options.config
+    // 3. Additional config passed into this invocation (explicit overrides)
+    // Finally we overwrite dynamic fields (instructions, voice, model, tools, tracing)
+    // to ensure they always reflect the current agent & runtime state.
+    const base: Partial<RealtimeSessionConfig> = {
+      ...(this.#lastSessionConfig ?? {}),
+      ...(this.options.config ?? {}),
+      ...(additionalConfig ?? {}),
+    };
+
+    // Note: Certain fields cannot be updated after the session begins, such as voice and model
+    const fullConfig: Partial<RealtimeSessionConfig> = {
+      ...base,
       instructions,
       voice: this.#currentAgent.voice,
       model: this.options.model,
       tools: this.#currentTools,
       tracing: tracingConfig,
-      ...additionalConfig,
     };
+
+    // Update our cache so subsequent updates inherit the full set including any
+    // dynamic fields we just overwrote.
+    this.#lastSessionConfig = fullConfig;
+
+    return fullConfig;
   }
 
   async updateAgent(newAgent: RealtimeAgent<TBaseContext>) {
@@ -689,6 +716,9 @@ export class RealtimeSession<
       url: options.url,
       initialSessionConfig: await this.#getSessionConfig(this.options.config),
     });
+    // Ensure the cached lastSessionConfig includes everything passed as the initial session config
+    // (the call above already set it via #getSessionConfig but in case additional overrides were
+    // passed directly here in the future we could merge them). For now it's a no-op.
 
     this.#history = [];
     this.emit('history_updated', this.#history);
diff --git a/packages/agents-realtime/test/realtimeSession.test.ts b/packages/agents-realtime/test/realtimeSession.test.ts
@@ -300,4 +300,20 @@ describe('RealtimeSession', () => {
     } as any);
     expect(startEvents).toBe(2);
   });
+
+  it('preserves custom audio formats across updateAgent', async () => {
+    const t = new FakeTransport();
+    const agent = new RealtimeAgent({ name: 'Orig', handoffs: [] });
+    const s = new RealtimeSession(agent, {
+      transport: t,
+      config: { inputAudioFormat: 'g711_ulaw', outputAudioFormat: 'g711_ulaw' },
+    });
+    await s.connect({ apiKey: 'test' });
+    const newAgent = new RealtimeAgent({ name: 'Next', handoffs: [] });
+    await s.updateAgent(newAgent);
+    // Find the last updateSessionConfig call
+    const last = t.updateSessionConfigCalls.at(-1)!;
+    expect(last.inputAudioFormat).toBe('g711_ulaw');
+    expect(last.outputAudioFormat).toBe('g711_ulaw');
+  });
 });

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +'@openai/agents-openai': patch
 +---
++
 +Pass through strict flag for function tools when using completion