Merge branch 'main' into main

seratch · web-flow · commit fcdfc801d00b · 2025-07-10T18:31:35.000+09:00
diff --git a/.changeset/funny-memes-cover.md b/.changeset/funny-memes-cover.md
@@ -0,0 +1,5 @@
+---
+'@openai/agents-realtime': patch
+---
+
+Correct typo in RealtimeTransportEventTypes in code and docs
diff --git a/.changeset/salty-actors-grin.md b/.changeset/salty-actors-grin.md
@@ -0,0 +1,5 @@
+---
+'@openai/agents-realtime': patch
+---
+
+Fixes issue #106 where overlapping user inputs caused null transcripts in history
diff --git a/docs/src/content/docs/guides/quickstart.mdx b/docs/src/content/docs/guides/quickstart.mdx
@@ -85,6 +85,8 @@ const historyFunFact = tool({
   name: 'history_fun_fact',
   // The description is used to describe **when** to use the tool by telling it **what** it does.
   description: 'Give a fun fact about a historical event',
+  // This tool takes no parameters, so we provide an empty Zod Object.
+  parameters: z.object({}),
   execute: async () => {
     // The output will be returned back to the Agent to use
     return 'Sharks are older than trees.';
diff --git a/docs/src/content/docs/guides/voice-agents/transport.mdx b/docs/src/content/docs/guides/voice-agents/transport.mdx
@@ -49,7 +49,7 @@ Use any recording/playback library to handle the raw PCM16 audio bytes.
 ### Building your own transport mechanism
 
 If you want to use a different speech-to-speech API or have your own custom transport mechanism, you
-can create your own by implementing the `RealtimeTransportLayer` interface and emit the `RealtimeTranportEventTypes` events.
+can create your own by implementing the `RealtimeTransportLayer` interface and emit the `RealtimeTransportEventTypes` events.
 
 ## Interacting with the Realtime API more directly
 
diff --git a/docs/src/content/docs/ja/guides/voice-agents/transport.mdx b/docs/src/content/docs/ja/guides/voice-agents/transport.mdx
@@ -45,7 +45,7 @@ WebRTC の代わりに WebSocket 接続を使用する場合は、セッショ
 
 ### 独自のトランスポートメカニズムの構築
 
-別の speech-to-speech API を使用したい場合や独自のトランスポートメカニズムを持っている場合は、`RealtimeTransportLayer` インターフェースを実装し、`RealtimeTranportEventTypes` イベントを発火することで独自実装が可能です。
+別の speech-to-speech API を使用したい場合や独自のトランスポートメカニズムを持っている場合は、`RealtimeTransportLayer` インターフェースを実装し、`RealtimeTransportEventTypes` イベントを発火することで独自実装が可能です。
 
 ## Realtime API との直接的なインタラクション
 
diff --git a/examples/nextjs/package.json b/examples/nextjs/package.json
@@ -16,7 +16,7 @@
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "lucide-react": "^0.515.0",
-    "next": "15.3.2",
+    "next": "15.3.3",
     "react": "^19.0.0",
     "react-dom": "^19.0.0",
     "tailwind-merge": "^3.3.1",
diff --git a/examples/nextjs/src/components/History.tsx b/examples/nextjs/src/components/History.tsx
@@ -74,7 +74,7 @@ function processItems(items: AgentInputItem[]): ProcessedItem[] {
                   return '';
                 })
                 .join('\n') || '⚫︎⚫︎⚫︎',
-        id: item.id ?? JSON.stringify(item.content),
+        id: item.id ?? '',
       });
     }
   }
@@ -100,7 +100,7 @@ export function History({ history }: HistoryProps) {
             <TextMessage
               text={item.content}
               isUser={item.role === 'user'}
-              key={item.id ?? idx}
+              key={item.id || idx}
             />
           );
         }
diff --git a/examples/realtime-next/package.json b/examples/realtime-next/package.json
@@ -13,7 +13,7 @@
     "@radix-ui/react-slot": "^1.2.3",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
-    "next": "15.3.2",
+    "next": "15.3.3",
     "react": "^19.0.0",
     "react-dom": "^19.0.0",
     "tailwind-merge": "^3.3.0",
diff --git a/packages/agents-core/src/runState.ts b/packages/agents-core/src/runState.ts
@@ -165,6 +165,8 @@ const serializedProcessedResponseSchema = z.object({
             arguments: z.string().optional(),
             status: z.string().optional(),
             output: z.string().optional(),
+            // this always exists but marked as optional for early version compatibility; when releasing 1.0, we can remove the nullable and optional
+            providerData: z.record(z.string(), z.any()).nullable().optional(),
           }),
         }),
         // HostedMCPTool
diff --git a/packages/agents-core/test/runState.test.ts b/packages/agents-core/test/runState.test.ts
@@ -278,7 +278,50 @@ describe('deserialize helpers', () => {
       functions: [],
       handoffs: [],
       computerActions: [{ toolCall: call, computer: tool }],
-      mcpApprovalRequests: [],
+      mcpApprovalRequests: [
+        {
+          requestItem: {
+            rawItem: {
+              type: 'hosted_tool_call',
+              name: 'fetch_generic_url_content',
+              status: 'in_progress',
+              providerData: {
+                id: 'mcpr_685bc3c47ed88192977549b5206db77504d4306d5de6ab36',
+                type: 'mcp_approval_request',
+                arguments:
+                  '{"url":"https://raw.githubusercontent.com/openai/codex/main/README.md"}',
+                name: 'fetch_generic_url_content',
+                server_label: 'gitmcp',
+              },
+            },
+            type: 'tool_approval_item',
+            agent: new Agent({ name: 'foo ' }),
+            toJSON: function (): any {
+              throw new Error('Function not implemented.');
+            },
+          },
+          mcpTool: {
+            type: 'hosted_tool',
+            name: 'hosted_mcp',
+            providerData: {
+              type: 'mcp',
+              server_label: 'gitmcp',
+              server_url: 'https://gitmcp.io/openai/codex',
+              require_approval: {
+                always: {
+                  tool_names: ['fetch_generic_url_content'],
+                },
+                never: {
+                  tool_names: [
+                    'search_codex_code',
+                    'fetch_codex_documentation',
+                  ],
+                },
+              },
+            },
+          },
+        },
+      ],
       toolsUsed: [],
       hasToolsOrApprovalsToRun: () => true,
     };
@@ -292,5 +335,15 @@ describe('deserialize helpers', () => {
     if (restored._currentStep?.type === 'next_step_handoff') {
       expect(restored._currentStep.newAgent).toBe(agent);
     }
+    expect(
+      restored._lastProcessedResponse?.mcpApprovalRequests[0].mcpTool,
+    ).toEqual(state._lastProcessedResponse?.mcpApprovalRequests[0].mcpTool);
+    expect(
+      restored._lastProcessedResponse?.mcpApprovalRequests[0].requestItem
+        .rawItem.providerData,
+    ).toEqual(
+      state._lastProcessedResponse?.mcpApprovalRequests[0].requestItem.rawItem
+        .providerData,
+    );
   });
 });
diff --git a/packages/agents-realtime/src/index.ts b/packages/agents-realtime/src/index.ts
@@ -19,7 +19,7 @@ export {
   TransportLayerTranscriptDelta,
   TransportError,
   TransportToolCallEvent,
-  RealtimeTranportEventTypes,
+  RealtimeTransportEventTypes,
 } from './transportLayerEvents';
 
 export {
diff --git a/packages/agents-realtime/src/openaiRealtimeBase.ts b/packages/agents-realtime/src/openaiRealtimeBase.ts
@@ -25,7 +25,7 @@ import {
   RealtimeTransportLayerConnectOptions,
 } from './transportLayer';
 import {
-  RealtimeTranportEventTypes,
+  RealtimeTransportEventTypes,
   TransportToolCallEvent,
 } from './transportLayerEvents';
 import { arrayBufferToBase64, diffRealtimeHistory } from './utils';
@@ -93,7 +93,7 @@ export type OpenAIRealtimeEventTypes = {
    * Triggered when the connection is closed.
    */
   disconnected: [];
-} & RealtimeTranportEventTypes;
+} & RealtimeTransportEventTypes;
 
 export abstract class OpenAIRealtimeBase
   extends EventEmitterDelegate<OpenAIRealtimeEventTypes>
diff --git a/packages/agents-realtime/src/realtimeSession.ts b/packages/agents-realtime/src/realtimeSession.ts
@@ -35,6 +35,7 @@ import { RealtimeAgent } from './realtimeAgent';
 import { RealtimeSessionEventTypes } from './realtimeSessionEvents';
 import type { ApiKey, RealtimeTransportLayer } from './transportLayer';
 import type { TransportToolCallEvent } from './transportLayerEvents';
+import type { InputAudioTranscriptionCompletedEvent } from './transportLayerEvents';
 import {
   getLastTextFromAudioOutputMessage,
   hasWebRTCSupport,
@@ -508,6 +509,26 @@ export class RealtimeSession<
   #setEventListeners() {
     this.#transport.on('*', (event) => {
       this.emit('transport_event', event);
+      // Handle completed user transcription events
+      if (
+        event.type === 'conversation.item.input_audio_transcription.completed'
+      ) {
+        try {
+          const completedEvent = event as InputAudioTranscriptionCompletedEvent;
+          this.#history = updateRealtimeHistory(
+            this.#history,
+            completedEvent,
+            this.#shouldIncludeAudioData,
+          );
+          this.#context.context.history = this.#history;
+          this.emit('history_updated', this.#history);
+        } catch (err) {
+          this.emit('error', {
+            type: 'error',
+            error: err,
+          });
+        }
+      }
     });
     this.#transport.on('audio', (event) => {
       this.emit('audio', event);
diff --git a/packages/agents-realtime/src/transportLayer.ts b/packages/agents-realtime/src/transportLayer.ts
@@ -6,7 +6,7 @@ import {
 } from './clientMessages';
 import { RealtimeItem } from './items';
 import {
-  RealtimeTranportEventTypes,
+  RealtimeTransportEventTypes,
   TransportToolCallEvent,
 } from './transportLayerEvents';
 
@@ -46,7 +46,7 @@ export type RealtimeTransportLayerConnectOptions = {
  * and the communication with the model.
  */
 export interface RealtimeTransportLayer
-  extends EventEmitter<RealtimeTranportEventTypes> {
+  extends EventEmitter<RealtimeTransportEventTypes> {
   status: 'connected' | 'disconnected' | 'connecting' | 'disconnecting';
 
   /**
diff --git a/packages/agents-realtime/src/transportLayerEvents.ts b/packages/agents-realtime/src/transportLayerEvents.ts
@@ -30,6 +30,16 @@ export type TransportLayerAudio = {
   responseId: string;
 };
 
+/**
+ * Event representing the completion of user audio transcription.
+ * Contains the finalized transcript string and the ID of the associated item.
+ */
+export type InputAudioTranscriptionCompletedEvent = {
+  type: 'conversation.item.input_audio_transcription.completed';
+  item_id: string;
+  transcript: string;
+};
+
 export type TransportLayerTranscriptDelta = {
   type: 'transcript_delta';
   itemId: string;
@@ -46,12 +56,13 @@ export type ConnectionStatus = 'connecting' | 'connected' | 'disconnected';
 export type TransportEvent =
   | TransportError
   | TransportToolCallEvent
+  | InputAudioTranscriptionCompletedEvent
   | {
       type: string;
       [key: string]: any;
     };
 
-export type RealtimeTranportEventTypes = {
+export type RealtimeTransportEventTypes = {
   /**
    * A raw event from the transport layer. Allows a user to tap directly into the events of the
    * transport layer.
diff --git a/packages/agents-realtime/src/utils.ts b/packages/agents-realtime/src/utils.ts
@@ -1,4 +1,5 @@
 import { RealtimeItem, RealtimeMessageItem } from './items';
+import type { InputAudioTranscriptionCompletedEvent } from './transportLayerEvents';
 import METADATA from './metadata';
 
 /**
@@ -182,9 +183,37 @@ export function removeAudioFromContent(
  */
 export function updateRealtimeHistory(
   history: RealtimeItem[],
-  event: RealtimeItem,
+  event: RealtimeItem | InputAudioTranscriptionCompletedEvent,
   shouldIncludeAudioData: boolean,
 ): RealtimeItem[] {
+  // Merge transcript into placeholder input_audio message
+  if (event.type === 'conversation.item.input_audio_transcription.completed') {
+    return history.map((item) => {
+      if (
+        item.itemId === event.item_id &&
+        item.type === 'message' &&
+        'role' in item &&
+        item.role === 'user'
+      ) {
+        const updatedContent = item.content.map((entry: any) => {
+          if (entry.type === 'input_audio') {
+            return {
+              ...entry,
+              transcript: event.transcript,
+            };
+          }
+          return entry;
+        });
+
+        return {
+          ...item,
+          content: updatedContent,
+          status: 'completed',
+        };
+      }
+      return item;
+    });
+  }
   const newEvent =
     !shouldIncludeAudioData && event.type === 'message'
       ? removeAudioFromContent(event as any)
diff --git a/packages/agents-realtime/test/stubs.ts b/packages/agents-realtime/test/stubs.ts
@@ -22,7 +22,7 @@ import type {
   RealtimeTransportLayerConnectOptions,
 } from '../src/transportLayer';
 import type { TransportToolCallEvent } from '../src/transportLayerEvents';
-import { RealtimeTranportEventTypes } from '../src/transportLayerEvents';
+import { RealtimeTransportEventTypes } from '../src/transportLayerEvents';
 
 export const TEST_MODEL_MESSAGE: protocol.AssistantMessageItem = {
   id: '123',
@@ -123,7 +123,7 @@ export class FakeModelProvider implements ModelProvider {
 }
 
 export class FakeTransport
-  extends EventEmitterDelegate<RealtimeTranportEventTypes>
+  extends EventEmitterDelegate<RealtimeTransportEventTypes>
   implements RealtimeTransportLayer
 {
   status: 'connected' | 'disconnected' | 'connecting' | 'disconnecting' =
@@ -135,7 +135,7 @@ export class FakeTransport
   sendAudioCalls: [ArrayBuffer, { commit?: boolean }][] = [];
   updateSessionConfigCalls: Partial<RealtimeSessionConfig>[] = [];
   closeCalls = 0;
-  eventEmitter = new RuntimeEventEmitter<RealtimeTranportEventTypes>();
+  eventEmitter = new RuntimeEventEmitter<RealtimeTransportEventTypes>();
   muteCalls: boolean[] = [];
   sendFunctionCallOutputCalls: [TransportToolCallEvent, string, boolean][] = [];
   interruptCalls = 0;
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +'@openai/agents-realtime': patch
 +---
++
 +Correct typo in RealtimeTransportEventTypes in code and docs
Original file line number	Diff line number	Diff line change
`@@ -74,7 +74,7 @@ function processItems(items: AgentInputItem[]): ProcessedItem[] {`
`74`	`74`	`return '';`
`75`	`75`	`})`
`76`	`76`	`.join('\n') \|\| '⚫︎⚫︎⚫︎',`
`77`		`- id: item.id ?? JSON.stringify(item.content),`
	`77`	`+ id: item.id ?? '',`
`78`	`78`	`});`
`79`	`79`	`}`
`80`	`80`	`}`
`@@ -100,7 +100,7 @@ export function History({ history }: HistoryProps) {`
`100`	`100`	`<TextMessage`
`101`	`101`	`text={item.content}`
`102`	`102`	`isUser={item.role === 'user'}`
`103`		`- key={item.id ?? idx}`
	`103`	`+ key={item.id \|\| idx}`
`104`	`104`	`/>`
`105`	`105`	`);`
`106`	`106`	`}`