fix: merge completed input_audio transcripts into realtime history (#151)

Author: vrtnis

.changeset/salty-actors-grin.md

@@ -0,0 +1,5 @@
+---
+'@openai/agents-realtime': patch
+---
+
+Fixes issue #106 where overlapping user inputs caused null transcripts in history

packages/agents-realtime/src/realtimeSession.ts

@@ -35,6 +35,7 @@ import { RealtimeAgent } from './realtimeAgent';
 import { RealtimeSessionEventTypes } from './realtimeSessionEvents';
 import type { ApiKey, RealtimeTransportLayer } from './transportLayer';
 import type { TransportToolCallEvent } from './transportLayerEvents';
+import type { InputAudioTranscriptionCompletedEvent } from './transportLayerEvents';
 import {
   getLastTextFromAudioOutputMessage,
   hasWebRTCSupport,
@@ -508,6 +509,26 @@ export class RealtimeSession<
   #setEventListeners() {
     this.#transport.on('*', (event) => {
       this.emit('transport_event', event);
+      // Handle completed user transcription events
+      if (
+        event.type === 'conversation.item.input_audio_transcription.completed'
+      ) {
+        try {
+          const completedEvent = event as InputAudioTranscriptionCompletedEvent;
+          this.#history = updateRealtimeHistory(
+            this.#history,
+            completedEvent,
+            this.#shouldIncludeAudioData,
+          );
+          this.#context.context.history = this.#history;
+          this.emit('history_updated', this.#history);
+        } catch (err) {
+          this.emit('error', {
+            type: 'error',
+            error: err,
+          });
+        }
+      }
     });
     this.#transport.on('audio', (event) => {
       this.emit('audio', event);

packages/agents-realtime/src/transportLayerEvents.ts

@@ -30,6 +30,16 @@ export type TransportLayerAudio = {
   responseId: string;
 };
 
+/**
+ * Event representing the completion of user audio transcription.
+ * Contains the finalized transcript string and the ID of the associated item.
+ */
+export type InputAudioTranscriptionCompletedEvent = {
+  type: 'conversation.item.input_audio_transcription.completed';
+  item_id: string;
+  transcript: string;
+};
+
 export type TransportLayerTranscriptDelta = {
   type: 'transcript_delta';
   itemId: string;
@@ -46,6 +56,7 @@ export type ConnectionStatus = 'connecting' | 'connected' | 'disconnected';
 export type TransportEvent =
   | TransportError
   | TransportToolCallEvent
+  | InputAudioTranscriptionCompletedEvent
   | {
       type: string;
       [key: string]: any;

packages/agents-realtime/src/utils.ts

@@ -1,4 +1,5 @@
 import { RealtimeItem, RealtimeMessageItem } from './items';
+import type { InputAudioTranscriptionCompletedEvent } from './transportLayerEvents';
 import METADATA from './metadata';
 
 /**
@@ -182,9 +183,37 @@ export function removeAudioFromContent(
  */
 export function updateRealtimeHistory(
   history: RealtimeItem[],
-  event: RealtimeItem,
+  event: RealtimeItem | InputAudioTranscriptionCompletedEvent,
   shouldIncludeAudioData: boolean,
 ): RealtimeItem[] {
+  // Merge transcript into placeholder input_audio message
+  if (event.type === 'conversation.item.input_audio_transcription.completed') {
+    return history.map((item) => {
+      if (
+        item.itemId === event.item_id &&
+        item.type === 'message' &&
+        'role' in item &&
+        item.role === 'user'
+      ) {
+        const updatedContent = item.content.map((entry: any) => {
+          if (entry.type === 'input_audio') {
+            return {
+              ...entry,
+              transcript: event.transcript,
+            };
+          }
+          return entry;
+        });
+
+        return {
+          ...item,
+          content: updatedContent,
+          status: 'completed',
+        };
+      }
+      return item;
+    });
+  }
   const newEvent =
     !shouldIncludeAudioData && event.type === 'message'
       ? removeAudioFromContent(event as any)