Improve the types of RealtimeAgent configuration

- inputAudioTranscription
- turnDetection

Author: seratch

.changeset/dull-beans-arrive.md

@@ -0,0 +1,5 @@
+---
+'@openai/agents-realtime': patch
+---
+
+Improve the types of turnDetection and inputAudioTranscription in RealtimeAgent configuration

examples/docs/voice-agents/turnDetection.ts

@@ -7,8 +7,8 @@ const session = new RealtimeSession(agent, {
     turnDetection: {
       type: 'semantic_vad',
       eagerness: 'medium',
-      create_response: true,
-      interrupt_response: true,
+      createResponse: true,
+      interruptResponse: true,
     },
   },
 });

packages/agents-realtime/src/clientMessages.ts

@@ -33,15 +33,46 @@ export type RealtimeTracingConfig =
     }
   | 'auto';
 
+export type RealtimeInputAudioTranscriptionConfig = {
+  language?: string;
+  model?: 'gpt-4o-transcribe' | 'gpt-4o-mini-transcribe' | 'whisper-1' | string;
+  prompt?: string;
+};
+
+export type RealtimeTurnDetectionConfigAsIs = {
+  type?: 'semantic_vad';
+  create_response?: boolean;
+  eagerness?: 'auto' | 'low' | 'medium' | 'high';
+  interrupt_response?: boolean;
+  prefix_padding_ms?: number;
+  silence_duration_ms?: number;
+  threshold?: number;
+};
+
+// The Realtime API accepts snake_cased keys, so when using this, this SDK coverts the keys to snake_case ones before passing it to the API
+export type RealtimeTurnDetectionConfigCamelCase = {
+  type?: 'semantic_vad';
+  createResponse?: boolean;
+  eagerness?: 'auto' | 'low' | 'medium' | 'high';
+  interruptResponse?: boolean;
+  prefixPaddingMs?: number;
+  silenceDurationMs?: number;
+  threshold?: number;
+};
+
+export type RealtimeTurnDetectionConfig =
+  | RealtimeTurnDetectionConfigAsIs
+  | RealtimeTurnDetectionConfigCamelCase;
+
 export type RealtimeSessionConfig = {
   model: string;
   instructions: string;
   modalities: ('text' | 'audio')[];
   voice: string;
   inputAudioFormat: RealtimeAudioFormat;
   outputAudioFormat: RealtimeAudioFormat;
-  inputAudioTranscription: Record<string, any>;
-  turnDetection: Record<string, any>;
+  inputAudioTranscription: RealtimeInputAudioTranscriptionConfig;
+  turnDetection: RealtimeTurnDetectionConfig;
   toolChoice: ModelSettingsToolChoice;
   tools: FunctionToolDefinition[];
   tracing?: RealtimeTracingConfig | null;

packages/agents-realtime/src/openaiRealtimeBase.ts

@@ -5,6 +5,8 @@ import {
   RealtimeClientMessage,
   RealtimeSessionConfig,
   RealtimeTracingConfig,
+  RealtimeTurnDetectionConfig,
+  RealtimeTurnDetectionConfigAsIs,
   RealtimeUserInput,
 } from './clientMessages';
 import {
@@ -390,7 +392,7 @@ export abstract class OpenAIRealtimeBase
         config.inputAudioTranscription ??
         DEFAULT_OPENAI_REALTIME_SESSION_CONFIG.inputAudioTranscription,
       turn_detection:
-        config.turnDetection ??
+        OpenAIRealtimeBase.buildTurnDetectionConfig(config.turnDetection) ??
         DEFAULT_OPENAI_REALTIME_SESSION_CONFIG.turnDetection,
       tool_choice:
         config.toolChoice ?? DEFAULT_OPENAI_REALTIME_SESSION_CONFIG.toolChoice,
@@ -406,6 +408,43 @@ export abstract class OpenAIRealtimeBase
     return sessionData;
   }
 
+  private static buildTurnDetectionConfig(
+    c: RealtimeTurnDetectionConfig | undefined,
+  ): RealtimeTurnDetectionConfigAsIs | undefined {
+    if (typeof c === 'undefined') {
+      return undefined;
+    }
+    return {
+      type: c.type,
+      create_response:
+        'createResponse' in c
+          ? c.createResponse
+          : 'create_response' in c
+            ? c.create_response
+            : undefined,
+      eagerness: c.eagerness,
+      interrupt_response:
+        'interruptResponse' in c
+          ? c.interruptResponse
+          : 'interrupt_response' in c
+            ? c.interrupt_response
+            : undefined,
+      prefix_padding_ms:
+        'prefixPaddingMs' in c
+          ? c.prefixPaddingMs
+          : 'prefix_padding_ms' in c
+            ? c.prefix_padding_ms
+            : undefined,
+      silence_duration_ms:
+        'silenceDurationMs' in c
+          ? c.silenceDurationMs
+          : 'silence_duration_ms' in c
+            ? c.silence_duration_ms
+            : undefined,
+      threshold: c.threshold,
+    };
+  }
+
   /**
    * Sets the internal tracing config. This is used to track the tracing config that has been set
    * during the session.create event.