use correct object key: output_modalities instead of modalities

see https://platform.openai.com/docs/api-reference/realtime-beta-sessions/session_object

Author: simllll

plugins/openai/src/realtime/api_proto.ts

@@ -190,7 +190,7 @@ export interface SessionResource {
   id: string;
   object: 'realtime.session';
   model: string;
-  modalities: ['text', 'audio'] | ['text']; // default: ["text", "audio"]
+  output_modalities: ['text', 'audio'] | ['text'] | ['audio']; // default: ["text", "audio"]
   instructions: string;
   voice: Voice; // default: "alloy"
   input_audio_format: AudioFormat; // default: "pcm16"
@@ -267,7 +267,7 @@ export interface SessionUpdateEvent extends BaseClientEvent {
   type: 'session.update';
   session: Partial<{
     model: Model;
-    modalities: ['text', 'audio'] | ['text'];
+    output_modalities: ['text', 'audio'] | ['text'] | ['audio'];
     instructions: string;
     voice: Voice;
     input_audio_format: AudioFormat;
@@ -350,7 +350,7 @@ export interface ConversationItemDeleteEvent extends BaseClientEvent {
 export interface ResponseCreateEvent extends BaseClientEvent {
   type: 'response.create';
   response?: Partial<{
-    modalities: ['text', 'audio'] | ['text'];
+    output_modalities: ['text', 'audio'] | ['text'] | ['audio'];
     instructions: string;
     voice: Voice;
     output_audio_format: AudioFormat;

plugins/openai/src/realtime/realtime_model.ts

@@ -406,7 +406,10 @@ export class RealtimeSession extends llm.RealtimeSession {
         voice: this.oaiRealtimeModel._options.voice,
         input_audio_format: 'pcm16',
         output_audio_format: 'pcm16',
-        modalities: this.oaiRealtimeModel._options.modalities as ['text', 'audio'] | ['text'],
+        output_modalities: this.oaiRealtimeModel._options.modalities as
+          | ['text', 'audio']
+          | ['text']
+          | ['audio'],
         turn_detection: this.oaiRealtimeModel._options.turnDetection,
         input_audio_transcription: this.oaiRealtimeModel._options.inputAudioTranscription,
         // TODO(shubhra): add inputAudioNoiseReduction