fix: emit audio_start event when audio begins

Author: vrtnis

packages/agents-realtime/src/realtimeSession.ts

@@ -182,6 +182,7 @@ export class RealtimeSession<
   #history: RealtimeItem[] = [];
   #shouldIncludeAudioData: boolean;
   #interruptedByGuardrail: Record<string, boolean> = {};
+  #audioStarted = false;
 
   constructor(
     public readonly initialAgent:
@@ -531,9 +532,14 @@ export class RealtimeSession<
       }
     });
     this.#transport.on('audio', (event) => {
+      if (!this.#audioStarted) {
+        this.#audioStarted = true;
+        this.emit('audio_start', this.#context, this.#currentAgent);
+      }
       this.emit('audio', event);
     });
     this.#transport.on('turn_started', () => {
+      this.#audioStarted = false;
       this.emit('agent_start', this.#context, this.#currentAgent);
       this.#currentAgent.emit('agent_start', this.#context, this.#currentAgent);
     });
@@ -548,6 +554,9 @@ export class RealtimeSession<
     });
 
     this.#transport.on('audio_done', () => {
+      if (this.#audioStarted) {
+        this.#audioStarted = false;
+      }
       this.emit('audio_stopped', this.#context, this.#currentAgent);
     });
 
@@ -648,6 +657,9 @@ export class RealtimeSession<
     });
 
     this.#transport.on('audio_interrupted', () => {
+      if (this.#audioStarted) {
+        this.#audioStarted = false;
+      }
       this.emit('audio_interrupted', this.#context, this.#currentAgent);
     });
 

packages/agents-realtime/test/realtimeSession.test.ts

@@ -267,4 +267,29 @@ describe('RealtimeSession', () => {
     transport.emit('audio_interrupted');
     expect(audioEvents).toBe(1);
   });
+
+  it('emits audio_start when audio begins', () => {
+    let startEvents = 0;
+    session.on('audio_start', () => startEvents++);
+    transport.emit('turn_started', {} as any);
+    transport.emit('audio', {
+      type: 'audio',
+      data: new ArrayBuffer(1),
+      responseId: 'r',
+    } as any);
+    transport.emit('audio', {
+      type: 'audio',
+      data: new ArrayBuffer(1),
+      responseId: 'r',
+    } as any);
+    expect(startEvents).toBe(1);
+    transport.emit('audio_done');
+    transport.emit('turn_started', {} as any);
+    transport.emit('audio', {
+      type: 'audio',
+      data: new ArrayBuffer(1),
+      responseId: 'r2',
+    } as any);
+    expect(startEvents).toBe(2);
+  });
 });