Skip to content

Commit 787968b

Browse files
threepointonethreepointone
authored
fix: use web standard event apis for twilio websocket (#127)
* fix: use web standard event apis for twilio websocket the twilio websocket extension used the node `ws` specific apis for attaching event handlers, which made it throw in a cloudflare environment. this patch adds a fix to the types, and uses `.addEventListener` on the websocket instead of `.on`. this also adds a fix to load the right shim for workerd when using `@openai/agents-realtime` * add a changeset * Update shiny-berries-kiss.md * pass tests * pass test * use websocket protocols in workerd when making a new websocket connection
1 parent c248a7d commit 787968b

File tree

11 files changed

+122
-66
lines changed

11 files changed

+122
-66
lines changed

‎.changeset/shiny-berries-kiss.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
'@openai/agents': patch
3+
'@openai/agents-extensions': patch
4+
'@openai/agents-realtime': patch
5+
---
6+
7+
fix: use web standard event apis for twilio websocket

‎examples/docs/extensions/twilio-basic.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ const agent = new RealtimeAgent({
99
// the OpenAI Realtime API.
1010
const twilioTransport = new TwilioRealtimeTransportLayer({
1111
// @ts-expect-error - this is not defined
12-
twilioWebSocket: websoketConnection,
12+
twilioWebSocket: websocketConnection,
1313
});
1414

1515
const session = new RealtimeSession(agent, {

‎packages/agents-extensions/src/TwilioRealtimeTransport.ts

Lines changed: 76 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,13 @@ import {
77
RealtimeSessionConfig,
88
} from '@openai/agents/realtime';
99
import { getLogger } from '@openai/agents';
10-
import type { WebSocket, MessageEvent } from 'ws';
10+
import type {
11+
WebSocket as NodeWebSocket,
12+
MessageEvent as NodeMessageEvent,
13+
ErrorEvent as NodeErrorEvent,
14+
} from 'ws';
15+
16+
import type { ErrorEvent } from 'undici-types';
1117

1218
/**
1319
* The options for the Twilio Realtime Transport Layer.
@@ -18,7 +24,7 @@ export type TwilioRealtimeTransportLayerOptions =
1824
* The websocket that is receiving messages from Twilio's Media Streams API. Typically the
1925
* connection gets passed into your request handler when running your WebSocket server.
2026
*/
21-
twilioWebSocket: WebSocket;
27+
twilioWebSocket: WebSocket | NodeWebSocket;
2228
};
2329

2430
/**
@@ -48,7 +54,7 @@ export type TwilioRealtimeTransportLayerOptions =
4854
* ```
4955
*/
5056
export class TwilioRealtimeTransportLayer extends OpenAIRealtimeWebSocket {
51-
#twilioWebSocket: WebSocket;
57+
#twilioWebSocket: WebSocket | NodeWebSocket;
5258
#streamSid: string | null = null;
5359
#audioChunkCount: number = 0;
5460
#lastPlayedChunkCount: number = 0;
@@ -82,74 +88,80 @@ export class TwilioRealtimeTransportLayer extends OpenAIRealtimeWebSocket {
8288
options.initialSessionConfig,
8389
);
8490
// listen to Twilio messages as quickly as possible
85-
this.#twilioWebSocket.on('message', (message: MessageEvent) => {
86-
try {
87-
const data = JSON.parse(message.toString());
88-
if (this.#logger.dontLogModelData) {
89-
this.#logger.debug('Twilio message:', data.event);
90-
} else {
91-
this.#logger.debug('Twilio message:', data);
92-
}
93-
this.emit('*', {
94-
type: 'twilio_message',
95-
message: data,
96-
});
97-
switch (data.event) {
98-
case 'media':
99-
if (this.status === 'connected') {
100-
this.sendAudio(utils.base64ToArrayBuffer(data.media.payload));
101-
}
102-
break;
103-
case 'mark':
104-
if (
105-
!data.mark.name.startsWith('done:') &&
106-
data.mark.name.includes(':')
107-
) {
108-
// keeping track of what the last chunk was that the user heard fully
109-
const count = Number(data.mark.name.split(':')[1]);
110-
if (Number.isFinite(count)) {
111-
this.#lastPlayedChunkCount = count;
112-
} else {
113-
this.#logger.warn(
114-
'Invalid mark name received:',
115-
data.mark.name,
116-
);
91+
this.#twilioWebSocket.addEventListener(
92+
'message',
93+
(message: MessageEvent | NodeMessageEvent) => {
94+
try {
95+
const data = JSON.parse(message.data.toString());
96+
if (this.#logger.dontLogModelData) {
97+
this.#logger.debug('Twilio message:', data.event);
98+
} else {
99+
this.#logger.debug('Twilio message:', data);
100+
}
101+
this.emit('*', {
102+
type: 'twilio_message',
103+
message: data,
104+
});
105+
switch (data.event) {
106+
case 'media':
107+
if (this.status === 'connected') {
108+
this.sendAudio(utils.base64ToArrayBuffer(data.media.payload));
117109
}
118-
} else if (data.mark.name.startsWith('done:')) {
119-
this.#lastPlayedChunkCount = 0;
120-
}
121-
break;
122-
case 'start':
123-
this.#streamSid = data.start.streamSid;
124-
break;
125-
default:
126-
break;
110+
break;
111+
case 'mark':
112+
if (
113+
!data.mark.name.startsWith('done:') &&
114+
data.mark.name.includes(':')
115+
) {
116+
// keeping track of what the last chunk was that the user heard fully
117+
const count = Number(data.mark.name.split(':')[1]);
118+
if (Number.isFinite(count)) {
119+
this.#lastPlayedChunkCount = count;
120+
} else {
121+
this.#logger.warn(
122+
'Invalid mark name received:',
123+
data.mark.name,
124+
);
125+
}
126+
} else if (data.mark.name.startsWith('done:')) {
127+
this.#lastPlayedChunkCount = 0;
128+
}
129+
break;
130+
case 'start':
131+
this.#streamSid = data.start.streamSid;
132+
break;
133+
default:
134+
break;
135+
}
136+
} catch (error) {
137+
this.#logger.error(
138+
'Error parsing message:',
139+
error,
140+
'Message:',
141+
message,
142+
);
143+
this.emit('error', {
144+
type: 'error',
145+
error,
146+
});
127147
}
128-
} catch (error) {
129-
this.#logger.error(
130-
'Error parsing message:',
131-
error,
132-
'Message:',
133-
message,
134-
);
148+
},
149+
);
150+
this.#twilioWebSocket.addEventListener('close', () => {
151+
if (this.status !== 'disconnected') {
152+
this.close();
153+
}
154+
});
155+
this.#twilioWebSocket.addEventListener(
156+
'error',
157+
(error: ErrorEvent | NodeErrorEvent) => {
135158
this.emit('error', {
136159
type: 'error',
137160
error,
138161
});
139-
}
140-
});
141-
this.#twilioWebSocket.on('close', () => {
142-
if (this.status !== 'disconnected') {
143162
this.close();
144-
}
145-
});
146-
this.#twilioWebSocket.on('error', (error) => {
147-
this.emit('error', {
148-
type: 'error',
149-
error,
150-
});
151-
this.close();
152-
});
163+
},
164+
);
153165
this.on('audio_done', () => {
154166
this.#twilioWebSocket.send(
155167
JSON.stringify({

‎packages/agents-extensions/test/TwilioRealtimeTransport.test.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ import { describe, test, expect, vi, beforeEach } from 'vitest';
22
import { EventEmitter } from 'events';
33
import { TwilioRealtimeTransportLayer } from '../src/TwilioRealtimeTransport';
44

5+
import type { MessageEvent as NodeMessageEvent } from 'ws';
6+
import type { MessageEvent } from 'undici-types';
7+
58
vi.mock('@openai/agents/realtime', () => {
69
// eslint-disable-next-line @typescript-eslint/no-require-imports
710
const { EventEmitter } = require('events');
@@ -31,6 +34,14 @@ class FakeTwilioWebSocket extends EventEmitter {
3134
close = vi.fn();
3235
}
3336

37+
// @ts-expect-error - we're making the node event emitter compatible with the browser event emitter
38+
FakeTwilioWebSocket.prototype.addEventListener = function (
39+
type: string,
40+
listener: (evt: MessageEvent | NodeMessageEvent) => void,
41+
) {
42+
this.on(type, (evt) => listener(type === 'message' ? { data: evt } : evt));
43+
};
44+
3445
const base64 = (data: string) => Buffer.from(data).toString('base64');
3546

3647
describe('TwilioRealtimeTransportLayer', () => {

‎packages/agents-extensions/test/index.test.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { describe, test, expect, vi } from 'vitest';
22
import { EventEmitter } from 'events';
33
import { TwilioRealtimeTransportLayer } from '../src';
4+
import type { MessageEvent as NodeMessageEvent } from 'ws';
45

56
vi.mock('ws', () => {
67
class FakeWebSocket {
@@ -30,6 +31,14 @@ class FakeTwilioWebSocket extends EventEmitter {
3031
close = vi.fn();
3132
}
3233

34+
// @ts-expect-error - we're making the node event emitter compatible with the browser event emitter
35+
FakeTwilioWebSocket.prototype.addEventListener = function (
36+
type: string,
37+
listener: (evt: MessageEvent | NodeMessageEvent) => void,
38+
) {
39+
this.on(type, (evt) => listener(type === 'message' ? { data: evt } : evt));
40+
};
41+
3342
describe('TwilioRealtimeTransportLayer', () => {
3443
test('should be available', () => {
3544
const transport = new TwilioRealtimeTransportLayer({

‎packages/agents-realtime/package.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@
2323
"default": "./dist/index.mjs"
2424
},
2525
"./_shims": {
26+
"workerd": {
27+
"require": "./dist/shims/shims-workerd.js",
28+
"types": "./dist/shims/shims-workerd.d.ts",
29+
"default": "./dist/shims/shims-workerd.mjs"
30+
},
2631
"browser": {
2732
"require": "./dist/shims/shims-browser.js",
2833
"types": "./dist/shims/shims-browser.d.ts",

‎packages/agents-realtime/src/openaiRealtimeWebsocket.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {
22
isBrowserEnvironment,
3+
useWebSocketProtocols,
34
WebSocket,
45
} from '@openai/agents-realtime/_shims';
56
import {
@@ -149,7 +150,8 @@ export class OpenAIRealtimeWebSocket
149150
);
150151
}
151152

152-
const websocketArguments = isBrowserEnvironment()
153+
// browsers and workerd should use the protocols argument, node should use the headers argument
154+
const websocketArguments = useWebSocketProtocols
153155
? [
154156
'realtime',
155157
// Auth

‎packages/agents-realtime/src/shims/shims-browser.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ export const WebSocket = globalThis.WebSocket;
44
export function isBrowserEnvironment(): boolean {
55
return true;
66
}
7+
export const useWebSocketProtocols = true;

‎packages/agents-realtime/src/shims/shims-node.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ export { WebSocket } from 'ws';
22
export function isBrowserEnvironment(): boolean {
33
return false;
44
}
5+
export const useWebSocketProtocols = false;

‎packages/agents-realtime/src/shims/shims-workerd.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
export const WebSocket = globalThis.WebSocket;
2+
export function isBrowserEnvironment(): boolean {
3+
return false;
4+
}
5+
export const useWebSocketProtocols = true;

0 commit comments

Comments
 (0)