feat: add a method to connect OpenAI agent to call

.env-example

@@ -1,2 +1,3 @@
 STREAM_API_KEY=<API key>
 STREAM_SECRET=<SECRET>
+OPENAI_API_KEY=<OpenAI API key>

.github/workflows/test.yml

@@ -3,6 +3,7 @@ env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   STREAM_API_KEY: ${{ vars.TEST_API_KEY }}
   STREAM_SECRET: ${{ secrets.TEST_SECRET }}
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
 on:
   push:

__tests__/agent.test.ts

@@ -0,0 +1,169 @@
+import { v4 as uuidv4 } from 'uuid';
+import { vi, describe, expect, it } from 'vitest';
+import { createTestClient } from './create-test-client.js';
+import { StreamClient } from '../src/StreamClient.js';
+
+const openAiApiKey = process.env.OPENAI_API_KEY!;
+const enableDebugLogging = false;
+
+async function createTestStreamAndRealtimeClients() {
+  const streamClient = createTestClient();
+  const call = streamClient.video.call('default', `call${uuidv4()}`);
+
+  const realtimeClient = await streamClient.video.connectOpenAi({
+    call,
+    openAiApiKey,
+    agentUserId: 'my-ai-friend',
+    validityInSeconds: 60 * 60,
+  });
+
+  if (enableDebugLogging) {
+    realtimeClient.on('conversation.updated', console.debug);
+    realtimeClient.on('realtime.event', console.debug);
+  }
+
+  return [streamClient, realtimeClient] as const;
+}
+
+describe.skip('AI agent integration', () => {
+  it('connects', async () => {
+    try {
+      await createTestStreamAndRealtimeClients();
+    } catch (e) {
+      console.log(e);
+      expect.unreachable();
+    }
+  });
+
+  it('should throw on invalid OpenAI credentials', async () => {
+    const streamClient = createTestClient();
+    const call = streamClient.video.call('default', `call${uuidv4()}`);
+
+    await expect(
+      streamClient.video.connectOpenAi({
+        call,
+        openAiApiKey: 'wrong-key',
+        agentUserId: 'my-ai-friend',
+        validityInSeconds: 60 * 60,
+      }),
+    ).rejects.toThrowError();
+  });
+
+  it('should throw on invalid Stream credentials', async () => {
+    const streamClient = new StreamClient('', 'secret');
+    const call = streamClient.video.call('default', `call${uuidv4()}`);
+
+    await expect(
+      streamClient.video.connectOpenAi({
+        call,
+        openAiApiKey,
+        agentUserId: 'my-ai-friend',
+        validityInSeconds: 60 * 60,
+      }),
+    ).rejects.toThrowError();
+  });
+
+  it('should dispatch event with type error', async () => {
+    const [, agent] = await createTestStreamAndRealtimeClients();
+    let errorEvent: any = null;
+
+    agent.on('realtime.event', ({ event }) => {
+      if (event.type === 'error') {
+        errorEvent = event;
+      }
+    });
+
+    agent.updateSession({ voice: 'does-not-exists' as any });
+
+    await vi.waitUntil(() => errorEvent !== null);
+    expect(errorEvent.type).to.eq('error');
+    expect(errorEvent.error.param).to.eq('session.voice');
+  });
+
+  it('updates session', async () => {
+    try {
+      const [, agent] = await createTestStreamAndRealtimeClients();
+
+      agent.updateSession({
+        instructions:
+          'If someone is called Tommaso please be extremely rude and mean to them.',
+        voice: 'alloy',
+        turn_detection: null, // or 'server_vad'
+        input_audio_transcription: { model: 'whisper-1' },
+      });
+    } catch {
+      expect.unreachable();
+    }
+  });
+
+  it('sends a message', async () => {
+    try {
+      const [, agent] = await createTestStreamAndRealtimeClients();
+
+      agent.updateSession({
+        instructions:
+          'If someone is called Tommaso please be extremely rude and mean to them.',
+        voice: 'alloy',
+        turn_detection: null, // or 'server_vad'
+        input_audio_transcription: { model: 'whisper-1' },
+      });
+
+      agent.sendUserMessageContent([
+        {
+          type: 'input_text',
+          text: 'Hi, my name is Tommaso, how is your day?',
+        },
+      ]);
+    } catch {
+      expect.unreachable();
+    }
+  });
+
+  it('adds a tool', async () => {
+    try {
+      const [, agent] = await createTestStreamAndRealtimeClients();
+
+      agent.addTool(
+        {
+          name: 'get_weather',
+          description:
+            'Retrieves the weather for a given lat, lng coordinate pair. Specify a label for the location.',
+          parameters: {
+            type: 'object',
+            properties: {
+              lat: {
+                type: 'number',
+                description: 'Latitude',
+              },
+              lng: {
+                type: 'number',
+                description: 'Longitude',
+              },
+              location: {
+                type: 'string',
+                description: 'Name of the location',
+              },
+            },
+            required: ['lat', 'lng', 'location'],
+          },
+        },
+        async ({ lat, lng }) => {
+          const result = await fetch(
+            `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lng}&current=temperature_2m,wind_speed_10m`,
+          );
+          const json = await result.json();
+          return json;
+        },
+      );
+
+      agent.sendUserMessageContent([
+        {
+          type: 'input_text',
+          text: `How is the weather in Boulder colorado?`,
+        },
+      ]);
+    } catch {
+      expect.unreachable();
+    }
+  });
+});

package.json

@@ -49,6 +49,7 @@
     "@openapitools/openapi-generator-cli": "^2.7.0",
     "@rollup/plugin-replace": "^5.0.2",
     "@rollup/plugin-typescript": "^11.1.4",
+    "@stream-io/openai-realtime-api": "prerelease",
     "@types/uuid": "^9.0.4",
     "@typescript-eslint/eslint-plugin": "^6.4.0",
     "dotenv": "^16.3.1",
@@ -74,6 +75,14 @@
     "jsonwebtoken": "^9.0.2",
     "uuid": "^9.0.1"
   },
+  "peerDependencies": {
+    "@stream-io/openai-realtime-api": "prerelease"
+  },
+  "peerDependenciesMeta": {
+    "@stream-io/openai-realtime-api": {
+      "optional": true
+    }
+  },
   "engines": {
     "node": ">=18.0.0"
   },

rollup.config.mjs

@@ -11,6 +11,7 @@ const nodeConfig = {
     {
       file: "dist/index.cjs.js",
       format: "cjs",
+      dynamicImportInCjs: false,
       sourcemap: true,
     },
     {

src/BaseApi.ts

@@ -4,7 +4,7 @@ import { APIError } from './gen/models';
 import { getRateLimitFromResponseHeader } from './utils/rate-limit';
 
 export class BaseApi {
-  constructor(private readonly apiConfig: ApiConfig) {}
+  constructor(protected readonly apiConfig: ApiConfig) {}
 
   protected sendRequest = async <T>(
     method: string,

src/StreamClient.ts

@@ -38,6 +38,7 @@ export class StreamClient extends CommonApi {
     super({ apiKey, token, timeout, baseUrl: chatBaseUrl });
 
     this.video = new StreamVideoClient({
+      streamClient: this,
       apiKey,
       token,
       timeout,

src/StreamVideoClient.ts

@@ -1,8 +1,63 @@
 import { VideoApi } from './gen/video/VideoApi';
 import { StreamCall } from './StreamCall';
+import type { StreamClient } from './StreamClient';
+import type { ApiConfig } from './types';
+import type {
+  RealtimeClient,
+  createRealtimeClient,
+} from '@stream-io/openai-realtime-api';
 
 export class StreamVideoClient extends VideoApi {
+  private readonly streamClient: StreamClient;
+
+  constructor({
+    streamClient,
+    ...apiConfig
+  }: ApiConfig & { streamClient: StreamClient }) {
+    super(apiConfig);
+    this.streamClient = streamClient;
+  }
+
   call = (type: string, id: string) => {
     return new StreamCall(this, type, id);
   };
+
+  connectOpenAi = async (options: {
+    call: StreamCall;
+    agentUserId: string;
+    openAiApiKey: string;
+    validityInSeconds: number;
+  }): Promise<RealtimeClient> => {
+    let doCreateRealtimeClient: typeof createRealtimeClient;
+
+    try {
+      doCreateRealtimeClient = (await import('@stream-io/openai-realtime-api'))
+        .createRealtimeClient;
+    } catch {
+      throw new Error(
+        'Cannot create Realtime API client. Is @stream-io/openai-realtime-api installed?',
+      );
+    }
+
+    if (!options.agentUserId) {
+      throw new Error('"agentUserId" must by specified in options');
+    }
+
+    const token = this.streamClient.generateCallToken({
+      user_id: options.agentUserId,
+      call_cids: [options.call.cid],
+      validity_in_seconds: options.validityInSeconds,
+    });
+
+    const realtimeClient = doCreateRealtimeClient({
+      baseUrl: this.apiConfig.baseUrl,
+      call: options.call,
+      streamApiKey: this.apiConfig.apiKey,
+      streamUserToken: token,
+      openAiApiKey: options.openAiApiKey,
+    });
+
+    await realtimeClient.connect();
+    return realtimeClient;
+  };
 }

tsconfig.json

@@ -1,8 +1,8 @@
 {
   "compilerOptions": {
     "outDir": "./dist",
-    "module": "ES2015",
-    "target": "ES2015",
+    "module": "ES2020",
+    "target": "ES2020",
     "lib": ["esnext", "dom"],
     "noEmitOnError": true,
     "noImplicitAny": true,

yarn.lock

@@ -365,6 +365,12 @@
     consola "^2.15.0"
     node-fetch "^2.6.1"
 
+"@openai/realtime-api-beta@openai/openai-realtime-api-beta#a5cb94824f625423858ebacb9f769226ca98945f":
+  version "0.0.0"
+  resolved "https://codeload.github.com/openai/openai-realtime-api-beta/tar.gz/a5cb94824f625423858ebacb9f769226ca98945f"
+  dependencies:
+    ws "^8.18.0"
+
 "@openapitools/openapi-generator-cli@^2.7.0":
   version "2.7.0"
   resolved "https://registry.npmjs.org/@openapitools/openapi-generator-cli/-/openapi-generator-cli-2.7.0.tgz"
@@ -482,6 +488,14 @@
   resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz"
   integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==
 
+"@stream-io/openai-realtime-api@prerelease":
+  version "0.0.0-24dd081d4a88212c621cfee273690bebd4a5f298"
+  resolved "https://registry.yarnpkg.com/@stream-io/openai-realtime-api/-/openai-realtime-api-0.0.0-24dd081d4a88212c621cfee273690bebd4a5f298.tgz#bb8aac285342f7390cead6414b3ebf7cdc1048b3"
+  integrity sha512-1CKZnKaXumPZ4lrzVIam8qE27UVyEFTs4wbir0opZYE8+e4whtkx8hfgiwbn/Y2yStO6yZpCjwtWVKyi2jd65Q==
+  dependencies:
+    "@openai/realtime-api-beta" openai/openai-realtime-api-beta#a5cb94824f625423858ebacb9f769226ca98945f
+    ws "^8.18.0"
+
 "@types/[email protected]", "@types/estree@^1.0.0":
   version "1.0.5"
   resolved "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz"
@@ -3470,6 +3484,11 @@ wrappy@1:
   resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz"
   integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
 
+ws@^8.18.0:
+  version "8.18.0"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc"
+  integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
+
 y18n@^5.0.5:
   version "5.0.8"
   resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz"