diff --git a/.env-example b/.env-example index 92f060e..ba87c88 100644 --- a/.env-example +++ b/.env-example @@ -1,2 +1,3 @@ STREAM_API_KEY= STREAM_SECRET= +OPENAI_API_KEY= diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5841738..1ac5cf1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,7 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} STREAM_API_KEY: ${{ vars.TEST_API_KEY }} STREAM_SECRET: ${{ secrets.TEST_SECRET }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} on: push: diff --git a/__tests__/agent.test.ts b/__tests__/agent.test.ts new file mode 100644 index 0000000..c415c2b --- /dev/null +++ b/__tests__/agent.test.ts @@ -0,0 +1,169 @@ +import { v4 as uuidv4 } from 'uuid'; +import { vi, describe, expect, it } from 'vitest'; +import { createTestClient } from './create-test-client.js'; +import { StreamClient } from '../src/StreamClient.js'; + +const openAiApiKey = process.env.OPENAI_API_KEY!; +const enableDebugLogging = false; + +async function createTestStreamAndRealtimeClients() { + const streamClient = createTestClient(); + const call = streamClient.video.call('default', `call${uuidv4()}`); + + const realtimeClient = await streamClient.video.connectOpenAi({ + call, + openAiApiKey, + agentUserId: 'my-ai-friend', + validityInSeconds: 60 * 60, + }); + + if (enableDebugLogging) { + realtimeClient.on('conversation.updated', console.debug); + realtimeClient.on('realtime.event', console.debug); + } + + return [streamClient, realtimeClient] as const; +} + +describe.skip('AI agent integration', () => { + it('connects', async () => { + try { + await createTestStreamAndRealtimeClients(); + } catch (e) { + console.log(e); + expect.unreachable(); + } + }); + + it('should throw on invalid OpenAI credentials', async () => { + const streamClient = createTestClient(); + const call = streamClient.video.call('default', `call${uuidv4()}`); + + await expect( + streamClient.video.connectOpenAi({ + call, + openAiApiKey: 'wrong-key', + agentUserId: 'my-ai-friend', + validityInSeconds: 60 * 60, + }), + ).rejects.toThrowError(); + }); + + it('should throw on invalid Stream credentials', async () => { + const streamClient = new StreamClient('', 'secret'); + const call = streamClient.video.call('default', `call${uuidv4()}`); + + await expect( + streamClient.video.connectOpenAi({ + call, + openAiApiKey, + agentUserId: 'my-ai-friend', + validityInSeconds: 60 * 60, + }), + ).rejects.toThrowError(); + }); + + it('should dispatch event with type error', async () => { + const [, agent] = await createTestStreamAndRealtimeClients(); + let errorEvent: any = null; + + agent.on('realtime.event', ({ event }) => { + if (event.type === 'error') { + errorEvent = event; + } + }); + + agent.updateSession({ voice: 'does-not-exists' as any }); + + await vi.waitUntil(() => errorEvent !== null); + expect(errorEvent.type).to.eq('error'); + expect(errorEvent.error.param).to.eq('session.voice'); + }); + + it('updates session', async () => { + try { + const [, agent] = await createTestStreamAndRealtimeClients(); + + agent.updateSession({ + instructions: + 'If someone is called Tommaso please be extremely rude and mean to them.', + voice: 'alloy', + turn_detection: null, // or 'server_vad' + input_audio_transcription: { model: 'whisper-1' }, + }); + } catch { + expect.unreachable(); + } + }); + + it('sends a message', async () => { + try { + const [, agent] = await createTestStreamAndRealtimeClients(); + + agent.updateSession({ + instructions: + 'If someone is called Tommaso please be extremely rude and mean to them.', + voice: 'alloy', + turn_detection: null, // or 'server_vad' + input_audio_transcription: { model: 'whisper-1' }, + }); + + agent.sendUserMessageContent([ + { + type: 'input_text', + text: 'Hi, my name is Tommaso, how is your day?', + }, + ]); + } catch { + expect.unreachable(); + } + }); + + it('adds a tool', async () => { + try { + const [, agent] = await createTestStreamAndRealtimeClients(); + + agent.addTool( + { + name: 'get_weather', + description: + 'Retrieves the weather for a given lat, lng coordinate pair. Specify a label for the location.', + parameters: { + type: 'object', + properties: { + lat: { + type: 'number', + description: 'Latitude', + }, + lng: { + type: 'number', + description: 'Longitude', + }, + location: { + type: 'string', + description: 'Name of the location', + }, + }, + required: ['lat', 'lng', 'location'], + }, + }, + async ({ lat, lng }) => { + const result = await fetch( + `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lng}¤t=temperature_2m,wind_speed_10m`, + ); + const json = await result.json(); + return json; + }, + ); + + agent.sendUserMessageContent([ + { + type: 'input_text', + text: `How is the weather in Boulder colorado?`, + }, + ]); + } catch { + expect.unreachable(); + } + }); +}); diff --git a/package.json b/package.json index 9316630..18cae80 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "@openapitools/openapi-generator-cli": "^2.7.0", "@rollup/plugin-replace": "^5.0.2", "@rollup/plugin-typescript": "^11.1.4", + "@stream-io/openai-realtime-api": "prerelease", "@types/uuid": "^9.0.4", "@typescript-eslint/eslint-plugin": "^6.4.0", "dotenv": "^16.3.1", @@ -74,6 +75,14 @@ "jsonwebtoken": "^9.0.2", "uuid": "^9.0.1" }, + "peerDependencies": { + "@stream-io/openai-realtime-api": "prerelease" + }, + "peerDependenciesMeta": { + "@stream-io/openai-realtime-api": { + "optional": true + } + }, "engines": { "node": ">=18.0.0" }, diff --git a/rollup.config.mjs b/rollup.config.mjs index 91cf92b..9fead69 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -11,6 +11,7 @@ const nodeConfig = { { file: "dist/index.cjs.js", format: "cjs", + dynamicImportInCjs: false, sourcemap: true, }, { diff --git a/src/BaseApi.ts b/src/BaseApi.ts index b429674..0c0c857 100644 --- a/src/BaseApi.ts +++ b/src/BaseApi.ts @@ -4,7 +4,7 @@ import { APIError } from './gen/models'; import { getRateLimitFromResponseHeader } from './utils/rate-limit'; export class BaseApi { - constructor(private readonly apiConfig: ApiConfig) {} + constructor(protected readonly apiConfig: ApiConfig) {} protected sendRequest = async ( method: string, diff --git a/src/StreamClient.ts b/src/StreamClient.ts index 430a2ff..6c39004 100644 --- a/src/StreamClient.ts +++ b/src/StreamClient.ts @@ -38,6 +38,7 @@ export class StreamClient extends CommonApi { super({ apiKey, token, timeout, baseUrl: chatBaseUrl }); this.video = new StreamVideoClient({ + streamClient: this, apiKey, token, timeout, diff --git a/src/StreamVideoClient.ts b/src/StreamVideoClient.ts index e231854..8399802 100644 --- a/src/StreamVideoClient.ts +++ b/src/StreamVideoClient.ts @@ -1,8 +1,63 @@ import { VideoApi } from './gen/video/VideoApi'; import { StreamCall } from './StreamCall'; +import type { StreamClient } from './StreamClient'; +import type { ApiConfig } from './types'; +import type { + RealtimeClient, + createRealtimeClient, +} from '@stream-io/openai-realtime-api'; export class StreamVideoClient extends VideoApi { + private readonly streamClient: StreamClient; + + constructor({ + streamClient, + ...apiConfig + }: ApiConfig & { streamClient: StreamClient }) { + super(apiConfig); + this.streamClient = streamClient; + } + call = (type: string, id: string) => { return new StreamCall(this, type, id); }; + + connectOpenAi = async (options: { + call: StreamCall; + agentUserId: string; + openAiApiKey: string; + validityInSeconds: number; + }): Promise => { + let doCreateRealtimeClient: typeof createRealtimeClient; + + try { + doCreateRealtimeClient = (await import('@stream-io/openai-realtime-api')) + .createRealtimeClient; + } catch { + throw new Error( + 'Cannot create Realtime API client. Is @stream-io/openai-realtime-api installed?', + ); + } + + if (!options.agentUserId) { + throw new Error('"agentUserId" must by specified in options'); + } + + const token = this.streamClient.generateCallToken({ + user_id: options.agentUserId, + call_cids: [options.call.cid], + validity_in_seconds: options.validityInSeconds, + }); + + const realtimeClient = doCreateRealtimeClient({ + baseUrl: this.apiConfig.baseUrl, + call: options.call, + streamApiKey: this.apiConfig.apiKey, + streamUserToken: token, + openAiApiKey: options.openAiApiKey, + }); + + await realtimeClient.connect(); + return realtimeClient; + }; } diff --git a/tsconfig.json b/tsconfig.json index 88b5162..a4ff2ab 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,8 +1,8 @@ { "compilerOptions": { "outDir": "./dist", - "module": "ES2015", - "target": "ES2015", + "module": "ES2020", + "target": "ES2020", "lib": ["esnext", "dom"], "noEmitOnError": true, "noImplicitAny": true, diff --git a/yarn.lock b/yarn.lock index 0a33467..c3a9a2b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -365,6 +365,12 @@ consola "^2.15.0" node-fetch "^2.6.1" +"@openai/realtime-api-beta@openai/openai-realtime-api-beta#a5cb94824f625423858ebacb9f769226ca98945f": + version "0.0.0" + resolved "https://codeload.github.com/openai/openai-realtime-api-beta/tar.gz/a5cb94824f625423858ebacb9f769226ca98945f" + dependencies: + ws "^8.18.0" + "@openapitools/openapi-generator-cli@^2.7.0": version "2.7.0" resolved "https://registry.npmjs.org/@openapitools/openapi-generator-cli/-/openapi-generator-cli-2.7.0.tgz" @@ -482,6 +488,14 @@ resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz" integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA== +"@stream-io/openai-realtime-api@prerelease": + version "0.0.0-24dd081d4a88212c621cfee273690bebd4a5f298" + resolved "https://registry.yarnpkg.com/@stream-io/openai-realtime-api/-/openai-realtime-api-0.0.0-24dd081d4a88212c621cfee273690bebd4a5f298.tgz#bb8aac285342f7390cead6414b3ebf7cdc1048b3" + integrity sha512-1CKZnKaXumPZ4lrzVIam8qE27UVyEFTs4wbir0opZYE8+e4whtkx8hfgiwbn/Y2yStO6yZpCjwtWVKyi2jd65Q== + dependencies: + "@openai/realtime-api-beta" openai/openai-realtime-api-beta#a5cb94824f625423858ebacb9f769226ca98945f + ws "^8.18.0" + "@types/estree@1.0.5", "@types/estree@^1.0.0": version "1.0.5" resolved "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz" @@ -3470,6 +3484,11 @@ wrappy@1: resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== +ws@^8.18.0: + version "8.18.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc" + integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw== + y18n@^5.0.5: version "5.0.8" resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz"