feat: add a method to connect OpenAI agent to call

Author: myandrienko

package.json

@@ -49,6 +49,7 @@
     "@openapitools/openapi-generator-cli": "^2.7.0",
     "@rollup/plugin-replace": "^5.0.2",
     "@rollup/plugin-typescript": "^11.1.4",
+    "@stream-io/openai-realtime-api": "prerelease",
     "@types/uuid": "^9.0.4",
     "@typescript-eslint/eslint-plugin": "^6.4.0",
     "dotenv": "^16.3.1",
@@ -74,6 +75,9 @@
     "jsonwebtoken": "^9.0.2",
     "uuid": "^9.0.1"
   },
+  "peerDependencies": {
+    "@stream-io/openai-realtime-api": "prerelease"
+  },
   "engines": {
     "node": ">=18.0.0"
   },

rollup.config.mjs

@@ -11,6 +11,7 @@ const nodeConfig = {
     {
       file: "dist/index.cjs.js",
       format: "cjs",
+      dynamicImportInCjs: false,
       sourcemap: true,
     },
     {

src/BaseApi.ts

@@ -4,7 +4,7 @@ import { APIError } from './gen/models';
 import { getRateLimitFromResponseHeader } from './utils/rate-limit';
 
 export class BaseApi {
-  constructor(private readonly apiConfig: ApiConfig) {}
+  constructor(protected readonly apiConfig: ApiConfig) {}
 
   protected sendRequest = async <T>(
     method: string,

src/StreamClient.ts

@@ -38,6 +38,7 @@ export class StreamClient extends CommonApi {
     super({ apiKey, token, timeout, baseUrl: chatBaseUrl });
 
     this.video = new StreamVideoClient({
+      streamClient: this,
       apiKey,
       token,
       timeout,

src/StreamVideoClient.ts

@@ -1,8 +1,63 @@
 import { VideoApi } from './gen/video/VideoApi';
 import { StreamCall } from './StreamCall';
+import type { StreamClient } from './StreamClient';
+import type { ApiConfig } from './types';
+import type {
+  RealtimeClient,
+  createRealtimeClient,
+} from '@stream-io/openai-realtime-api';
 
 export class StreamVideoClient extends VideoApi {
+  private streamClient: StreamClient;
+
+  constructor({
+    streamClient,
+    ...apiConfig
+  }: ApiConfig & { streamClient: StreamClient }) {
+    super(apiConfig);
+    this.streamClient = streamClient;
+  }
+
   call = (type: string, id: string) => {
     return new StreamCall(this, type, id);
   };
+
+  connectOpenAi = async (options: {
+    call: StreamCall;
+    agentUserId: string;
+    openAiApiKey: string;
+    validityInSeconds: number;
+  }): Promise<RealtimeClient> => {
+    let doCreateRealtimeClient: typeof createRealtimeClient;
+
+    try {
+      doCreateRealtimeClient = (await import('@stream-io/openai-realtime-api'))
+        .createRealtimeClient;
+    } catch {
+      throw new Error(
+        'Cannot create Realtime API client. Is @stream-io/openai-realtime-api installed?',
+      );
+    }
+
+    if (!options.agentUserId) {
+      throw new Error('"agentUserId" must by specified in options');
+    }
+
+    const token = this.streamClient.generateCallToken({
+      user_id: options.agentUserId,
+      call_cids: [options.call.cid],
+      validity_in_seconds: options.validityInSeconds,
+    });
+
+    const realtimeClient = doCreateRealtimeClient({
+      baseUrl: this.apiConfig.baseUrl,
+      call: options.call,
+      streamApiKey: this.apiConfig.apiKey,
+      streamUserToken: token,
+      openAiApiKey: options.openAiApiKey,
+    });
+
+    await realtimeClient.connect();
+    return realtimeClient;
+  };
 }

tsconfig.json

@@ -1,8 +1,8 @@
 {
   "compilerOptions": {
     "outDir": "./dist",
-    "module": "ES2015",
-    "target": "ES2015",
+    "module": "ES2020",
+    "target": "ES2020",
     "lib": ["esnext", "dom"],
     "noEmitOnError": true,
     "noImplicitAny": true,

yarn.lock

@@ -365,6 +365,12 @@
     consola "^2.15.0"
     node-fetch "^2.6.1"
 
+"@openai/realtime-api-beta@openai/openai-realtime-api-beta#a5cb94824f625423858ebacb9f769226ca98945f":
+  version "0.0.0"
+  resolved "https://codeload.github.com/openai/openai-realtime-api-beta/tar.gz/a5cb94824f625423858ebacb9f769226ca98945f"
+  dependencies:
+    ws "^8.18.0"
+
 "@openapitools/openapi-generator-cli@^2.7.0":
   version "2.7.0"
   resolved "https://registry.npmjs.org/@openapitools/openapi-generator-cli/-/openapi-generator-cli-2.7.0.tgz"
@@ -482,6 +488,14 @@
   resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz"
   integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==
 
+"@stream-io/openai-realtime-api@prerelease":
+  version "0.0.0-24dd081d4a88212c621cfee273690bebd4a5f298"
+  resolved "https://registry.yarnpkg.com/@stream-io/openai-realtime-api/-/openai-realtime-api-0.0.0-24dd081d4a88212c621cfee273690bebd4a5f298.tgz#bb8aac285342f7390cead6414b3ebf7cdc1048b3"
+  integrity sha512-1CKZnKaXumPZ4lrzVIam8qE27UVyEFTs4wbir0opZYE8+e4whtkx8hfgiwbn/Y2yStO6yZpCjwtWVKyi2jd65Q==
+  dependencies:
+    "@openai/realtime-api-beta" openai/openai-realtime-api-beta#a5cb94824f625423858ebacb9f769226ca98945f
+    ws "^8.18.0"
+
 "@types/[email protected]", "@types/estree@^1.0.0":
   version "1.0.5"
   resolved "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz"
@@ -3470,6 +3484,11 @@ wrappy@1:
   resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz"
   integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==
 
+ws@^8.18.0:
+  version "8.18.0"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc"
+  integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
+
 y18n@^5.0.5:
   version "5.0.8"
   resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz"