realtime function call

Author: Oceania2018

src/lib/services/api-endpoints.js

@@ -46,6 +46,7 @@ export const endpoints = {
 
     // agent realtime interaction
     agentInitRealtimeSessionUrl: `${host}/agent/{agentId}/realtime/session`,
+    agentFunctionCallUrl: `${host}/agent/{agentId}/function/{functionName}/execute`,
 
     // router
     routerSettingUrl: `${host}/router/settings`,

src/lib/services/llm-realtime-service.js

@@ -1,12 +1,16 @@
 import { endpoints } from '$lib/services/api-endpoints.js';
 import { replaceUrl } from '$lib/helpers/http';
 import axios from 'axios';
+import { json } from '@sveltejs/kit';
 
 export const llmRealtime = {
     /** @type {RTCPeerConnection} */
     pc: new RTCPeerConnection(),
-    /** @param {string} agentId */
-    async start(agentId) {
+    /** 
+     * @param {string} agentId 
+     * @param {function} onMessageReceived
+    */
+    async start(agentId, onMessageReceived) {
         const session = await initRealtimeSession(agentId);
         const EPHEMERAL_KEY = session.client_secret.value;
 
@@ -25,10 +29,13 @@ export const llmRealtime = {
 
         // Set up data channel for sending and receiving events
         const dc = this.pc.createDataChannel("oai-events");
-        dc.addEventListener("message", (e) => {
+        dc.addEventListener("message", async (e) => {
             // Realtime server events appear here!
             var data = JSON.parse(e.data);
             console.log(data);
+            if (data.type === "response.done" && data.response.status == "completed") {
+                await handleServerEvents(agentId, data, dc);
+            }
         });
 
         // Start the session using the Session Description Protocol (SDP)
@@ -51,6 +58,20 @@ export const llmRealtime = {
           sdp: await sdpResponse.text(),
         };
         await this.pc.setRemoteDescription(answer);
+
+        // send on data channel connect is open
+        dc.onopen = () => {
+            console.log("Data channel is open");
+            /*const responseCreate = {
+                type: "response.create",
+                response: {
+                modalities: ["audio", "text"],
+                instructions: "Write a haiku about code",
+                },
+            };
+            dc.send(JSON.stringify(responseCreate));*/
+        };
+
     },
 
     stop() {
@@ -68,4 +89,58 @@ export async function initRealtimeSession(agentId) {
     let url = replaceUrl(endpoints.agentInitRealtimeSessionUrl, {agentId: agentId});
     var response = await axios.get(url);
     return response.data;
+}
+
+/**
+ * Handle server events
+ * @param {string} agentId
+ * @param {Object} data
+ * @param {RTCDataChannel} dc
+ */
+async function handleServerEvents(agentId, data, dc) {
+    // for each response.output, do something with it
+    data.response.output.forEach(async completion => {
+        if (completion.type === "function_call") {
+            const result = await callFunction(agentId, completion.name, completion.arguments);
+            console.log(result);
+
+            // send the result back to the model
+            const conversationItemCreation = {
+                "type": "conversation.item.create",
+                "item": {
+                    "call_id": completion.call_id,
+                    "type": "function_call_output",
+                    "output": result
+                }
+            };
+            dc.send(JSON.stringify(conversationItemCreation));
+
+            /*const conversationItemResponse = {
+                "type": "response.create",
+                "response": {
+                    "modalities": ["text", "audio"],
+                    "instructions": "Please continue the conversation based on the function output.",
+                }
+            };
+            dc.send(JSON.stringify(conversationItemResponse));*/
+        }
+    });
+}
+
+/**
+ * Execute agent function call
+ * @param {string} agentId
+ * @param {string} functionName
+ * @param {string} args
+ * @returns {Promise<string>}
+ */
+async function callFunction(agentId, functionName, args) {
+    let url = replaceUrl(endpoints.agentFunctionCallUrl, {
+        agentId: agentId,
+        functionName: functionName
+    });
+    
+    const functionArgs = JSON.parse(args);
+    var response = await axios.post(url, functionArgs);
+    return JSON.stringify(response.data);
 }

src/routes/chat/[agentId]/[conversationId]/chat-box.svelte

@@ -659,7 +659,9 @@
 		if (disableSpeech) return;
 
 		if (!isListening) {
-			llmRealtime.start(params.agentId);
+			llmRealtime.start(params.agentId, message => {
+				console.log(message);
+			});
 			isListening = true;
 			microphoneIcon = "microphone";
 		} else {