init realtime session

Author: Oceania2018

README.md

@@ -29,6 +29,9 @@ npm run dev
 
 # or start the server and open the app in a new browser tab
 npm run dev -- --open
+
+# or start the server with different .env
+npm run dev -- --mode botsharp
 ```
 
 You can override the `.env` values by creating a local env file named `.env.local` if needed.

src/lib/helpers/types/realtimeTypes.js

@@ -0,0 +1,15 @@
+/**
+ * @typedef {Object} RealtimeSession
+ * @property {string} id
+ * @property {string} object
+ * @property {string} model
+ * @property {RealtimeSessionClientSecret} clientSecret
+ */
+
+/**
+ * @typedef {Object} RealtimeSessionClientSecret
+ * @property {string} value
+ * @property {BigInt} expiresAt
+ */
+
+export default {};

src/lib/services/api-endpoints.js

@@ -44,6 +44,9 @@ export const endpoints = {
     // agent instruct
     instructCompletionUrl: `${host}/instruct/{agentId}`,
 
+    // agent realtime interaction
+    agentInitRealtimeSessionUrl: `${host}/agent/{agentId}/realtime/session`,
+
     // router
     routerSettingUrl: `${host}/router/settings`,
 

src/lib/services/llm-realtime-service.js

@@ -0,0 +1,71 @@
+import { endpoints } from '$lib/services/api-endpoints.js';
+import { replaceUrl } from '$lib/helpers/http';
+import axios from 'axios';
+
+export const llmRealtime = {
+    /** @type {RTCPeerConnection} */
+    pc: new RTCPeerConnection(),
+    /** @param {string} agentId */
+    async start(agentId) {
+        const session = await initRealtimeSession(agentId);
+        const EPHEMERAL_KEY = session.client_secret.value;
+
+        this.pc = new RTCPeerConnection();
+
+        // Set up to play remote audio from the model
+        const audioEl = document.createElement("audio");
+        audioEl.autoplay = true;
+        this.pc.ontrack = e => audioEl.srcObject = e.streams[0];
+
+        // Add local audio track for microphone input in the browser
+        const ms = await navigator.mediaDevices.getUserMedia({
+            audio: true
+        });
+        this.pc.addTrack(ms.getTracks()[0]);
+
+        // Set up data channel for sending and receiving events
+        const dc = this.pc.createDataChannel("oai-events");
+        dc.addEventListener("message", (e) => {
+            // Realtime server events appear here!
+            var data = JSON.parse(e.data);
+            console.log(data);
+        });
+
+        // Start the session using the Session Description Protocol (SDP)
+        const offer = await this.pc.createOffer();
+        await this.pc.setLocalDescription(offer);
+
+        const baseUrl = "https://api.openai.com/v1/realtime";
+        const sdpResponse = await fetch(`${baseUrl}?model=${session.model}`, {
+          method: "POST",
+          body: offer.sdp,
+          headers: {
+            Authorization: `Bearer ${EPHEMERAL_KEY}`,
+            "Content-Type": "application/sdp"
+          },
+        });
+        
+        /** @type {RTCSessionDescriptionInit} */
+        const answer = {
+          type: "answer",
+          sdp: await sdpResponse.text(),
+        };
+        await this.pc.setRemoteDescription(answer);
+    },
+
+    stop() {
+        // Stop the peer connection
+        this.pc.close();
+    }
+}
+
+/**
+ * Execute agent realtime interaction
+ * @param {string} agentId
+ * @returns {Promise<import('$realtimeTypes').RealtimeSession>}
+ */
+export async function initRealtimeSession(agentId) {
+    let url = replaceUrl(endpoints.agentInitRealtimeSessionUrl, {agentId: agentId});
+    var response = await axios.get(url);
+    return response.data;
+}

src/routes/chat/[agentId]/[conversationId]/chat-box.svelte

@@ -46,7 +46,7 @@
 	} from '$env/static/public';
 	import { BOT_SENDERS, LEARNER_ID, TRAINING_MODE, USER_SENDERS, ADMIN_ROLES } from '$lib/helpers/constants';
 	import { signalr } from '$lib/services/signalr-service.js';
-	import { webSpeech } from '$lib/services/web-speech.js';
+	import { llmRealtime } from '$lib/services/llm-realtime-service.js';
 	import { newConversation } from '$lib/services/conversation-service';
 	import DialogModal from '$lib/common/DialogModal.svelte';
 	import HeadTitle from '$lib/common/HeadTitle.svelte';
@@ -181,6 +181,7 @@
 	let isOpenTagModal = false;
 	let isSendingMsg = false;
 	let isThinking = false;
+	let isListening = false;
 	let isLite = false;
 	let isFrame = false;
 	let loadEditor = false;
@@ -470,7 +471,6 @@
 
     /** @param {import('$conversationTypes').ChatResponseModel} message */
     function onMessageReceivedFromAssistant(message) {
-		// webSpeech.utter(message.text);
 		dialogs.push({
 			...message,
 			is_chat_message: true
@@ -658,17 +658,15 @@
     async function startListen() {
 		if (disableSpeech) return;
 
-		microphoneIcon = "microphone";
-		webSpeech.onSpeechToTextDetected = (transcript) => {
-			if (!!!_.trim(transcript) || isSendingMsg) return;
-
-			sendChatMessage(transcript).then(() => {
-				microphoneIcon = "microphone-off";
-			}).catch(() => {
-				microphoneIcon = "microphone-off";
-			});
+		if (!isListening) {
+			llmRealtime.start(params.agentId);
+			isListening = true;
+			microphoneIcon = "microphone";
+		} else {
+			llmRealtime.stop();
+			isListening = false;
+			microphoneIcon = "microphone-off";
 		}
-		webSpeech.start();
 	}
 
 	/** @param {any} e */

svelte.config.js

@@ -16,6 +16,7 @@ const config = {
 			$userTypes: './src/lib/helpers/types/userTypes.js',
 			$roleTypes: './src/lib/helpers/types/roleTypes.js',
 			$pluginTypes: './src/lib/helpers/types/pluginTypes.js',
+			$realtimeTypes: './src/lib/helpers/types/realtimeTypes.js',
 		},
 
 		// for static deployment