1+ import { PUBLIC_SERVICE_URL } from "$env/static/public" ;
2+ import { AudioRecordingWorklet } from "$lib/helpers/realtime/pcmProcessor" ;
3+
4+ // @ts -ignore
5+ const AudioContext = window . AudioContext || window . webkitAudioContext ;
6+
7+ const sampleRate = 24000 ;
8+
9+ /** @type {AudioContext } */
10+ let audioCtx = new AudioContext ( ) ;
11+
12+ /** @type {any[] } */
13+ let audioQueue = [ ] ;
14+
15+ /** @type {boolean } */
16+ let isPlaying = false ;
17+
18+ /** @type {WebSocket | null } */
19+ let socket = null ;
20+
21+ /** @type {MediaStream | null } */
22+ let mediaStream = null ;
23+
24+ /** @type {AudioWorkletNode | null } */
25+ let workletNode = null ;
26+
27+ /** @type {MediaStreamAudioSourceNode | null } */
28+ let micSource = null ;
29+
30+ export const realtimeChat = {
31+
32+ /**
33+ * @param {string } agentId
34+ * @param {string } conversationId
35+ */
36+ start ( agentId , conversationId ) {
37+ reset ( ) ;
38+ const wsUrl = buildWebsocketUrl ( ) ;
39+ socket = new WebSocket ( `${ wsUrl } ${ agentId } ${ conversationId } ) ;
40+
41+ socket . onopen = async ( ) => {
42+ console . log ( "WebSocket connected" ) ;
43+
44+ socket ?. send ( JSON . stringify ( {
45+ event : "start"
46+ } ) ) ;
47+
48+ mediaStream = await navigator . mediaDevices . getUserMedia ( { audio : true } ) ;
49+ audioCtx = new AudioContext ( { sampleRate : sampleRate } ) ;
50+
51+ const workletName = "audio-recorder-worklet" ;
52+ const src = createWorkletFromSrc ( workletName , AudioRecordingWorklet ) ;
53+ await audioCtx . audioWorklet . addModule ( src ) ;
54+
55+ workletNode = new AudioWorkletNode ( audioCtx , workletName ) ;
56+ micSource = audioCtx . createMediaStreamSource ( mediaStream ) ;
57+ micSource . connect ( workletNode ) ;
58+
59+ workletNode . port . onmessage = event => {
60+ const arrayBuffer = event . data . data . int16arrayBuffer ;
61+ if ( arrayBuffer && socket ?. readyState === WebSocket . OPEN ) {
62+ if ( event . data . data . speaking ) {
63+ reset ( ) ;
64+ }
65+ const arrayBufferString = arrayBufferToBase64 ( arrayBuffer ) ;
66+ socket . send ( JSON . stringify ( {
67+ event : 'media' ,
68+ body : {
69+ payload : arrayBufferString
70+ }
71+ } ) ) ;
72+ }
73+ } ;
74+ } ;
75+
76+ socket . onmessage = ( /** @type {MessageEvent } */ e ) => {
77+ try {
78+ const json = JSON . parse ( e . data ) ;
79+ if ( json . event === 'media' && ! ! json . media . payload ) {
80+ const data = json . media . payload ;
81+ enqueueAudioChunk ( data ) ;
82+ }
83+ } catch {
84+ // console.error('Error when parsing message');
85+ }
86+ } ;
87+
88+ socket . onclose = ( ) => {
89+ console . log ( "Websocket closed" ) ;
90+ } ;
91+
92+ socket . onerror = ( /** @type {Event } */ e ) => {
93+ console . error ( 'WebSocket error' , e ) ;
94+ } ;
95+ } ,
96+
97+ stop ( ) {
98+ reset ( ) ;
99+
100+ if ( mediaStream ) {
101+ mediaStream . getTracks ( ) . forEach ( t => t . stop ( ) ) ;
102+ mediaStream = null ;
103+ }
104+
105+ if ( workletNode ) {
106+ micSource ?. disconnect ( workletNode ) ;
107+ workletNode . port . close ( ) ;
108+ workletNode . disconnect ( ) ;
109+ micSource = null ;
110+ workletNode = null ;
111+ }
112+
113+ if ( socket ?. readyState === WebSocket . OPEN ) {
114+ socket . send ( JSON . stringify ( {
115+ event : 'disconnect'
116+ } ) ) ;
117+ socket . close ( ) ;
118+ socket = null ;
119+ }
120+ }
121+ } ;
122+
123+
124+ function buildWebsocketUrl ( ) {
125+ let url = '' ;
126+ const host = PUBLIC_SERVICE_URL . split ( '://' ) ;
127+
128+ if ( PUBLIC_SERVICE_URL . startsWith ( 'https' ) ) {
129+ url = `wss:${ host [ 1 ] } ;
130+ } else if ( PUBLIC_SERVICE_URL . startsWith ( 'http' ) ) {
131+ url = `ws:${ host [ 1 ] } ;
132+ }
133+
134+ return url ;
135+ }
136+
137+ function reset ( ) {
138+ isPlaying = false ;
139+ audioQueue = [ ] ;
140+ }
141+
142+ /**
143+ * @param {string } base64Audio
144+ */
145+ function enqueueAudioChunk ( base64Audio ) {
146+ const arrayBuffer = base64ToArrayBuffer ( base64Audio ) ;
147+ const float32Data = convert16BitPCMToFloat32 ( arrayBuffer ) ;
148+
149+ const audioBuffer = audioCtx . createBuffer ( 1 , float32Data . length , sampleRate ) ;
150+ audioBuffer . getChannelData ( 0 ) . set ( float32Data ) ;
151+ audioQueue . push ( audioBuffer ) ;
152+
153+ if ( ! isPlaying ) {
154+ playNext ( ) ;
155+ }
156+ }
157+
158+ function playNext ( ) {
159+ if ( audioQueue . length === 0 ) {
160+ isPlaying = false ;
161+ return ;
162+ }
163+
164+ isPlaying = true ;
165+ const buffer = audioQueue . shift ( ) ;
166+
167+ const source = audioCtx . createBufferSource ( ) ;
168+ source . buffer = buffer ;
169+ source . connect ( audioCtx . destination ) ;
170+ source . onended = ( ) => {
171+ playNext ( ) ;
172+ } ;
173+ source . start ( ) ;
174+ }
175+
176+
177+ /**
178+ * @param {string } workletName
179+ * @param {string } workletSrc
180+ */
181+ function createWorkletFromSrc ( workletName , workletSrc ) {
182+ const script = new Blob (
183+ [ `registerProcessor("${ workletName } ${ workletSrc } ] ,
184+ {
185+ type : "application/javascript" ,
186+ } ,
187+ ) ;
188+
189+ return URL . createObjectURL ( script ) ;
190+ } ;
191+
192+
193+ /**
194+ * @param {ArrayBuffer } buffer
195+ */
196+ function arrayBufferToBase64 ( buffer ) {
197+ var binary = "" ;
198+ var bytes = new Uint8Array ( buffer ) ;
199+ var len = bytes . byteLength ;
200+ for ( var i = 0 ; i < len ; i ++ ) {
201+ binary += String . fromCharCode ( bytes [ i ] ) ;
202+ }
203+ return btoa ( binary ) ;
204+ } ;
205+
206+ /**
207+ * @param {string } base64
208+ */
209+ function base64ToArrayBuffer ( base64 ) {
210+ const binaryStr = atob ( base64 ) ;
211+ const len = binaryStr . length ;
212+ const bytes = new Uint8Array ( len ) ;
213+
214+ for ( let i = 0 ; i < len ; i ++ ) {
215+ bytes [ i ] = binaryStr . charCodeAt ( i ) ;
216+ }
217+ return bytes . buffer ;
218+ } ;
219+
220+ /**
221+ * @param {ArrayBuffer } buffer
222+ */
223+ function convert16BitPCMToFloat32 ( buffer ) {
224+ const chunk = new Uint8Array ( buffer ) ;
225+ const output = new Float32Array ( chunk . length / 2 ) ;
226+ const dataView = new DataView ( chunk . buffer ) ;
227+
228+ for ( let i = 0 ; i < chunk . length / 2 ; i ++ ) {
229+ try {
230+ const int16 = dataView . getInt16 ( i * 2 , true ) ;
231+ output [ i ] = int16 / 32768 ;
232+ } catch ( e ) {
233+ console . error ( e ) ;
234+ }
235+ }
236+ return output ;
237+ } ;
0 commit comments