[TTS Avatar][JS, Python] Trigger reconnection for websockets disconnection (#2782)

* [TalkingAvatar] Add sample code for TTS talking avatar real-time API

* sample codes for batch avatar synthesis

* Address repository check failure

* update

* [Avatar] Update real time avatar sample code to support multi-lingual

* [avatar] update real time avatar chat sample to receive GPT response streamingly

* [Live Avatar] update chat sample to make some refinements

* [TTS Avatar] Update real-time sample to support 1. non-continuous recognition mode 2. a button to stop speaking 3. user can type query without speech

* [TTS Avatar] Update real time avatar sample to support auto-reconnect

* Don't reset message history when re-connecting

* [talking avatar] update real time sample to support using cached local video for idle status, to help save customer cost

* Update chat.html and README.md

* Update batch avatar sample to use mp4 as default format, to avoid defaultly showing slow speed with vp9

* A minor refinement

* Some refinement

* Some bug fixing

* Refine the reponse receiving logic for AOAI streaming mode, to make it more robust

* [Talking Avatar] update real-time sample code to log result id (turn id) for ease of debugging

* [Talking Avatar] Update avatar live chat sample, to upgrade AOAI API version from 2023-03-15-preview to 2023-12-01-preview

* [Talking Avatar][Live Chat] Update AOAI API to be long term support version 2023-06-01-preview

* [Talking Avatar] Add real time avatar sample code for server/client hybrid web app, with server code written in python

* Some refinements

* Add README.md

* Fix repo check failure: files that are neither marked as binary nor text, please extend .gitattributes

* [Python][TTS Avatar] Add chat sample

* [Python][TTS Avatar] Add chat sample - continue

* Support multiple clients management

* Update README.md

* [Python][TTS Avatar] Support customized ICE server

* [Talking Avatar][Python] Support stop speaking

* Tolerat speech sdk to unsupport sending message with connection

* [Python][TTS Avatar] Send local SDP as post body instead of header, to avoid header size over limit

* [python][avatar] update requirements.txt to add the missing dependencies

* [python][avatar] update real-time sample to make auto-connection more smoothy

* [Python][Avatar] Fix some small bugs

* [python][avatar] Support AAD authorization on private endpoint

* [Java][Android][Avatar] Add Android sample code for real time avatar

* Code refinement

* More refinement

* More refinement

* Update README.md

* [Java][Android][Avatar] Remove AddStream method, which is not available with Unified Plan SDP semantics, and use AddTrack per suggestion

* [Python][Avatar][Live] Get speaking status from WebRTC event, and remove the checkSpeakingStatus API from backend code

* [Java][Android][Live Avatar] Update the sample to demonstrate switching audio output device to loud speaker

* [Python][Avatar][Live] Switch from REST API to SDK for calling AOAI

* [Python][Avatar][Live] Trigger barging at first recognizing event which is earlier

* [Python][Avatar][Live] Enable continuous conversation by default

* [Python][Avatar][Live] Disable multi-lingual by default for better latency

* [Python][Avatar][Live] Configure shorter segmentation silence timeout for quicker SR

* [Live Avatar][Python, CSharp] Add logging for latency

* [TTS Avatar][Live][Python, CSharp, JS] Fix a bug to correctly clean up audio player

* [TTS Avatar][Live][JavaScript] Output display text with a slower rate, to follow the avatar speaking progress

* Make the display text / speech alignment able for on/off

* [TTS Avatar][Live][CSharp] Output display text with a slower rate, to follow the avatar speaking progress

* Create an auto-deploy file

* Unlink the containerApp yinhew-avatar-app from this repo

* Delete unnecessary file

* [talking avatar][python] Update real time sample to add option to connect with server through WebSocket, and do STT on server side

* [TTS Avatar][Live][js] update sample code for support of setting of background image and remote TURN server URL

* [talking avatar][live][python] make sure host can still start up without AOAI resource

* [Talking Avatar][Live] update sample code to close WS connection in-time, when user closes/refreshes web page, or auto-reconnection is applied

* Some refinement to connection object

* Update csharp sample as well

* [Talking Avatar][Live][Python] Check ICE token fetching success

* [Talking Avatar][Live][Python] Add VAD for interruption with lower delay

* [TTS Avatar][JS] Continue speaking unfinished sentences after reconnection

* [TTS Avatar][Python] Continue speaking unfinished sentences after reconnection

* [TTS Avatar][Python] Trigger reconnection for websockets disconnection

* [TTS Avatar][JS] Trigger reconnection for websockets disconnection

---------

Co-authored-by: Yulin Li <yulili@microsoft.com>

Author: yinhew

samples/js/browser/avatar/js/chat.js

@@ -14,10 +14,12 @@ var enableQuickReply = false
 var quickReplies = [ 'Let me take a look.', 'Let me check.', 'One moment, please.' ]
 var byodDocRegex = new RegExp(/\[doc(\d+)\]/g)
 var isSpeaking = false
+var isReconnecting = false
 var speakingText = ""
 var spokenTextQueue = []
 var repeatSpeakingSentenceAfterReconnection = true
 var sessionActive = false
+var userClosedSession = false
 var lastSpeakTime
 var imgUrl = ""
 
@@ -207,6 +209,7 @@ function setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential) {
                     }
                 }
 
+                isReconnecting = false
                 setTimeout(() => { sessionActive = true }, 5000) // Set session active after 5 seconds
             }
         }
@@ -223,7 +226,23 @@ function setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential) {
                 subtitles.innerHTML = speakingText
             } else if (webRTCEvent.event.eventType === 'EVENT_TYPE_SESSION_END' || webRTCEvent.event.eventType === 'EVENT_TYPE_SWITCH_TO_IDLE') {
                 subtitles.hidden = true
+                if (webRTCEvent.event.eventType === 'EVENT_TYPE_SESSION_END') {
+                    if (document.getElementById('autoReconnectAvatar').checked && !userClosedSession && !isReconnecting) {
+                        // Session disconnected unexpectedly, need reconnect
+                        console.log(`[${(new Date()).toISOString()}] The WebSockets got disconnected, need reconnect.`)
+                        isReconnecting = true
+
+                        // Release the existing avatar connection
+                        if (avatarSynthesizer !== undefined) {
+                            avatarSynthesizer.close()
+                        }
+
+                        // Setup a new avatar connection
+                        connectAvatar()
+                    }
+                }
             }
+
             console.log("[" + (new Date()).toISOString() + "] WebRTC event received: " + e.data)
         }
     })
@@ -620,6 +639,7 @@ function checkHung() {
                     sessionActive = false
                     if (document.getElementById('autoReconnectAvatar').checked) {
                         console.log(`[${(new Date()).toISOString()}] The video stream got disconnected, need reconnect.`)
+                        isReconnecting = true
                         // Release the existing avatar connection
                         if (avatarSynthesizer !== undefined) {
                             avatarSynthesizer.close()
@@ -670,6 +690,7 @@ window.startSession = () => {
         return
     }
 
+    userClosedSession = false
     connectAvatar()
 }
 
@@ -687,6 +708,7 @@ window.stopSession = () => {
         document.getElementById('localVideo').hidden = true
     }
 
+    userClosedSession = true
     disconnectAvatar()
 }
 

samples/python/web/avatar/static/js/chat.js

@@ -12,6 +12,7 @@ var peerConnection
 var isSpeaking = false
 var isReconnecting = false
 var sessionActive = false
+var userClosedSession = false
 var recognitionStartedTime
 var chatRequestSentTime
 var chatResponseReceivedTime
@@ -249,6 +250,14 @@ function setupWebRTC(iceServerUrl, iceServerUsername, iceServerCredential) {
                 isSpeaking = false
                 lastSpeakTime = new Date()
                 document.getElementById('stopSpeaking').disabled = true
+            } else if (e.data.includes("EVENT_TYPE_SESSION_END")) {
+                if (document.getElementById('autoReconnectAvatar').checked && !userClosedSession && !isReconnecting) {
+                    // Session disconnected unexpectedly, need reconnect
+                    console.log(`[${(new Date()).toISOString()}] The WebSockets got disconnected, need reconnect.`)
+                    isReconnecting = true
+                    connectAvatar()
+                    createSpeechRecognizer()
+                }
             }
         }
     })
@@ -477,6 +486,8 @@ window.startSession = () => {
         setupWebSocket()
     }
 
+    userClosedSession = false
+
     createSpeechRecognizer()
     if (document.getElementById('useLocalVideoForIdle').checked) {
         document.getElementById('startSession').disabled = true
@@ -532,6 +543,7 @@ window.stopSession = () => {
         document.getElementById('localVideo').hidden = true
     }
 
+    userClosedSession = true // Indicating the session was closed by user on purpose, not due to network issue
     disconnectAvatar(true)
 }