improved handling of audio play + mic release events

Author: fquirin

www/scripts/sepiaFW.audioRecorder.js

@@ -4,9 +4,6 @@ function sepiaFW_build_audio_recorder(){
 
 	//Debug modules and interfaces?
 	AudioRecorder.debugInterfaces = false;
-	
-	//Parameters and states
-	AudioRecorder.isRecording = false;
 
 	//set default parameters for audio recorder
 	AudioRecorder.setup = function(successCallback, errorCallback){
@@ -15,21 +12,12 @@ function sepiaFW_build_audio_recorder(){
 
 	//---- broadcasting -----
 
-	function broadcastRecorderRequested(){
-		//console.log('broadcastRecorderRequested');
-	}
-	function broadcastRecorderStopRequested(){
-		//console.log('broadcastRecorderStopRequested');
-	}
-	function broadcastRecorderClosed(){
-		//console.log('broadcastRecorderClosed');
-	}
-	function broadcastRecorderStarted(){
-		//console.log('broadcastRecorderStarted');
-	}
-	function broadcastRecorderStopped(){
-		//console.log('broadcastRecorderStopped');
-	}
+	//TODO: do we still need these events? (check below: 'dispatchRecorderEvent')
+	function broadcastRecorderRequested(){}
+	function broadcastRecorderStopRequested(){}
+	function broadcastRecorderClosed(){}
+	function broadcastRecorderStarted(){}
+	function broadcastRecorderStopped(){}
 	function broadcastRecorderError(e){
 		//console.log('broadcastRecorderError');
 	}
@@ -39,8 +27,22 @@ function sepiaFW_build_audio_recorder(){
 	if (SepiaFW.webAudio.defaultProcessorOptions){
 		SepiaFW.webAudio.defaultProcessorOptions.moduleFolder = "audio-modules";
 	}
+
+	//event dispatcher
+	function dispatchRecorderEvent(data){
+		//console.error("WebAudio recorder event", data);		//DEBUG
+		document.dispatchEvent(new CustomEvent('sepia_web_audio_recorder', { detail: data}));
+	}
+
+	//The one and ONLY recorder instance:
 	var sepiaWebAudioProcessor;
 
+	//should run parallel to AudioPlayer output?
+	AudioRecorder.mayMicRunParallelToAudioOut = function(){
+		return !SepiaFW.ui.isMobile;	//mobile is ATM usually not good with Input + Output at same time
+		//NOTE: for wake-word we check 'WakeTriggers.allowWakeWordDuringStream' and user can force this
+	}
+
 	var activeAudioModules = [];
 	var activeAudioModuleCapabilities = {};		//e.g.: resample, wakeWordDetection, vad, waveEncoding, volume, speechRecognition
 	AudioRecorder.webAudioHasCapability = function(testCap){
@@ -131,12 +133,6 @@ function sepiaFW_build_audio_recorder(){
 	if (useLegacyMicInterface) micAudioRecorderOptions.sourceType = "scriptProcessor";
 	//var legacyScriptProcessorBufferSize = 512;		//use 'micAudioRecorderOptions.processorBufferSize'
 
-	//event dispatcher
-	function dispatchRecorderEvent(data){
-		//console.error("WebAudio recorder event", data);		//DEBUG
-		document.dispatchEvent(new CustomEvent('sepia_web_audio_recorder', { detail: data}));
-	}
-
 	function defaultOnProcessorReady(sepiaWebAudioProcessor, msg){
 		SepiaFW.debug.info("AudioRecorder - onProcessorReady", sepiaWebAudioProcessor, msg);	//DEBUG
 	}
@@ -167,11 +163,6 @@ function sepiaFW_build_audio_recorder(){
 		return (!!sepiaWebAudioProcessor && sepiaWebAudioProcessor.isInitialized() && sepiaWebAudioProcessor.isProcessing());
 	}
 	AudioRecorder.startWebAudioRecorder = function(successCallback, noopCallback, errorCallback){
-		/*if (AudioRecorder.isRecording){
-			SepiaFW.debug.err("AudioRecorder error: Tried to capture audio but was already running!");
-			errorCallback({name: "AudioRecorder: not started!", message: "Audio capture was already running."});
-			return;
-		}*/
 		if (sepiaWebAudioProcessor){
 			sepiaWebAudioProcessor.start(successCallback, noopCallback, errorCallback);
 		}else{

www/scripts/sepiaFW.speechAudioProcessor.js

@@ -186,6 +186,7 @@ function sepiaFW_build_speech_audio_proc(){
 	var maxVadTime = 10000;
 
 	var isWaitingToRecord = false;		//use for..?
+	var isReleasing = false;
 	//var recognizerWaitingForResult = false;	//TODO: implement?
 
 	var asrModuleGateIsOpen = false;	//equivalent to: 'isRecording'
@@ -359,6 +360,7 @@ function sepiaFW_build_speech_audio_proc(){
 		_asrLogCallback('ASR ERROR - ABORT');
 		//reset states
 		isWaitingToRecord = false;
+		isReleasing = false;
 		asrModuleGateIsOpen = false;	//TODO: can we trust this?
 
 		if (Recognizer.onerror){
@@ -424,6 +426,7 @@ function sepiaFW_build_speech_audio_proc(){
 			SpeechRecognition.recognitionModule = buildWebSocketAsrModule();
 
 			//states set 2 - start
+			isReleasing = false;
 			abortRecognition = false;
 			asrModuleGateIsOpen = false;
 
@@ -481,6 +484,7 @@ function sepiaFW_build_speech_audio_proc(){
 				//STATE: audioend
 				onAudioEnd();	//trigger or not?
 				//TODO: just wait for result? release after certain time? what if errorCallback triggers?
+				//NOTE: releasing the recorder will kill all pending results (use idle event?)
 			});
 		}
 	}
@@ -492,17 +496,29 @@ function sepiaFW_build_speech_audio_proc(){
 		//NOTE: expected to send END event - should be triggered by stop
 	}
 
+	//a release function - currently just used to prevent multiple calls to release
+	function releaseThisRecorder(callback){
+		if (!isReleasing && SpeechRecognition.recognitionModule){
+			isReleasing = true;
+			SepiaFW.audioRecorder.stopAndReleaseIfActive(function(){
+				if (callback) callback();
+				//... the rest is below in event handler
+			});
+		}
+	}
+
 	//listen to global events to make sure state is updated correctly
 	document.addEventListener("sepia_web_audio_recorder", recoderEventListener);
 	function recoderEventListener(e){
 		var data = e.detail;
 		if (!data || !data.event) return;
 
 		//TODO: is this correct?
-		if (data.event == "release" && (asrModuleGateIsOpen || isWaitingToRecord)){
+		if (data.event == "release" && SpeechRecognition.recognitionModule){
 			//reset state
 			asrModuleGateIsOpen = false;
 			isWaitingToRecord = false;
+			isReleasing = false;
 			SpeechRecognition.recognitionModule = undefined;
 
 		}else if (data.event == "audioend" && (asrModuleGateIsOpen || isWaitingToRecord)){
@@ -512,6 +528,23 @@ function sepiaFW_build_speech_audio_proc(){
 			isWaitingToRecord = false;
 		}
 	}
+	document.addEventListener('sepia_audio_player_event', audioPlayerEventListener, true);
+	function audioPlayerEventListener(e){
+		var data = e.detail;
+		if (!data || !data.action) return;
+		//console.error("SpeechRecognition Processor - sepia_audio_player_event", data, "exists?", !!SpeechRecognition.recognitionModule);		//DEBUG
+		if (SpeechRecognition.recognitionModule){
+			//Audio player start
+			if (data.action == "prepare" || data.action == "start" || data.action == "resume"){
+				//make sure the recorder is released when audio out is running
+				if (!SepiaFW.audioRecorder.mayMicRunParallelToAudioOut()){
+					releaseThisRecorder(function(){
+						SepiaFW.debug.info("SpeechRecognition Processor - Released due to audio player '" + data.action + "' event");
+					});
+				}
+			}
+		}
+	}
 
 	return SpeechRecognition;
 }

www/scripts/sepiaFW.wakeTriggers.js

@@ -89,13 +89,13 @@ function sepiaFW_build_wake_triggers() {
 			hasAudioPlayerEventListeners = true;
 			document.addEventListener('sepia_audio_player_event', function(e){
 				if (e.detail){
-					//console.error(e.detail);
+					//console.error("WakeTriggers - sepia_audio_player_event", e.detail);		//DEBUG
 					if (WakeTriggers.useWakeWord && WakeTriggers.engineLoaded){
 						//Audio player start
-						if (e.detail.action == "start" || e.detail.action == "resume"){
-							if (WakeTriggers.isListening() && !WakeTriggers.allowWakeWordDuringStream){
+						if (e.detail.action == "prepare" || e.detail.action == "start" || e.detail.action == "resume"){
+							if (!isStopping && WakeTriggers.isListening() && !WakeTriggers.allowWakeWordDuringStream){
 								SepiaFW.debug.info("WakeTriggers - Stopping wake-word listener due to audio player '" + e.detail.action + "' event");
-								WakeTriggers.stopListeningToWakeWords();
+								WakeTriggers.stopListeningToWakeWords();	//note without 'keepalive' this is RELEASE
 							}
 						//Audio player stop
 						}else if (e.detail.action == "stop" || e.detail.action == "pause"){