Skip to content

Commit c3d0fff

Browse files
fquirinfquirin
committed
improved audio-event tracking and audio resume;
1 parent bebcf66 commit c3d0fff

File tree

5 files changed

+120
-49
lines changed

5 files changed

+120
-49
lines changed

‎www/scripts/sepiaFW.audio.js‎

Lines changed: 79 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,18 @@ function sepiaFW_build_audio(){
33
var AudioPlayer = {};
44
var TTS = {}; //TTS parameters for SepiaFW external TTS like Acapela. I've tried to seperate TTS and AudioPlayer as good as possible, but there might be some bugs using both
55

6-
//Parameters and states
7-
TTS.isSpeaking = false; //state: is assistant speaking?
8-
AudioPlayer.isPlaying = false;
9-
AudioPlayer.isLoading = false;
6+
//Parameters and states:
107

118
var player; //AudioPlayer for music and stuff
129
var player2; //AudioPlayer for sound effects
1310
var speaker; //Player for TTS
1411
var doInitAudio = true; //workaround to activate scripted audio on touch devices
1512
var audioOnEndFired = false; //state: prevent doublefireing of audio onend onpause
13+
14+
TTS.isSpeaking = false; //state: is assistant speaking?
15+
AudioPlayer.isPlaying = false; //NOTE: this applys only to player(1) aka the stream player
16+
AudioPlayer.isLoading = false; //" "
17+
1618
AudioPlayer.getMusicPlayer = function(){
1719
return player;
1820
}
@@ -46,8 +48,8 @@ function sepiaFW_build_audio(){
4648
&& ((new Date().getTime() - SepiaFW.android.lastReceivedMediaAppTS) < (1000*60*15)); //This is pure guessing ...
4749
return isInternalPlayerStreaming || isYouTubePlayerStreaming || isAndroidPlayerStreaming;
4850
}
49-
AudioPlayer.broadcastAudioEvent = function(source, action, playerObject){
50-
console.error("audio event: " + source + " - " + action);
51+
AudioPlayer.getLastActiveAudioStreamPlayer = function(){
52+
return lastAudioPlayerEventSource;
5153
}
5254

5355
//AudioContext stuff:
@@ -69,13 +71,31 @@ function sepiaFW_build_audio(){
6971
var audioVolD;
7072
var lastAudioStream = 'sounds/empty.mp3';
7173
var beforeLastAudioStream = 'sounds/empty.mp3';
74+
var lastAudioPlayerEventSource = ''; //Note: this does not include TTS and effects player
7275
var mainAudioIsOnHold = false;
7376
var mainAudioStopRequested = false;
7477
var orgVolume = 1.0;
7578
var orgGain = 1.0;
7679
var FADE_OUT_VOL = 0.05; //note: on some devices audio is actually stopped so this value does not apply
7780

7881
//---- broadcasting -----
82+
83+
AudioPlayer.broadcastAudioEvent = function(source, action, playerObject){
84+
//stream, effects, tts-player, unknown - start, stop, error, fadeOut, fadeIn
85+
//android-intent - stop, start
86+
//youtube-embedded - start, resume, pause, hold
87+
source = source.toLowerCase();
88+
action = action.toLowerCase();
89+
if (source == "stream" || source.indexOf("youtube") >= 0 || source.indexOf("android") >= 0){
90+
lastAudioPlayerEventSource = source;
91+
}
92+
var event = new CustomEvent('sepia_audio_player_event', { detail: {
93+
source: source,
94+
action: action
95+
}});
96+
document.dispatchEvent(event);
97+
//console.error("audio event: " + source + " - " + action);
98+
}
7999

80100
function broadcastAudioRequested(){
81101
//EXAMPLE:
@@ -257,11 +277,15 @@ function sepiaFW_build_audio(){
257277
AudioPlayer.isPlaying = false;
258278
}
259279
broadcastAudioFinished();
260-
if (!audioPlayer.dataset.tts){
280+
if (audioPlayer == player){
261281
mainAudioIsOnHold = false;
262282
mainAudioStopRequested = true; //We try to prevent the race-condition with that (1)
263283
if (gotPlayerAudioContext) playerGainNode.gain.value = orgGain;
264284
else audioPlayer.volume = orgVolume;
285+
}else if (audioPlayer == player2){
286+
//TODO: ?
287+
}else if (audioPlayer == speaker){
288+
//TODO: ?
265289
}
266290
//SEE AudioPlayer stop button for more, e.g. Android stop
267291
}
@@ -291,15 +315,15 @@ function sepiaFW_build_audio(){
291315
if (!mainAudioStopRequested){ //(if forced ..) We try to prevent the race-condition with that (2)
292316
mainAudioIsOnHold = true;
293317
}
294-
SepiaFW.audio.broadcastAudioEvent("stream", "fadeOut", player);
318+
AudioPlayer.broadcastAudioEvent("stream", "fadeOut", player);
295319
}
296320
}
297321
AudioPlayer.fadeInMainIfOnHold = function(){
298322
if (mainAudioIsOnHold){
299323
//fade to original volume
300324
AudioPlayer.playerFadeToOriginalVolume();
301325
mainAudioIsOnHold = false;
302-
SepiaFW.audio.broadcastAudioEvent("stream", "fadeIn", player);
326+
AudioPlayer.broadcastAudioEvent("stream", "fadeIn", player);
303327
}/*else{
304328
//just restore volume
305329
if (!gotPlayerAudioContext){
@@ -578,20 +602,35 @@ function sepiaFW_build_audio(){
578602

579603
//play audio by url
580604
AudioPlayer.playURL = function(audioURL, audioPlayer, onStartCallback, onEndCallback, onErrorCallback){
581-
if (!audioPlayer || audioPlayer === '1' || audioPlayer == 1){
605+
if (!audioPlayer || audioPlayer === '1' || audioPlayer == 1 || audioPlayer == 'stream'){
582606
audioPlayer = player;
607+
}else if (audioPlayer === '2' || audioPlayer == 2 || audioPlayer == 'effects'){
608+
audioPlayer = player2;
609+
}else if (audioPlayer === 'tts'){
610+
audioPlayer = speaker;
583611
}
584-
else if (audioPlayer === '2' || audioPlayer == 2 || audioPlayer === 'tts') audioPlayer = speaker;
585612

586-
if (audioURL && !audioPlayer.dataset.tts){
613+
if (audioURL && audioPlayer == player){
587614
beforeLastAudioStream = lastAudioStream;
588615
lastAudioStream = audioURL;
589616
}
590617
if (!audioURL) audioURL = lastAudioStream;
591618

592619
audioOnEndFired = false;
593-
if (!audioPlayer.dataset.tts){
620+
if (audioPlayer == player){
594621
broadcastAudioRequested();
622+
623+
//stop all other audio sources
624+
if (SepiaFW.client.controls){
625+
SepiaFW.client.controls.media({
626+
action: "stop",
627+
skipFollowUp: true
628+
});
629+
}
630+
}else if (audioPlayer == player2){
631+
//TODO: ?
632+
}else if (audioPlayer == speaker){
633+
//TODO: ?
595634
}
596635

597636
audioPlayer.preload = 'auto';
@@ -601,51 +640,55 @@ function sepiaFW_build_audio(){
601640
audioPlayer.src = audioURL;
602641
audioPlayer.oncanplay = function() {
603642
SepiaFW.debug.info("AUDIO: can be played now (oncanplay event)"); //debug
604-
if (!audioPlayer.dataset.tts){
643+
if (audioPlayer == player){
605644
AudioPlayer.isPlaying = true;
606645
broadcastAudioStarted();
607646
AudioPlayer.fadeInMainIfOnHold();
608647
mainAudioIsOnHold = false;
609648
mainAudioStopRequested = false;
610-
}else{
649+
}else if (audioPlayer == player2){
650+
//TODO: ?
651+
}else if (audioPlayer == speaker){
611652
TTS.isSpeaking = true;
612653
}
613654
AudioPlayer.isLoading = false;
614655
//callback
615656
if (onStartCallback) onStartCallback();
616657
if (audioPlayer == player){
617-
SepiaFW.audio.broadcastAudioEvent("stream", "start", audioPlayer);
658+
AudioPlayer.broadcastAudioEvent("stream", "start", audioPlayer);
618659
}else if (audioPlayer == player2){
619-
SepiaFW.audio.broadcastAudioEvent("effects", "start", audioPlayer);
660+
AudioPlayer.broadcastAudioEvent("effects", "start", audioPlayer);
620661
}else if (audioPlayer == speaker){
621-
SepiaFW.audio.broadcastAudioEvent("tts-player", "start", audioPlayer);
662+
AudioPlayer.broadcastAudioEvent("tts-player", "start", audioPlayer);
622663
}else{
623-
SepiaFW.audio.broadcastAudioEvent("unknown", "start", audioPlayer);
664+
AudioPlayer.broadcastAudioEvent("unknown", "start", audioPlayer);
624665
}
625666
};
626667
audioPlayer.onpause = function() {
627668
if (!audioOnEndFired){
628669
SepiaFW.debug.info("AUDIO: ended (onpause event)"); //debug
629670
audioOnEndFired = true;
630-
if (!audioPlayer.dataset.tts){
671+
if (audioPlayer == player){
631672
AudioPlayer.isPlaying = false;
632673
mainAudioStopRequested = false; //from here on we rely on AudioPlayer.isPlaying
633674
broadcastAudioFinished();
634675
//mainAudioIsOnHold = false; //<- set in stop method, here we might actually really want to stop-for-hold
635-
}else{
676+
}else if (audioPlayer == player2){
677+
//TODO: ?
678+
}else if (audioPlayer == speaker){
636679
TTS.isSpeaking = false;
637680
}
638681
AudioPlayer.isLoading = false;
639682
//callback
640683
if (onEndCallback) onEndCallback();
641684
if (audioPlayer == player){
642-
SepiaFW.audio.broadcastAudioEvent("stream", "stop", audioPlayer);
685+
AudioPlayer.broadcastAudioEvent("stream", "stop", audioPlayer);
643686
}else if (audioPlayer == player2){
644-
SepiaFW.audio.broadcastAudioEvent("effects", "stop", audioPlayer);
687+
AudioPlayer.broadcastAudioEvent("effects", "stop", audioPlayer);
645688
}else if (audioPlayer == speaker){
646-
SepiaFW.audio.broadcastAudioEvent("tts-player", "stop", audioPlayer);
689+
AudioPlayer.broadcastAudioEvent("tts-player", "stop", audioPlayer);
647690
}else{
648-
SepiaFW.audio.broadcastAudioEvent("unknown", "stop", audioPlayer);
691+
AudioPlayer.broadcastAudioEvent("unknown", "stop", audioPlayer);
649692
}
650693
}
651694
};
@@ -660,25 +703,27 @@ function sepiaFW_build_audio(){
660703
if (audioPlayer.error.code === 4){
661704
SepiaFW.ui.showInfo('Cannot play the selected audio stream. Sorry!'); //TODO: localize
662705
}
663-
if (!audioPlayer.dataset.tts){
706+
if (audioPlayer == player){
664707
broadcastAudioError();
665708
mainAudioIsOnHold = false;
666709
mainAudioStopRequested = false;
667710
AudioPlayer.isPlaying = false;
668-
}else{
711+
}else if (audioPlayer == player2){
712+
//TODO: ?
713+
}else if (audioPlayer == speaker){
669714
TTS.isSpeaking = false;
670715
}
671716
AudioPlayer.isLoading = false;
672717
//callback
673718
if (onErrorCallback) onErrorCallback();
674719
if (audioPlayer == player){
675-
SepiaFW.audio.broadcastAudioEvent("stream", "error", audioPlayer);
720+
AudioPlayer.broadcastAudioEvent("stream", "error", audioPlayer);
676721
}else if (audioPlayer == player2){
677-
SepiaFW.audio.broadcastAudioEvent("effects", "error", audioPlayer);
722+
AudioPlayer.broadcastAudioEvent("effects", "error", audioPlayer);
678723
}else if (audioPlayer == speaker){
679-
SepiaFW.audio.broadcastAudioEvent("tts-player", "error", audioPlayer);
724+
AudioPlayer.broadcastAudioEvent("tts-player", "error", audioPlayer);
680725
}else{
681-
SepiaFW.audio.broadcastAudioEvent("unknown", "error", audioPlayer);
726+
AudioPlayer.broadcastAudioEvent("unknown", "error", audioPlayer);
682727
}
683728
};
684729
audioPlayer.play();
@@ -716,7 +761,7 @@ function sepiaFW_build_audio(){
716761
broadcastAudioStarted();
717762
//callback
718763
if (onStartCallback) onStartCallback;
719-
SepiaFW.audio.broadcastAudioEvent("effects", "start", audioPlayer);
764+
AudioPlayer.broadcastAudioEvent("effects", "start", audioPlayer);
720765
};
721766
audioPlayer.onpause = function() {
722767
if (!audioOnEndFired){
@@ -732,7 +777,7 @@ function sepiaFW_build_audio(){
732777
*/
733778
//callback
734779
if (onEndCallback) onEndCallback();
735-
SepiaFW.audio.broadcastAudioEvent("effects", "stop", audioPlayer);
780+
AudioPlayer.broadcastAudioEvent("effects", "stop", audioPlayer);
736781
}
737782
};
738783
audioPlayer.onended = function() {
@@ -753,7 +798,7 @@ function sepiaFW_build_audio(){
753798
*/
754799
//callback
755800
if (onErrorCallback) onErrorCallback();
756-
SepiaFW.audio.broadcastAudioEvent("effects", "error", audioPlayer);
801+
AudioPlayer.broadcastAudioEvent("effects", "error", audioPlayer);
757802
};
758803
audioPlayer.play();
759804
}

‎www/scripts/sepiaFW.client.controls.js‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -181,25 +181,29 @@ function sepiaFW_build_client_controls(){
181181

182182
//RESUME
183183
}else if (controlData.action == "resume"){
184-
//TODO: only working for YouTube and Android(?) right now
185-
var isInternalPlayerStreaming = SepiaFW.audio.isMusicPlayerStreaming() || SepiaFW.audio.isMainOnHold();
186-
if (!isInternalPlayerStreaming){
187-
//Player and platform specific additional RESUME methods
184+
//try to find last active player
185+
var lastActivePlayer = SepiaFW.audio.getLastActiveAudioStreamPlayer();
186+
var isAnyPlayerStreaming = SepiaFW.audio.isAnyAudioSourceActive();
187+
if (!isAnyPlayerStreaming){
188188
var sentAdditionalEvent = false;
189-
190-
if (SepiaFW.ui.cards.youTubePlayerGetState() == 2){ //2: paused
189+
190+
//try right order first
191+
if (!lastActivePlayer){
192+
//TODO: ?
193+
194+
}else if (lastActivePlayer == "stream"){
195+
//Internal stream
196+
sentAdditionalEvent = SepiaFW.audio.resumeLastAudioStream();
197+
198+
}else if (lastActivePlayer == "youtube-embedded" && SepiaFW.ui.cards.youTubePlayerGetState() == 2){ //2: paused
191199
//YouTube embedded player
192200
sentAdditionalEvent = (SepiaFW.ui.cards.youTubePlayerControls("resume") > 0);
193201

194-
}else if (SepiaFW.ui.isAndroid){
202+
}else if (lastActivePlayer == "android-intent" && SepiaFW.ui.isAndroid){
195203
//we do this only if we have a recent Android media event - otherwhise it will activate all music apps
196204
var requireMediaAppPackage = true;
197205
sentAdditionalEvent = SepiaFW.android.broadcastMediaButtonDownUpIntent(126, requireMediaAppPackage);
198206
//126: KEYCODE_MEDIA_PLAY
199-
200-
//Last try is internal player
201-
}else{
202-
sentAdditionalEvent = SepiaFW.audio.resumeLastAudioStream();
203207
}
204208
}
205209
//TODO: add iOS and Windows?

‎www/scripts/sepiaFW.ui.cards.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1105,7 +1105,7 @@ function sepiaFW_build_ui_cards(){
11051105
var f = document.createElement('iframe');
11061106
f.id = 'youTubeWebPlayer-' + playerId;
11071107
f.allow = allowIframe;
1108-
f.sandbox = "allow-same-origin allow-scripts allow-presentation";
1108+
f.sandbox = "allow-same-origin allow-scripts allow-presentation allow-popups";
11091109
f.frameBorder = 0;
11101110
f.style.width = "100%"; f.style.height = "280px"; f.style.overflow = "hidden";
11111111
f.style.border = "4px solid"; f.style.borderColor = "#212121";

‎www/scripts/sepiaFW.ui.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ function sepiaFW_build_ui(){
9292
tagName: document.activeElement.tagName
9393
}});
9494
document.dispatchEvent(event);
95-
console.error("new active ele.: " + (document.activeElement.id || document.activeElement.className || document.activeElement.tagName));
95+
//console.error("new active ele.: " + (document.activeElement.id || document.activeElement.className || document.activeElement.tagName));
9696
}, 100);
9797
}
9898
var activeElementChangeBuffer = undefined;

‎www/scripts/sepiaFW.wakeTriggers.js‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,29 @@ function sepiaFW_build_wake_triggers() {
5656
SepiaFW.debug.info("Wake-word during audio streaming is " + ((WakeTriggers.allowWakeWordDuringStream)? "ALLOWED" : "NOT ALLOWED"));
5757

5858
//Add onActive action:
59-
SepiaFW.client.addOnActiveOneTimeAction(wakeTriggersOnActiveAction);
59+
SepiaFW.client.addOnActiveOneTimeAction(wakeTriggersOnActiveAction);
60+
61+
//Event listeners
62+
document.addEventListener('sepia_audio_player_event', function(e){
63+
if (e.detail){
64+
//console.error(e.detail);
65+
if (WakeTriggers.useWakeWord && WakeTriggers.engineLoaded){
66+
//Audio player start
67+
if (e.detail.action == "start" || e.detail.action == "resume"){
68+
if (WakeTriggers.isListening() && !WakeTriggers.allowWakeWordDuringStream){
69+
SepiaFW.debug.info("WakeTriggers - Stopping wake-word listener due to audio player '" + e.detail.action + "' event");
70+
WakeTriggers.stopListeningToWakeWords();
71+
}
72+
//Audio player stop
73+
}else if (e.detail.action == "stop" || e.detail.action == "pause"){
74+
if ((SepiaFW.animate.assistant.getState() == "idle") && !WakeTriggers.isListening()){
75+
SepiaFW.debug.info("WakeTriggers - Scheduling wake-word listener restart due to audio player '" + e.detail.action + "' event");
76+
WakeTriggers.listenToWakeWords(undefined, undefined, true);
77+
}
78+
}
79+
}
80+
}
81+
}, true);
6082
}
6183
function wakeTriggersOnActiveAction(){
6284
//start setup?
@@ -101,7 +123,7 @@ function sepiaFW_build_wake_triggers() {
101123
//console.error("Audio playing? " + isAudioStreamActive); //DEBUG
102124
if (isAudioStreamActive && !WakeTriggers.allowWakeWordDuringStream){
103125
return;
104-
}else if (WakeTriggers.useWakeWord && WakeTriggers.engineLoaded && !WakeTriggers.isListening()){
126+
}else if (WakeTriggers.useWakeWord && WakeTriggers.engineLoaded && !WakeTriggers.isListening() && (SepiaFW.animate.assistant.getState() == "idle")){
105127
//call without delay NOW
106128
WakeTriggers.listenToWakeWords(onSuccessCallback, onErrorCallback, false);
107129
}

0 commit comments

Comments
 (0)