Skip to content

Commit 3a50c00

Browse files
fquirinfquirin
committed
improved wake-word and audio-events handling; new ww option;
1 parent 39b4f39 commit 3a50c00

File tree

9 files changed

+175
-17
lines changed

9 files changed

+175
-17
lines changed

‎www/css/sepiaFW-style.css‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1317,6 +1317,11 @@ body {
13171317
#sepiaFW-nav-label-online-status.connecting {
13181318
opacity: 0.50;
13191319
}
1320+
#sepiaFW-nav-label-online-status.wake-word-active:after,
1321+
#sepiaFW-nav-label-online-status.wake-word-active:before {
1322+
content: ' • ';
1323+
color: #f00;
1324+
}
13201325
#sepiaFW-nav-label-note {
13211326
display: flex;
13221327
align-items: center;

‎www/scripts/sepiaFW.android.js‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,15 +70,24 @@ function sepiaFW_build_android(){
7070
if (Android.lastRequestMediaAppPackage){
7171
intent.package = Android.lastRequestMediaAppPackage;
7272
}
73+
var tried;
7374
if (requireMediaAppPackage && intent.package){
7475
Android.intentBroadcast(intent);
75-
return true; //sent
76+
tried = true; //sent
7677
}else if (!requireMediaAppPackage){
7778
Android.intentBroadcast(intent);
78-
return true; //sent
79+
tried = true; //sent
7980
}else{
80-
return false; //not sent
81+
tried = false; //not sent
8182
}
83+
if (tried){
84+
if (code == 127){
85+
SepiaFW.audio.broadcastAudioEvent("android-intent", "stop", undefined);
86+
}else if (code == 126 || code == 87 || code == 88){
87+
SepiaFW.audio.broadcastAudioEvent("android-intent", "start", undefined);
88+
}
89+
}
90+
return tried;
8291
}
8392
//Simulate donw-up key event (some apps won't accept only one event)
8493
Android.broadcastMediaButtonDownUpIntent = function(code, requireMediaAppPackage){

‎www/scripts/sepiaFW.audio.js‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ function sepiaFW_build_audio(){
66
//Parameters and states
77
TTS.isSpeaking = false; //state: is assistant speaking?
88
AudioPlayer.isPlaying = false;
9+
AudioPlayer.isLoading = false;
910

1011
var player; //AudioPlayer for music and stuff
1112
var player2; //AudioPlayer for sound effects
@@ -35,6 +36,19 @@ function sepiaFW_build_audio(){
3536
AudioPlayer.getTtsPlayer = function(){
3637
return speaker;
3738
}
39+
40+
//Try to find out if any music player is active (playing or onHold and waiting to resume)
41+
AudioPlayer.isAnyAudioSourceActive = function(){
42+
//Stop internal player
43+
var isInternalPlayerStreaming = AudioPlayer.isLoading || AudioPlayer.isMusicPlayerStreaming() || AudioPlayer.isMainOnHold() || TTS.isPlaying;
44+
var isYouTubePlayerStreaming = SepiaFW.ui.cards.youTubePlayerGetState() == 1 || SepiaFW.ui.cards.youTubePlayerIsOnHold();
45+
var isAndroidPlayerStreaming = SepiaFW.ui.isAndroid && SepiaFW.android.lastReceivedMediaData && SepiaFW.android.lastReceivedMediaData.playing
46+
&& ((new Date().getTime() - SepiaFW.android.lastReceivedMediaAppTS) < (1000*60*15)); //This is pure guessing ...
47+
return isInternalPlayerStreaming || isYouTubePlayerStreaming || isAndroidPlayerStreaming;
48+
}
49+
AudioPlayer.broadcastAudioEvent = function(source, action, playerObject){
50+
console.error("audio event: " + source + " - " + action);
51+
}
3852

3953
//AudioContext stuff:
4054
AudioPlayer.useAudioContext = false; //experimental feature for things like gain control and music visualization (unfortunately fails for quite a few radio streams)
@@ -238,6 +252,9 @@ function sepiaFW_build_audio(){
238252
if (!audioPlayer) audioPlayer = player;
239253
if (AudioPlayer.isPlaying){
240254
audioPlayer.pause(); //NOTE: possible race condition here if onPause callback triggers after fadeOutMain (then AudioPlayer.isPlaying will be true)
255+
}else{
256+
AudioPlayer.isLoading = false;
257+
AudioPlayer.isPlaying = false;
241258
}
242259
broadcastAudioFinished();
243260
if (!audioPlayer.dataset.tts){
@@ -274,13 +291,15 @@ function sepiaFW_build_audio(){
274291
if (!mainAudioStopRequested){ //(if forced ..) We try to prevent the race-condition with that (2)
275292
mainAudioIsOnHold = true;
276293
}
294+
SepiaFW.audio.broadcastAudioEvent("stream", "fadeOut", player);
277295
}
278296
}
279297
AudioPlayer.fadeInMainIfOnHold = function(){
280298
if (mainAudioIsOnHold){
281299
//fade to original volume
282300
AudioPlayer.playerFadeToOriginalVolume();
283301
mainAudioIsOnHold = false;
302+
SepiaFW.audio.broadcastAudioEvent("stream", "fadeIn", player);
284303
}/*else{
285304
//just restore volume
286305
if (!gotPlayerAudioContext){
@@ -576,6 +595,8 @@ function sepiaFW_build_audio(){
576595
}
577596

578597
audioPlayer.preload = 'auto';
598+
AudioPlayer.isLoading = true;
599+
579600
//console.log("Audio-URL: " + audioURL); //DEBUG
580601
audioPlayer.src = audioURL;
581602
audioPlayer.oncanplay = function() {
@@ -589,8 +610,18 @@ function sepiaFW_build_audio(){
589610
}else{
590611
TTS.isSpeaking = true;
591612
}
613+
AudioPlayer.isLoading = false;
592614
//callback
593615
if (onStartCallback) onStartCallback();
616+
if (audioPlayer == player){
617+
SepiaFW.audio.broadcastAudioEvent("stream", "start", audioPlayer);
618+
}else if (audioPlayer == player2){
619+
SepiaFW.audio.broadcastAudioEvent("effects", "start", audioPlayer);
620+
}else if (audioPlayer == speaker){
621+
SepiaFW.audio.broadcastAudioEvent("tts-player", "start", audioPlayer);
622+
}else{
623+
SepiaFW.audio.broadcastAudioEvent("unknown", "start", audioPlayer);
624+
}
594625
};
595626
audioPlayer.onpause = function() {
596627
if (!audioOnEndFired){
@@ -604,8 +635,18 @@ function sepiaFW_build_audio(){
604635
}else{
605636
TTS.isSpeaking = false;
606637
}
638+
AudioPlayer.isLoading = false;
607639
//callback
608640
if (onEndCallback) onEndCallback();
641+
if (audioPlayer == player){
642+
SepiaFW.audio.broadcastAudioEvent("stream", "stop", audioPlayer);
643+
}else if (audioPlayer == player2){
644+
SepiaFW.audio.broadcastAudioEvent("effects", "stop", audioPlayer);
645+
}else if (audioPlayer == speaker){
646+
SepiaFW.audio.broadcastAudioEvent("tts-player", "stop", audioPlayer);
647+
}else{
648+
SepiaFW.audio.broadcastAudioEvent("unknown", "stop", audioPlayer);
649+
}
609650
}
610651
};
611652
audioPlayer.onended = function() {
@@ -627,8 +668,18 @@ function sepiaFW_build_audio(){
627668
}else{
628669
TTS.isSpeaking = false;
629670
}
671+
AudioPlayer.isLoading = false;
630672
//callback
631673
if (onErrorCallback) onErrorCallback();
674+
if (audioPlayer == player){
675+
SepiaFW.audio.broadcastAudioEvent("stream", "error", audioPlayer);
676+
}else if (audioPlayer == player2){
677+
SepiaFW.audio.broadcastAudioEvent("effects", "error", audioPlayer);
678+
}else if (audioPlayer == speaker){
679+
SepiaFW.audio.broadcastAudioEvent("tts-player", "error", audioPlayer);
680+
}else{
681+
SepiaFW.audio.broadcastAudioEvent("unknown", "error", audioPlayer);
682+
}
632683
};
633684
audioPlayer.play();
634685
}
@@ -656,18 +707,23 @@ function sepiaFW_build_audio(){
656707

657708
audioPlayer.src = alarmSound;
658709
audioPlayer.preload = 'auto';
710+
AudioPlayer.isLoading = true;
711+
659712
audioPlayer.oncanplay = function() {
660713
SepiaFW.debug.info("AUDIO: can be played now (oncanplay event)"); //debug
661714
AudioPlayer.isPlaying = true;
715+
AudioPlayer.isLoading = false;
662716
broadcastAudioStarted();
663717
//callback
664718
if (onStartCallback) onStartCallback;
719+
SepiaFW.audio.broadcastAudioEvent("effects", "start", audioPlayer);
665720
};
666721
audioPlayer.onpause = function() {
667722
if (!audioOnEndFired){
668723
SepiaFW.debug.info("AUDIO: ended (onpause event)"); //debug
669724
audioOnEndFired = true;
670725
AudioPlayer.isPlaying = false;
726+
AudioPlayer.isLoading = false;
671727
broadcastAudioFinished();
672728
//reset audio URL
673729
/*
@@ -676,6 +732,7 @@ function sepiaFW_build_audio(){
676732
*/
677733
//callback
678734
if (onEndCallback) onEndCallback();
735+
SepiaFW.audio.broadcastAudioEvent("effects", "stop", audioPlayer);
679736
}
680737
};
681738
audioPlayer.onended = function() {
@@ -688,13 +745,15 @@ function sepiaFW_build_audio(){
688745
SepiaFW.debug.info("AUDIO: error occured! - code: " + audioPlayer.error.code); //debug
689746
broadcastAudioError();
690747
AudioPlayer.isPlaying = false;
748+
AudioPlayer.isLoading = false;
691749
//reset audio URL
692750
/*
693751
audioPlayer.preload = 'none';
694752
audioPlayer.src = emptySound;
695753
*/
696754
//callback
697755
if (onErrorCallback) onErrorCallback();
756+
SepiaFW.audio.broadcastAudioEvent("effects", "error", audioPlayer);
698757
};
699758
audioPlayer.play();
700759
}

‎www/scripts/sepiaFW.ui.animate.js‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,11 @@ function sepiaFW_build_animate(){
33
var Animate = {};
44
Animate.assistant = {};
55
Animate.audio = {};
6+
Animate.wakeWord = {};
67
Animate.channels = {};
8+
9+
//timers
10+
//...
711

812
//general animations
913

@@ -30,17 +34,18 @@ function sepiaFW_build_animate(){
3034
&& (SepiaFW.assistant && SepiaFW.assistant.isWaitingForDialog)
3135
){
3236
//console.log('Mic auto-trigger window - source: ' + source);
33-
SepiaFW.speech.toggleSmartMic(); //TODO: TEST!
37+
SepiaFW.speech.toggleSmartMic();
3438
return true;
3539
}else{
3640
return false;
3741
}
3842
}
3943
}
4044
function possibilityToSwitchOnWakeWordListener(source){
45+
//check and schedule
4146
if (SepiaFW.wakeTriggers.useWakeWord && SepiaFW.wakeTriggers.engineLoaded && !SepiaFW.wakeTriggers.isListening()){
42-
//console.log('Wake-word window - source: ' + source); //TODO: implement
43-
SepiaFW.wakeTriggers.listenToWakeWords();
47+
//console.log('Wake-word window - source: ' + source); //TODO: use?
48+
SepiaFW.wakeTriggers.listenToWakeWords(undefined, undefined, true);
4449
}
4550
}
4651
function possibilityToCleanCommandQueue(){
@@ -160,7 +165,7 @@ function sepiaFW_build_animate(){
160165
document.dispatchEvent(event);
161166
}
162167

163-
//audio player animations
168+
//audio player animations:
164169

165170
Animate.audio.playerActive = function(){
166171
//$("#sepiaFW-audio-ctrls-title").addClass("playerActive");
@@ -171,7 +176,16 @@ function sepiaFW_build_animate(){
171176
$("#sepiaFW-audio-ctrls-soundbars").removeClass("playerActive");
172177
}
173178

174-
//channel animations for missed off-channel messages
179+
//wake-word animations:
180+
181+
Animate.wakeWord.active = function(){
182+
$('#sepiaFW-nav-label-online-status').addClass("wake-word-active");
183+
}
184+
Animate.wakeWord.inactive = function(){
185+
$('#sepiaFW-nav-label-online-status').removeClass("wake-word-active");
186+
}
187+
188+
//channel animations for missed off-channel messages:
175189

176190
var markedChannels = new Set();
177191
Animate.channels.setStateCheckChannels = function(){

‎www/scripts/sepiaFW.ui.build.js‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,12 +156,17 @@ function sepiaFW_build_ui_build(){
156156

157157
//Confirm box
158158
Build.askConfirm = function(msg, okCallback, abortCallback){
159-
var r = confirm(msg);
159+
/* var r = confirm(msg);
160160
if (r == true) {
161161
if (okCallback) okCallback();
162162
} else {
163163
if (abortCallback) abortCallback();
164-
}
164+
} */
165+
SepiaFW.ui.askForConfirmation(msg, function(){
166+
if (okCallback) okCallback();
167+
}, function(){
168+
if (abortCallback) abortCallback();
169+
});
165170
}
166171

167172
//MAIN UI FUNCTIONALITY

‎www/scripts/sepiaFW.ui.cards.js‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1068,7 +1068,7 @@ function sepiaFW_build_ui_cards(){
10681068
//Spotify
10691069
if (data.brand == "Spotify"){
10701070
var webPlayerDiv = document.createElement('DIV');
1071-
webPlayerDiv.className = "spotifyWebPlayer cardBodyItem fullWidthItem";
1071+
webPlayerDiv.className = "spotifyWebPlayer embeddedWebPlayer cardBodyItem fullWidthItem";
10721072
var contentUrl = "https://" + linkUrl.replace("spotify:", "open.spotify.com/embed/").replace(":play", "").replace(/:/g, "/").trim();
10731073
webPlayerDiv.innerHTML = '<iframe '
10741074
+ 'src="' + contentUrl + '" width="100%" height="80" frameborder="0" allowtransparency="true" '
@@ -1079,7 +1079,7 @@ function sepiaFW_build_ui_cards(){
10791079
//Apple Music
10801080
}else if (data.brand == "Apple Music"){
10811081
var webPlayerDiv = document.createElement('DIV');
1082-
webPlayerDiv.className = "appleMusicWebPlayer cardBodyItem fullWidthItem";
1082+
webPlayerDiv.className = "appleMusicWebPlayer embeddedWebPlayer cardBodyItem fullWidthItem";
10831083
var contentUrl;
10841084
if (linkUrl.indexOf("/artist/") > 0){
10851085
contentUrl = linkUrl.replace(/^https:\/\/.*?\//, "https://geo.itunes.apple.com/"); //TODO: basically not working
@@ -1101,7 +1101,7 @@ function sepiaFW_build_ui_cards(){
11011101
//console.log('YouTube'); //DEBUG
11021102
var webPlayerDiv = document.createElement('DIV');
11031103
var playerId = currentLinkItemId++;
1104-
webPlayerDiv.className = "youTubeWebPlayer cardBodyItem fullWidthItem"
1104+
webPlayerDiv.className = "youTubeWebPlayer embeddedWebPlayer cardBodyItem fullWidthItem"
11051105
var f = document.createElement('iframe');
11061106
f.id = 'youTubeWebPlayer-' + playerId;
11071107
f.allow = allowIframe;
@@ -1741,6 +1741,9 @@ function sepiaFW_build_ui_cards(){
17411741
}else if (data.info.playerState == 1){
17421742
clearTimeout(youtubeSkipTimer);
17431743
clearTimeout(youTubePlayConfirmTimer);
1744+
SepiaFW.audio.broadcastAudioEvent("youtube-embedded", "start");
1745+
}else if (data.info.playerState == 2){
1746+
SepiaFW.audio.broadcastAudioEvent("youtube-embedded", "pause");
17441747
}
17451748
}
17461749
}
@@ -1844,6 +1847,7 @@ function sepiaFW_build_ui_cards(){
18441847
if (Cards.youTubePlayerGetState() == 1 && !youTubePlayerStopRequested){
18451848
Cards.youTubePlayerControls("pause");
18461849
youTubePlayerIsOnHold = true;
1850+
SepiaFW.audio.broadcastAudioEvent("youtube-embedded", "hold");
18471851
return true;
18481852
}else{
18491853
return false;
@@ -1852,6 +1856,7 @@ function sepiaFW_build_ui_cards(){
18521856
onFadeInRequest: function(){
18531857
if (Cards.youTubePlayerIsOnHold() && Cards.youTubePlayerGetState() == 2){
18541858
Cards.youTubePlayerControls("resume");
1859+
SepiaFW.audio.broadcastAudioEvent("youtube-embedded", "resume");
18551860
return true;
18561861
}else{
18571862
return false;

0 commit comments

Comments
 (0)