Skip to content

Commit bebcf66

Browse files
fquirinfquirin
committed
added vad.js; added active-element change event;
1 parent 3a50c00 commit bebcf66

File tree

3 files changed

+300
-1
lines changed

3 files changed

+300
-1
lines changed

‎www/index.html‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,7 @@ <h2><i class="material-icons md-txt">people</i>&nbsp;<script>SepiaFW.local.w('ac
308308
<!--<script src="https://code.jquery.com/jquery-3.1.1.min.js" integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8=" crossorigin="anonymous"></script>-->
309309
<script src="scripts/sjcl.js"></script>
310310
<script src="scripts/recorder.js"></script>
311+
<script src="scripts/vad.js"></script>
311312
<script src="scripts/hammer.min.js"></script>
312313
<script src="scripts/hammer-time.min.js"></script>
313314
<script src="scripts/clexi-0.8.0.js"></script>

‎www/scripts/sepiaFW.ui.js‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,30 @@ function sepiaFW_build_ui(){
7373
}
7474
}
7575

76+
//Listen to change of active element (Note: not very reliable)
77+
UI.listenToActiveElementChange = function(){
78+
window.addEventListener('focus', function(e){
79+
dispatchActiveElementChangeEvent();
80+
}, true);
81+
window.addEventListener('blur', function(e){
82+
dispatchActiveElementChangeEvent();
83+
}, true);
84+
}
85+
function dispatchActiveElementChangeEvent(){
86+
clearTimeout(activeElementChangeBuffer);
87+
activeElementChangeBuffer = setTimeout(function(){
88+
//note: cannot happen faster than every Xms
89+
var event = new CustomEvent('sepia_active_element_change', { detail: {
90+
id: document.activeElement.id,
91+
className: document.activeElement.className,
92+
tagName: document.activeElement.tagName
93+
}});
94+
document.dispatchEvent(event);
95+
console.error("new active ele.: " + (document.activeElement.id || document.activeElement.className || document.activeElement.tagName));
96+
}, 100);
97+
}
98+
var activeElementChangeBuffer = undefined;
99+
76100
//Open a view or frame by key (e.g. for URL parameter 'view=xy')
77101
UI.openViewOrFrame = function(openView){
78102
openView = openView.replace(".html", "").trim();
@@ -584,9 +608,10 @@ function sepiaFW_build_ui(){
584608
//listen to visibilityChangeEvent
585609
UI.listenToVisibilityChange();
586610

587-
//listen to mouse stuff
611+
//listen to mouse stuff and active element
588612
//UI.trackMouse();
589613
//UI.trackTouch();
614+
UI.listenToActiveElementChange();
590615

591616
//execute stuff on ready:
592617

@@ -1067,6 +1092,20 @@ function sepiaFW_build_ui(){
10671092
}
10681093
});
10691094
}
1095+
UI.askForConfirmation = function(question, allowedCallback, refusedCallback){
1096+
UI.showPopup(question, {
1097+
buttonOneName : SepiaFW.local.g('ok'),
1098+
buttonOneAction : function(){
1099+
//yes
1100+
if (allowedCallback) allowedCallback();
1101+
},
1102+
buttonTwoName : SepiaFW.local.g('abort'),
1103+
buttonTwoAction : function(){
1104+
//no
1105+
if (refusedCallback) refusedCallback();
1106+
}
1107+
});
1108+
}
10701109

10711110
//Test for support of special sepiaFW trigger events
10721111
UI.elementSupportsCustomTriggers = function(ele){

‎www/scripts/vad.js‎

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
/*
2+
https://github.com/kdavis-mozilla/vad.js
3+
BSD 3-Clause License
4+
5+
Copyright (c) 2015, Kelly Davis
6+
All rights reserved.
7+
8+
Redistribution and use in source and binary forms, with or without modification,
9+
are permitted provided that the following conditions are met:
10+
11+
* Redistributions of source code must retain the above copyright notice, this
12+
list of conditions and the following disclaimer.
13+
14+
* Redistributions in binary form must reproduce the above copyright notice, this
15+
list of conditions and the following disclaimer in the documentation and/or
16+
other materials provided with the distribution.
17+
18+
* Neither the name of the {organization} nor the names of its
19+
contributors may be used to endorse or promote products derived from
20+
this software without specific prior written permission.
21+
22+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
23+
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
24+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
25+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
26+
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
27+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
29+
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32+
*/
33+
34+
(function(window) {
35+
36+
var VAD = function(options) {
37+
// Default options
38+
this.options = {
39+
fftSize: 512,
40+
bufferLen: 512,
41+
voice_stop: function() {},
42+
voice_start: function() {},
43+
smoothingTimeConstant: 0.99,
44+
energy_offset: 1e-8, // The initial offset.
45+
energy_threshold_ratio_pos: 2, // Signal must be twice the offset
46+
energy_threshold_ratio_neg: 0.5, // Signal must be half the offset
47+
energy_integration: 1, // Size of integration change compared to the signal per second.
48+
filter: [
49+
{f: 200, v:0}, // 0 -> 200 is 0
50+
{f: 2000, v:1} // 200 -> 2k is 1
51+
],
52+
source: null,
53+
context: null
54+
};
55+
56+
// User options
57+
for(var option in options) {
58+
if(options.hasOwnProperty(option)) {
59+
this.options[option] = options[option];
60+
}
61+
}
62+
63+
// Require source
64+
if(!this.options.source)
65+
throw new Error("The options must specify a MediaStreamAudioSourceNode.");
66+
67+
// Set this.options.context
68+
this.options.context = this.options.source.context;
69+
70+
// Calculate time relationships
71+
this.hertzPerBin = this.options.context.sampleRate / this.options.fftSize;
72+
this.iterationFrequency = this.options.context.sampleRate / this.options.bufferLen;
73+
this.iterationPeriod = 1 / this.iterationFrequency;
74+
75+
var DEBUG = true;
76+
if(DEBUG) console.log(
77+
'Vad' +
78+
' | sampleRate: ' + this.options.context.sampleRate +
79+
' | hertzPerBin: ' + this.hertzPerBin +
80+
' | iterationFrequency: ' + this.iterationFrequency +
81+
' | iterationPeriod: ' + this.iterationPeriod
82+
);
83+
84+
this.setFilter = function(shape) {
85+
this.filter = [];
86+
for(var i = 0, iLen = this.options.fftSize / 2; i < iLen; i++) {
87+
this.filter[i] = 0;
88+
for(var j = 0, jLen = shape.length; j < jLen; j++) {
89+
if(i * this.hertzPerBin < shape[j].f) {
90+
this.filter[i] = shape[j].v;
91+
break; // Exit j loop
92+
}
93+
}
94+
}
95+
}
96+
97+
this.setFilter(this.options.filter);
98+
99+
this.ready = {};
100+
this.vadState = false; // True when Voice Activity Detected
101+
102+
// Energy detector props
103+
this.energy_offset = this.options.energy_offset;
104+
this.energy_threshold_pos = this.energy_offset * this.options.energy_threshold_ratio_pos;
105+
this.energy_threshold_neg = this.energy_offset * this.options.energy_threshold_ratio_neg;
106+
107+
this.voiceTrend = 0;
108+
this.voiceTrendMax = 10;
109+
this.voiceTrendMin = -10;
110+
this.voiceTrendStart = 5;
111+
this.voiceTrendEnd = -5;
112+
113+
// Create analyser
114+
this.analyser = this.options.context.createAnalyser();
115+
this.analyser.smoothingTimeConstant = this.options.smoothingTimeConstant; // 0.99;
116+
this.analyser.fftSize = this.options.fftSize;
117+
118+
this.floatFrequencyData = new Float32Array(this.analyser.frequencyBinCount);
119+
120+
// Setup local storage of the Linear FFT data
121+
this.floatFrequencyDataLinear = new Float32Array(this.floatFrequencyData.length);
122+
123+
// Connect this.analyser
124+
this.options.source.connect(this.analyser);
125+
126+
// Create ScriptProcessorNode
127+
this.scriptProcessorNode = this.options.context.createScriptProcessor(this.options.bufferLen, 1, 1);
128+
129+
// Connect scriptProcessorNode (Theretically, not required)
130+
this.scriptProcessorNode.connect(this.options.context.destination);
131+
132+
// Create callback to update/analyze floatFrequencyData
133+
var self = this;
134+
this.scriptProcessorNode.onaudioprocess = function(event) {
135+
self.analyser.getFloatFrequencyData(self.floatFrequencyData);
136+
self.update();
137+
self.monitor();
138+
};
139+
140+
// Connect scriptProcessorNode
141+
this.options.source.connect(this.scriptProcessorNode);
142+
143+
// log stuff
144+
this.logging = false;
145+
this.log_i = 0;
146+
this.log_limit = 100;
147+
148+
this.triggerLog = function(limit) {
149+
this.logging = true;
150+
this.log_i = 0;
151+
this.log_limit = typeof limit === 'number' ? limit : this.log_limit;
152+
}
153+
154+
this.log = function(msg) {
155+
if(this.logging && this.log_i < this.log_limit) {
156+
this.log_i++;
157+
console.log(msg);
158+
} else {
159+
this.logging = false;
160+
}
161+
}
162+
163+
this.update = function() {
164+
// Update the local version of the Linear FFT
165+
var fft = this.floatFrequencyData;
166+
for(var i = 0, iLen = fft.length; i < iLen; i++) {
167+
this.floatFrequencyDataLinear[i] = Math.pow(10, fft[i] / 10);
168+
}
169+
this.ready = {};
170+
}
171+
172+
this.getEnergy = function() {
173+
if(this.ready.energy) {
174+
return this.energy;
175+
}
176+
177+
var energy = 0;
178+
var fft = this.floatFrequencyDataLinear;
179+
180+
for(var i = 0, iLen = fft.length; i < iLen; i++) {
181+
energy += this.filter[i] * fft[i] * fft[i];
182+
}
183+
184+
this.energy = energy;
185+
this.ready.energy = true;
186+
187+
return energy;
188+
}
189+
190+
this.monitor = function() {
191+
var energy = this.getEnergy();
192+
var signal = energy - this.energy_offset;
193+
194+
if(signal > this.energy_threshold_pos) {
195+
this.voiceTrend = (this.voiceTrend + 1 > this.voiceTrendMax) ? this.voiceTrendMax : this.voiceTrend + 1;
196+
} else if(signal < -this.energy_threshold_neg) {
197+
this.voiceTrend = (this.voiceTrend - 1 < this.voiceTrendMin) ? this.voiceTrendMin : this.voiceTrend - 1;
198+
} else {
199+
// voiceTrend gets smaller
200+
if(this.voiceTrend > 0) {
201+
this.voiceTrend--;
202+
} else if(this.voiceTrend < 0) {
203+
this.voiceTrend++;
204+
}
205+
}
206+
207+
var start = false, end = false;
208+
if(this.voiceTrend > this.voiceTrendStart) {
209+
// Start of speech detected
210+
start = true;
211+
} else if(this.voiceTrend < this.voiceTrendEnd) {
212+
// End of speech detected
213+
end = true;
214+
}
215+
216+
// Integration brings in the real-time aspect through the relationship with the frequency this functions is called.
217+
var integration = signal * this.iterationPeriod * this.options.energy_integration;
218+
219+
// Idea?: The integration is affected by the voiceTrend magnitude? - Not sure. Not doing atm.
220+
221+
// The !end limits the offset delta boost till after the end is detected.
222+
if(integration > 0 || !end) {
223+
this.energy_offset += integration;
224+
} else {
225+
this.energy_offset += integration * 10;
226+
}
227+
this.energy_offset = this.energy_offset < 0 ? 0 : this.energy_offset;
228+
this.energy_threshold_pos = this.energy_offset * this.options.energy_threshold_ratio_pos;
229+
this.energy_threshold_neg = this.energy_offset * this.options.energy_threshold_ratio_neg;
230+
231+
// Broadcast the messages
232+
if(start && !this.vadState) {
233+
this.vadState = true;
234+
this.options.voice_start();
235+
}
236+
if(end && this.vadState) {
237+
this.vadState = false;
238+
this.options.voice_stop();
239+
}
240+
241+
this.log(
242+
'e: ' + energy +
243+
' | e_of: ' + this.energy_offset +
244+
' | e+_th: ' + this.energy_threshold_pos +
245+
' | e-_th: ' + this.energy_threshold_neg +
246+
' | signal: ' + signal +
247+
' | int: ' + integration +
248+
' | voiceTrend: ' + this.voiceTrend +
249+
' | start: ' + start +
250+
' | end: ' + end
251+
);
252+
253+
return signal;
254+
}
255+
};
256+
257+
window.VAD = VAD;
258+
259+
})(window);

0 commit comments

Comments
 (0)