added MFCC data to SEPIA vad worker + display in test;

Author: fquirin

src/modules/sepia-vad-worker.js

@@ -328,6 +328,7 @@ function process(data) {
 		//Process if we have enough frames
 		var vadResults = [];
 		var loudnessResults = [];
+		var mfcc = [];
 		while (_processRingBuffer.framesAvailable >= _vadBufferSize) {
 			//pull samples
 			_processRingBuffer.pull(_vadBuffer);
@@ -343,6 +344,8 @@ function process(data) {
 			_maxLoudness = Math.max(_maxLoudness, loudness);
 			_movingAvgLoudness = getWeightedMovingAverage(_movingAvgLoudness, loudness, _movingAvgLoudnessWeight);
 
+			mfcc.push(features.mfcc);
+			
 			//activity check
 			var voiceActivity = (loudness/_movingAvgLoudness) > (1 + vadMode/10)? 1 : 0;
 			vadResults.push(voiceActivity);
@@ -367,6 +370,7 @@ function process(data) {
 				voiceActivity: vadResults,
 				voiceEnergy: voiceEnergy,
 				voiceLoudness: loudnessResults,
+				mfcc: mfcc,
 				movingAvgLoudness: _movingAvgLoudness,
 				maxLoudness: _maxLoudness
 			});
@@ -387,8 +391,6 @@ function start(options) {
 function stop(options) {
 	//TODO: anything to do?
 	//NOTE: timing of this signal is not very well defined
-	_movingAvgLoudness = undefined;
-	_maxLoudness = 0;
 }
 function reset(options) {
     //TODO: clean up worker and prep. for restart
@@ -399,6 +401,8 @@ function release(options){
 	_processRingBuffer = null;
 	_vadBuffer = null;
 	_previousVadBuffer = null;
+	_movingAvgLoudness = undefined;
+	_maxLoudness = 0;
 }
 
 //--- helpers ---

test-commons.js

@@ -68,6 +68,12 @@ function usePlot(index, ele){
 		if (container) container.style.display = p.use? "" : "none";
 	}
 }
+function useHeatmap(index, ele){
+	useHeatmaps[index] = ele.checked;
+	if (heatmaps[index]) heatmaps[index].resetMax();
+}
+var useHeatmaps = {};
+var heatmaps = {};
 
 function addChartContainerToPage(){
 	var ele = document.createElement("div");

test.html

@@ -128,11 +128,13 @@ <h1>SEPIA Web Audio Processor</h1>
 		<label>Plot 2:</label><input type="checkbox" id="usePlot2" checked onchange="usePlot(2, this);">
 		<label>Plot 3:</label><input type="checkbox" id="usePlot3" onchange="usePlot(3, this);">
 		<label>Plot 4:</label><input type="checkbox" id="usePlot4" checked onchange="usePlot(4, this);">
+		<label>Heatmap 1:</label><input type="checkbox" id="useHeatmap1" onchange="useHeatmap(1, this);">
 	</div>
 	<div id="chart1" class="chart"></div>
 	<div id="chart2" class="chart"></div>
 	<div id="chart3" class="chart" style="display: none;"></div>
 	<div id="chart4" class="chart"></div>
+	<div id="heatmap1" class="chart"></div>
 </div>
 <script type='text/javascript' src="test-commons.js"></script>
 <script type='text/javascript'>
@@ -622,7 +624,25 @@ <h1>SEPIA Web Audio Processor</h1>
 				waveEncoderSetGate('close');
 			}
 		}
+		if (useHeatmaps[1] && data.mfcc != undefined){
+			if (!heatmap1){
+				let colorIndex = 4;
+				heatmap1 = new uPlot.lazy.Heatmap(document.getElementById("heatmap1"), {
+					dataPixelWidth: 4,
+					dataPixelHeight: 4,
+					colorIndex: colorIndex,
+					maxDataPoints: 150
+				});
+				heatmaps[1] = heatmap1;
+			}
+			data.mfcc.forEach(function(d, i){
+				heatmap1.addDataArray(d);
+			});
+			heatmap1.draw();
+		}
 	}
+	var heatmap1;
+	
 	var vadAutoActivate = true;
 	var vadAutoActivateSequenceWasTrigger = false;
 	var vadMinRecordTimeAfterActivate = 3000;