changed the way 'vadMode' works for SEPIA VAD worker

Author: fquirin

src/modules/sepia-vad-worker.js

@@ -42,10 +42,16 @@ let channelCount;
 let inputSampleSize;
 let processBufferSize;	//defines '_processRingBuffer' size together with 'inputSampleSize'
 let vadMode;
+let vadThreshold;
+let vadDefaultThresholds = {
+	1: 1.5,
+	2: 3.5,
+	3: 3.5
+}
 let isFloat32Input;		//default false
 
 let voiceEnergy;
-let voiceEnergyCap = 50;
+let voiceEnergyCap = 42;
 let voiceEnergyDropRate = 2;
 let _samplesToTimeMsFactor;
 
@@ -60,8 +66,9 @@ let _transferFun;
 //parameters to calculate vad
 let movingAvgLoudness;
 let maxLoudness = 0;
-let _movingAvgLoudnessWeight = 800;		//TODO: make variable and normalize with sample-rate
-let _vadThreshold;
+let _movingAvgLoudnessWeight = 400;		//TODO: make variable and normalize with sample-rate
+let _mfccDynamicWeightsArray;
+let _mfccLastArray;
 let _warmUpFrames;
 let _totalFrames;
 
@@ -98,15 +105,22 @@ function init(){
 	var allowedBufferSizes = [8192, 4096, 2048, 1024, 512, 256, 128];		//recommended: 10-30ms frame length, e.g. 512/16000 = 32ms (recommended)
 	_vadBufferSize = 0;
 	for (let i=0; i<allowedBufferSizes.length; i++){
-		if (processBufferSize == allowedBufferSizes[i] || processBufferSize % allowedBufferSizes[i] == 0){
-			_vadFrames = processBufferSize / allowedBufferSizes[i];
-			_vadBufferSize = allowedBufferSizes[i];
-			break;
+		//common cases
+		if (inputSampleRate == 16000 && processBufferSize >= 512){
+			_vadBufferSize = 512;
+			
+		//best fallback
+		}else{
+			if (processBufferSize == allowedBufferSizes[i] || processBufferSize % allowedBufferSizes[i] == 0){
+				_vadBufferSize = allowedBufferSizes[i];
+				break;
+			}
 		}
 	}
 	if (_vadBufferSize == 0){
 		throw JSON.stringify(new BufferSizeException("The 'bufferSize' has to be equal or a multiple of: " + allowedBufferSizes.join(", ")));
 	}else{
+		_vadFrames = processBufferSize / _vadBufferSize;
 		_samplesToTimeMsFactor = 1000/inputSampleRate;
 		_vadFrameTimeMs = Math.round(_vadBufferSize * _samplesToTimeMsFactor);
 		if (_vadFrameTimeMs < 5 || _vadFrameTimeMs > 86){
@@ -129,12 +143,21 @@ function init(){
 		}
 	}
 
+	//Meyda requirements (2)
+	Meyda.sampleRate = inputSampleRate;
+	Meyda.bufferSize = _vadBufferSize;
+	if (!Meyda.bufferSize || (Meyda.bufferSize & (Meyda.bufferSize -1) != 0)){
+		throw JSON.stringify({name: "VadModuleMeydaError", message: "Meyda buffer-size must be power of 2, e.g. 128, 256, 512, 1024, ..."});
+	}
+	
 	movingAvgLoudness = undefined;
 	maxLoudness = 0;
+	_mfccDynamicWeightsArray = ArrayOps.newCommon1dArray(Meyda.numberOfMFCCCoefficients, 1);
+	_mfccLastArray = ArrayOps.newCommon1dArray(Meyda.numberOfMFCCCoefficients, 0);
 
 	if (useSequenceAnalyzer){
 		//Buffer the start of a sequence to analyze for keywords/trigger/wake-words etc.
-		let sequenceStartFrames = Math.round((sequenceTimeForTrigger + 1000)/1000 * (inputSampleRate/inputSampleSize));		//TODO: why is this almost 2 times more than expected?
+		let sequenceStartFrames = Math.round((sequenceTimeForTrigger + 1000)/1000 * (inputSampleRate/_vadBufferSize));		//TODO: why is this almost 2 times more than expected?
 		mfccSequenceStartBuffer = ArrayOps.newCommon2dArray(sequenceStartFrames, Meyda.numberOfMFCCCoefficients, 0);
 		loudnessSequenceStartBuffer = ArrayOps.newCommon1dArray(sequenceStartFrames, 0);
 		feature1SequenceStartBuffer = ArrayOps.newCommon1dArray(sequenceStartFrames, 0);
@@ -156,6 +179,7 @@ function ready(){
 			inputIsFloat32: isFloat32Input,
 			processBufferSize: processBufferSize,
 			vadMode: vadMode,
+			vadThreshold: vadThreshold,
 			vadFramesMax: _vadFrames,
 			vadBufferSize: _vadBufferSize,
 			vadFrameTimeMs: _vadFrameTimeMs,
@@ -171,8 +195,8 @@ function constructWorker(options) {
 	channelCount = 1;	//options.setup.channelCount || 1;		//TODO: only MONO atm
 	inputSampleSize = options.setup.inputSampleSize || 512;
 	processBufferSize = options.setup.bufferSize || inputSampleSize;
-	vadMode = (options.setup.vadMode != undefined)? options.setup.vadMode : 3;
-	_vadThreshold = (1 + vadMode/10);
+	vadMode = options.setup.vadMode || 3;
+	vadThreshold = options.setup.vadThreshold || vadDefaultThresholds[vadMode] || 3;
 	_warmUpFrames = Math.round(2*inputSampleRate/inputSampleSize);		//input- or processBufferSize? We want ~2s so input makes sense
 	_totalFrames = 0;
 
@@ -193,8 +217,8 @@ function constructWorker(options) {
 	}
 
 	//Meyda options and defaults
-	Meyda.melBands = 26;
-	Meyda.numberOfMFCCCoefficients = 13;
+	Meyda.melBands = 40; //40 26;
+	Meyda.numberOfMFCCCoefficients = 20; //13;
 	Meyda.windowingFunction = "hanning";	//"hamming"
 	var meydaRequiredFeatures = ["mfcc", "loudness"];		
 	//https://meyda.js.org/audio-features.html: "spectralCentroid", "spectralFlatness", "spectralFlux" (requires previous spec. but is buggy!?)
@@ -211,12 +235,7 @@ function constructWorker(options) {
 			}
 		});
 	}
-	//Meyda requirements
-	Meyda.sampleRate = inputSampleRate;
-	Meyda.bufferSize = inputSampleSize;
-	if (!Meyda.bufferSize || (Meyda.bufferSize & (Meyda.bufferSize -1) != 0)){
-		throw {name: "VadModuleMeydaError", message: "Meyda buffer-size must be power of 2, e.g. 128, 256, 512, 1024, ..."};
-	}
+	//Meyda requirements (1)
 	if (!meydaFeatures){
 		Meyda.features = meydaRequiredFeatures;
 	}else{
@@ -335,11 +354,11 @@ function resetSequence(){
 	_sequenceIsDone = false;
 	_sequenceCheckedTrigger = false;
 }
-function registerEvent(code, msg, data){
+function registerEvent(code, _msg, data){
 	var msg = {
 		vadSequenceCode: code,
-		vadSequenceMsg: msg
-	};
+		vadSequenceMsg: _msg
+	}
 	switch (code){
 		//case 1: voice start
 		//case 2: sequence start
@@ -363,6 +382,27 @@ function registerEvent(code, msg, data){
 	postMessage(msg);
 }
 
+//classify voice activity
+function getVoiceActivity(mfccArray, loudnessNorm, averageLoudness){
+	if (vadMode == 3){
+		var sum = 0;
+		for (let i=0; i<mfccArray.length; i++){
+			let change = Math.abs(mfccArray[i]/_mfccLastArray[i] - 1);
+			if (change < 0.20){
+				_mfccDynamicWeightsArray[i] = _mfccDynamicWeightsArray[i] * 0.66;
+			}else{
+				_mfccDynamicWeightsArray[i] = 1.0; //Math.min(1.0, _mfccDynamicWeightsArray[i] + 0.75);
+			}
+			_mfccLastArray[i] = mfccArray[i];
+			sum += Math.abs(mfccArray[i] * _mfccDynamicWeightsArray[i]);
+		}
+		var signal = sum/mfccArray.length - averageLoudness;
+		return (signal > vadThreshold? 1 : 0);		
+	}else{
+		return (loudnessNorm > vadThreshold? 1 : 0);
+	}
+}
+
 function process(data) {
 	//expected: data.samples, data.sampleRate, data.channels, data.type
 	//might have: data.rms	-	TODO: make use of?
@@ -402,34 +442,38 @@ function process(data) {
 		let mfcc = new Array(vadFramesAvailable);
 		//let moreFeatures = new Array(vadFramesAvailable);
 		let n = 0;
-		while (_processRingBuffer.framesAvailable >= _vadBufferSize) {
+		while (_processRingBuffer.framesAvailable >= _vadBufferSize){
 			//pull samples
 			_processRingBuffer.pull(_vadBuffer);
 
 			//Meyda features
 			let features = Meyda.extract(Meyda.features, _vadBuffer[0]);	//we don't add '_previousVadBuffer[0]' because it saves time and 'spectralFlux' is buggy
 			//console.log("features_meyda", features);
 
-			//let loudness = (features.loudness.specific[1] + features.loudness.specific[2] + features.loudness.specific[3]); //'specific' shows each loudness on bark scale, 'total' is the sum
-			//let loudness = features.loudness.total;
-			let loudness = features.loudness.specific.slice(1, 5).reduce(function(a, b){ return a + b; });
+			let loudness;
+			//loudness = (features.loudness.specific[1] + features.loudness.specific[2] + features.loudness.specific[3]); //'specific' shows each loudness on bark scale, 'total' is the sum
+			//loudness = features.loudness.total;
+			if (vadMode == 1){
+				loudness = features.loudness.specific.slice(1, 5).reduce(function(a, b){ return a + b; });		//1-5 on the bark scale
+			}else{
+				loudness = features.mfcc.reduce(function(a, c){ return (a + Math.abs(c)); })/features.mfcc.length;	//avg(...abs(MFCC[i]))
+			}
 			maxLoudness = Math.max(maxLoudness, loudness);
 			if (movingAvgLoudness == undefined){
-				movingAvgLoudness = loudness * _vadThreshold;
+				movingAvgLoudness = loudness * vadThreshold;
 			}
 			if (_totalFrames < _warmUpFrames){
 				movingAvgLoudness = getWeightedMovingAverage(movingAvgLoudness, loudness, 10);
 			}else{
 				movingAvgLoudness = getWeightedMovingAverage(movingAvgLoudness, loudness, _movingAvgLoudnessWeight);
 			}
 
-			mfcc[n] = features.mfcc;
-			
 			//activity check
-			let loudnessNorm = (loudness/movingAvgLoudness);
-			let voiceActivity = loudnessNorm > _vadThreshold? 1 : 0;
+			let loudnessNorm = (loudness - movingAvgLoudness);
+			let voiceActivity = getVoiceActivity(features.mfcc, loudnessNorm, movingAvgLoudness);
 			vadResults[n] = voiceActivity;
 			loudnessResults[n] = loudness;
+			mfcc[n] = features.mfcc;
 			//moreFeatures[n] = [];
 
 			//voice energy and sequence check
@@ -442,10 +486,10 @@ function process(data) {
 			}
 			if (useSequenceAnalyzer){
 				ArrayOps.pushAndShift(mfccSequenceStartBuffer, features.mfcc);
-				ArrayOps.pushAndShift(loudnessSequenceStartBuffer, (loudnessNorm > 1)? (loudnessNorm-1) : 0);	//loudness
+				ArrayOps.pushAndShift(loudnessSequenceStartBuffer, loudnessNorm); //(loudnessNorm > 1)? (loudnessNorm-1) : 0);	//loudness
 				ArrayOps.pushAndShift(feature1SequenceStartBuffer, voiceActivity);
 				ArrayOps.pushAndShift(feature2SequenceStartBuffer, voiceEnergy);
-				ArrayOps.pushAndShift(feature3SequenceStartBuffer, 0);
+				ArrayOps.pushAndShift(feature3SequenceStartBuffer, features.loudness.specific);
 				sequenceDetector(voiceActivity);
 			}
 			n++;
@@ -490,6 +534,7 @@ function release(options){
 	//_previousVadBuffer = null;
 	_totalFrames = 0;
 	mfccSequenceStartBuffer = null;
+	_mfccDynamicWeightsArray = null;
 	loudnessSequenceStartBuffer = null;
 	feature1SequenceStartBuffer = null;
 	feature2SequenceStartBuffer = null;

test-commons.js

@@ -112,13 +112,19 @@ function plotData(data, plotIndex, expandData){
 		}
 	}else{
 		var ele = addChartContainerToPage();
-		var x = uPlot.lazy.createSequence(0, data.length);
-		uPlot.lazy.plot({
+		var conf = {
 			targetElement: ele,
 			showPoints: false,
-			strokeWidth: 1,
-			data: [x, data]
-		});
+			strokeWidth: 1
+		}
+		if (expandData){
+			var x = uPlot.lazy.createSequence(0, data[0].length);
+			conf.data = [x, ...data];
+		}else{
+			var x = uPlot.lazy.createSequence(0, data.length);
+			conf.data = [x, data];
+		}
+		uPlot.lazy.plot(conf);
 	}
 }
 function drawHeatmap(data, hmIndex, maxPoints){
@@ -138,4 +144,12 @@ function drawHeatmap(data, hmIndex, maxPoints){
 		heatmap.addDataArray(d);
 	});
 	heatmap.draw();
+}
+function createArrayWithStartValue(n, startValue){
+	if (startValue == undefined) startValue = 0;
+	var array = new Array(n);
+	for (let i=0; i<n; i++){
+		array[i] = startValue;
+	}
+	return array;
 }

test2.html

@@ -6,23 +6,23 @@
 	<meta name="viewport" content="width=device-width, initial-scale=1">
 	<title>SEPIA Web Audio Processor Test</title>
 
-	<script src="src/visualization/uPlot.iife.min.js"></script>
-	<script src="src/visualization/uPlot-lazy.min.js"></script>
-	<script src="src/visualization/uPlot-lazy-heatmap.min.js"></script>
-	<link rel="stylesheet" href="src/visualization/uPlot.min.css">
+	<script src="src/visualization/uPlot.iife.min.js?v=0.6.9"></script>
+	<script src="src/visualization/uPlot-lazy.min.js?v=0.6.9"></script>
+	<script src="src/visualization/uPlot-lazy-heatmap.min.js?v=0.6.9"></script>
+	<link rel="stylesheet" href="src/visualization/uPlot.min.css?v=0.6.9">
 
-	<script src="src/polyfills/audioworklet-polyfill.js"></script>
-	<script type="text/javascript" src="src/processor.js"></script>
+	<script src="src/polyfills/audioworklet-polyfill.js?v=0.6.9"></script>
+	<script type="text/javascript" src="src/processor.js?v=0.6.9"></script>
 	<script>
 		//set correct modules folder
 		SepiaFW.webAudio.defaultProcessorOptions.moduleFolder = "src/modules";
 	</script>
 
-	<script type="text/javascript" src="src/resources/pcm-convert.js"></script>
-	<script type="text/javascript" src="src/resources/opus-file-splitter.js"></script>
-	<script type="text/javascript" src="src/resources/fir-filter-resampler.js"></script>
+	<script type="text/javascript" src="src/resources/pcm-convert.js?v=0.6.9"></script>
+	<script type="text/javascript" src="src/resources/opus-file-splitter.js?v=0.6.9"></script>
+	<script type="text/javascript" src="src/resources/fir-filter-resampler.js?v=0.6.9"></script>
 
-	<link rel="stylesheet" type="text/css" href="tests.css">
+	<link rel="stylesheet" type="text/css" href="tests.css?v=0.6.9">
 	<style></style>
 </head>
 <body>
@@ -73,7 +73,7 @@ <h1>SEPIA Web Audio Processor</h1>
 		<label><b>Resampler (Speex WASM)</b></label>
 		<label>Sample-Rate:</label><input id="resamplerSampleRate" class="small" value="16000" onchange="setResamplerSampleRate(this);" placeholder="16000">
 		<label>Buffer-Size:</label><input id="resamplerBufferSize" class="small" value="512" onchange="setResamplerBufferSize(this);" placeholder="512"><div style="flex: 1 0 100%;"></div>
-		<label>Quality:</label><input id="resampleQuality" class="small" value="7" onchange="setResampleQuality(this);" placeholder="0-10 (0: fastest, 10: best)">
+		<label>Quality:</label><input id="resampleQuality" class="small" value="3" onchange="setResampleQuality(this);" placeholder="0-10 (0: fastest, 10: best)">
 		<div id="resamplerGainBox" class="slidecontainer" style="display: none;">
 			<label>Gain:</label><input id="resamplerGain" type="range" min="1" max="50" step="1" value="1" class="slider"><input type="number" id="resamplerGainShow" class="small">
 		</div>
@@ -85,7 +85,8 @@ <h1>SEPIA Web Audio Processor</h1>
 	<div id="vadWorkerControls" class="contorlGroup">
 		<label><b>VAD</b></label>
 		<label>Buffer-Size:</label><input id="vadWorkerBufferSize" class="small" value="512" placeholder="960" onchange="setVadWorkerBufferSize(+this.value);">
-		<label>Mode:</label><input id="vadWorkerVadMode" class="small" value="3" placeholder="1-3 (3: agressive)" onchange="setVadWorkerVadMode(+this.value);">
+		<label>Mode:</label><input id="vadWorkerVadMode" class="small" type="number" value="1" placeholder="1-3 (3: agressive)" onchange="setVadWorkerVadMode(+this.value);">
+		<label>Threshold:</label><input id="vadWorkerVadThreshold" class="small" type="number" value="0" step="0.1" title="e.g. ~1.3 (mode 1) or ~4 (mode 2-3)" onchange="setVadWorkerVadThreshold(+this.value);">
 	</div>
 	<div id="wakeWordWorkerControls" class="contorlGroup">
 		<label><b>Wake-Word</b></label>
@@ -340,6 +341,7 @@ <h1>SEPIA Web Audio Processor</h1>
 						inputSampleSize: resamplerBufferSize,	//output bufferSize of previous module
 						bufferSize: vadWorkerBufferSize,
 						vadMode: vadWorkerVadMode,
+						vadThreshold: vadWorkerVadThreshold,
 						//voiceEnergyCap: 50,
 						//voiceEnergyDropRate: 2,
 						sequence: {
@@ -597,8 +599,10 @@ <h1>SEPIA Web Audio Processor</h1>
 	}
 	function setVoiceActivity(data){
 		//console.log("vad", data);
-		if (data.voiceLoudness != undefined && data.voiceLoudness.length == 1){
-			plotData([[data.voiceLoudness[0]/data.movingAvgLoudness], 1, [data.maxLoudness/data.movingAvgLoudness]], 3, true);	//[data.movingAvgLoudness]
+		if (data.voiceLoudness != undefined){
+			data.voiceLoudness.forEach(function(ld){
+				plotData([[ld - data.movingAvgLoudness], 0, [data.maxLoudness - data.movingAvgLoudness]], 3, true);	//[data.movingAvgLoudness]
+			});
 		}else if (data.voiceActivity != undefined){
 			plotData(data.voiceActivity, 3);
 		}
@@ -628,12 +632,14 @@ <h1>SEPIA Web Audio Processor</h1>
 				console.log("vadSequenceCode=6", data);
 				addTitleToPage("VAD result group");
 				if (data.loudnessProfile){
-					plotData(data.loudnessProfile);
+					//plotData(data.loudnessProfile);
+					plotData([data.loudnessProfile, createArrayWithStartValue(data.loudnessProfile.length, vadWorkerVadThreshold)], undefined, true);
 				}
 				if (data.featuresArray){
 					plotData(data.featuresArray[0]);
 					plotData(data.featuresArray[1]);
-					plotData(data.featuresArray[2]);
+					//plotData(data.featuresArray[2]);
+					drawHeatmap(data.featuresArray[2], undefined, data.featuresArray[2].length);
 				}
 				if (data.mfccProfile){
 					drawHeatmap(data.mfccProfile, undefined, data.mfccProfile.length);
@@ -802,14 +808,18 @@ <h1>SEPIA Web Audio Processor</h1>
 	useWaveEncoder(doUseWaveEncoder);
 
 	var doUseVadWorker = document.getElementById("useVadWorker").checked;
-	var vadWorkerBufferSize = +document.getElementById("vadWorkerBufferSize").value; 	//e.g. 960;
-	var vadWorkerVadMode = +document.getElementById("vadWorkerVadMode").value; 			//e.g. 3;
+	var vadWorkerBufferSize = +document.getElementById("vadWorkerBufferSize").value; 	//e.g. 512 (Meyda) or 960 (WebRTC);
+	var vadWorkerVadMode = +document.getElementById("vadWorkerVadMode").value; 				//e.g. 3;
+	var vadWorkerVadThreshold = +document.getElementById("vadWorkerVadThreshold").value; 	//e.g. 4 (Meyda only, mode 1: 1.3, mode 2-n: 3-4;
 	function setVadWorkerBufferSize(val){
 		vadWorkerBufferSize = val;
 	}
 	function setVadWorkerVadMode(val){
 		vadWorkerVadMode = val;
 	}
+	function setVadWorkerVadThreshold(val){
+		vadWorkerVadThreshold = val;
+	}
 	function useVadWorker(useIt){
 		doUseVadWorker = useIt;
 		if (useIt){