improved audio recorder and fixed dynamic downsampling

fquirin · fquirin · commit f023e1ee74a0 · 2019-09-14T12:47:33.000+02:00
diff --git a/www/scripts/recorder.js b/www/scripts/recorder.js
@@ -22,33 +22,23 @@ DEALINGS IN THE SOFTWARE.
 (function (window) {
 
     var Recorder = function(source, audioProcessor, startFun, stopFun) {
-        var bufferLen = 4096;
         var websocket;
         var audioContext = source.context;
+        var inputSampleRate = (audioContext)? audioContext.sampleRate : source.sampleRate;
+        var outputSampleRate = 16000;
 
         var recording = false;
 
         function processAudio(inputAudioFrame){
             if (!recording) return;
 
-            var length = Math.floor(inputAudioFrame.length / 3);
-            var result = new Float32Array(length);
-
-            var index = 0;
-            var inputIndex = 0;
-
-            while (index < length) {
-                result[index++] = inputAudioFrame[inputIndex];
-                inputIndex += 3;
-            }
+            //downsample
+            var result = downsampleFloat32(inputAudioFrame, inputSampleRate, outputSampleRate);
 
             var offset = 0;
-            var buffer = new ArrayBuffer(length * 2);
+            var buffer = new ArrayBuffer(result.length * 2);
             var view = new DataView(buffer);
-            for (var i = 0; i < result.length; i++, offset += 2) {
-                var s = Math.max(-1, Math.min(1, result[i]));
-                view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
-            }
+            floatTo16BitPCM(view, offset, result);
 
             //console.log('Recorder onaudioprocess - view: ' + view); 		//DEBUG
             if (websocket){
@@ -63,6 +53,7 @@ DEALINGS IN THE SOFTWARE.
             }
         //Web-Audio
         }else if (audioContext){
+            var bufferLen = 4096;
             let processNode;
             if ('createScriptProcessor' in audioContext){
                 processNode = audioContext.createScriptProcessor(bufferLen, 1, 1);
@@ -112,7 +103,6 @@ DEALINGS IN THE SOFTWARE.
         this.sendHeader = function(ws){
             var sampleLength = 1000000;
             var mono = true;
-            var sampleRate = 16000;                 //TODO: this might not be true! We cannot guarantee that!
             var buffer = new ArrayBuffer(44);
             var view = new DataView(buffer);
 
@@ -131,9 +121,9 @@ DEALINGS IN THE SOFTWARE.
             //channel count
             view.setUint16(22, mono ? 1 : 2, true);
             //sample rate
-            view.setUint32(24, sampleRate, true);
+            view.setUint32(24, outputSampleRate, true);
             //byte rate (sample rate * block align)
-            view.setUint32(28, sampleRate * 2, true);
+            view.setUint32(28, outputSampleRate * 2, true);
             //block align (channel count * bytes per sample)
             view.setUint16(32, 2, true);
             //bits per sample
@@ -146,11 +136,78 @@ DEALINGS IN THE SOFTWARE.
 			//console.log('Recorder sendHeader - view: ' + view); 		//DEBUG
             ws.send(view);
         }
-		function writeString(view, offset, string){
-			for (var i = 0; i < string.length; i++){
+        
+        //--- conversion methods ---
+
+        function writeString(view, offset, string){
+			for (let i = 0; i < string.length; i++){
 				view.setUint8(offset + i, string.charCodeAt(i));
 			}
-		}
+        }
+
+        function floatTo16BitPCM(output, offset, input){
+            for (let i = 0; i < input.length; i++, offset += 2){
+                let s = Math.max(-1, Math.min(1, input[i]));
+                output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
+            }
+        }
+
+        function downsampleAndConvertToInt16(buffer, sampleRate, outSampleRate){
+            if (outSampleRate == sampleRate){
+                return buffer;
+            }
+            if (outSampleRate > sampleRate){
+                console.error("Recorder - Downsampling to " + outSampleRate + " failed! Input sampling rate was too low: " + sampleRate);
+                return buffer;
+            }
+            var sampleRateRatio = sampleRate / outSampleRate;
+            var newLength = Math.round(buffer.length / sampleRateRatio);
+            var result = new Int16Array(newLength);
+            var offsetResult = 0;
+            var offsetBuffer = 0;
+            while (offsetResult < result.length){
+                var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
+                var accum = 0, count = 0;
+                for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++){
+                    accum += buffer[i];
+                    count++;
+                }
+                result[offsetResult] = Math.min(1, accum / count)*0x7FFF;
+                offsetResult++;
+                offsetBuffer = nextOffsetBuffer;
+            }
+            return result.buffer;
+        }
+        function downsampleFloat32(array, sampleRate, outSampleRate){
+            if (outSampleRate == sampleRate){
+                var result = new Float32Array(array.length);
+                for (let i = 0 ; i < array.length ; i++){
+                    result.push(array[i]);
+                }
+                return array;
+            }
+            if (outSampleRate > sampleRate){
+                console.error("Recorder - Downsampling to " + outSampleRate + " failed! Input sampling rate was too low: " + sampleRate);
+                return downsampleFloat32(array, sampleRate, sampleRate);
+            }
+            var sampleRateRatio = sampleRate / outSampleRate;
+            var newLength = Math.round(array.length / sampleRateRatio);
+            var result = new Float32Array(newLength);
+            var offsetResult = 0;
+            var offsetBuffer = 0;
+            while (offsetResult < result.length){
+                var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
+                var accum = 0, count = 0;
+                for (var i = offsetBuffer; i < nextOffsetBuffer && i < array.length; i++){
+                    accum += array[i];
+                    count++;
+                }
+                result[offsetResult] = accum / count;
+                offsetResult++;
+                offsetBuffer = nextOffsetBuffer;
+            }
+            return result;
+        }
     };
 
     window.RecorderJS = Recorder;
diff --git a/www/scripts/sepiaFW.audioRecorder.js b/www/scripts/sepiaFW.audioRecorder.js
@@ -113,17 +113,22 @@ function sepiaFW_build_audio_recorder(){
 		broadcastRecorderStopped();
 	}
 	function closeAudioContext(audioContext, success, error){
-		if (audioContext.state == "closed"){
-			if (success) success();
-		}else{
-			audioContext.close().then(function() {
-				//console.log('CLOSED audio-context');
-				broadcastRecorderClosed();
+		if (audioContext){
+			if (audioContext.state == "closed"){
 				if (success) success();
-			}).catch(function(e){
-				broadcastRecorderError(e);
-				if (error) error(e);
-			});
+			}else{
+				audioContext.close().then(function() {
+					//console.log('CLOSED audio-context');
+					broadcastRecorderClosed();
+					if (success) success();
+				}).catch(function(e){
+					broadcastRecorderError(e);
+					if (error) error(e);
+				});
+			}
+		}else{
+			broadcastRecorderClosed();
+			if (success) success();
 		}
 	}
 	
@@ -259,11 +264,12 @@ function sepiaFW_build_audio_recorder(){
 
 	//AudioProcessor (replacement for scriptProcessor of AudioContext)
 	function AudioInputPluginProcessor(){
-		this.onaudioprocess = function(inputAudioFrame){
+		var self = this;
+		self.onaudioprocess = function(inputAudioFrame){
 			//to be defined by RecorderInstance
 		};
-		this.onaudioreceive = function(evt){
-			this.onaudioprocess(evt.data);
+		self.onaudioreceive = function(evt){
+			self.onaudioprocess(evt.data);
 		};
 	}
 	
@@ -351,8 +357,8 @@ function sepiaFW_build_audio_recorder(){
 					bufferSize: 4096,
 					channels: 1,
 					format: audioinput.FORMAT.PCM_16BIT,
-					audioSourceType: audioinput.AUDIOSOURCE_TYPE.UNPROCESSED,	//VOICE_COMMUNICATION
-					normalize: false,
+					audioSourceType: audioinput.AUDIOSOURCE_TYPE.VOICE_COMMUNICATION,	//VOICE_COMMUNICATION UNPROCESSED DEFAULT
+					normalize: true,
 					streamToWebAudio: false
 				};
 				var audioProc = new AudioInputPluginProcessor();
diff --git a/www/xtensions/picovoice/picovoiceAudioManager.js b/www/xtensions/picovoice/picovoiceAudioManager.js
@@ -1,6 +1,4 @@
 let PicovoiceAudioManager = (function() {
-    const inputBufferLength = 4096;
-
     var engine;
     var processCallback;
     var isProcessing = false;
@@ -28,10 +26,13 @@ let PicovoiceAudioManager = (function() {
                 return;
             }
             //console.log('+');
-            //-------------------------
+            
+            //fill inputAudioBuffer
             for (let i = 0 ; i < inputAudioFrame.length ; i++) {
-                inputAudioBuffer.push((inputAudioFrame[i]) * 32767);
+                inputAudioBuffer.push((inputAudioFrame[i]) * 32767);    //0x7FFF
             }
+
+            //downsample if necessary
             while(inputAudioBuffer.length * engine.sampleRate / inputSampleRate > engine.frameLength) {
                 let result = new Int16Array(engine.frameLength);
                 let bin = 0;
@@ -56,7 +57,6 @@ let PicovoiceAudioManager = (function() {
 
                 processCallback(keywordIndex);
             }
-            //-------------------------
         };
 
         //Custom
@@ -66,6 +66,7 @@ let PicovoiceAudioManager = (function() {
             }
         //Web-Audio
         }else if (audioContext){
+            let inputBufferLength = 4096;
             let engineNode = audioContext.createScriptProcessor(inputBufferLength, 1, 1);
             engineNode.onaudioprocess = function(ev){
                 let inputAudioFrame = ev.inputBuffer.getChannelData(0);
