New Emscripten audioWorklet draft

addons/ofxEmscripten/libs/html5audio/include/html5audio.h

@@ -28,7 +28,7 @@ extern "C"{
 	extern void html5audio_sound_set_pan(int sound, double pan);
 	extern void html5audio_sound_free(int sound);
 
-	extern void html5audio_stream_create(int bufferSize, int inputChannels, int outputChannels, float * inbuffer, float * outbuffer, html5audio_stream_callback callback, void * userData);
+	extern void html5audio_stream_create(int audioWorkletNode, int numInputChannels);
 	extern void html5audio_stream_free();
 	extern bool html5audio_sound_is_loaded(int sound);
 }

addons/ofxEmscripten/libs/html5audio/lib/emscripten/library_html5audio.js

@@ -24,11 +24,13 @@ var LibraryHTML5Audio = {
         }
     },
 
+    html5audio_context_create__deps: ['$emscriptenRegisterAudioObject'],
     html5audio_context_create: function () {
        try {
             // Fix up for prefixing
             window.AudioContext = window.AudioContext || window.webkitAudioContext;
-            var context = new AudioContext({});
+            var context = new AudioContext({ sampleRate: 44100 });
+            var id = emscriptenRegisterAudioObject(context);
 
             // Fix issue with chrome autoplay policy
             document.addEventListener('mousedown', function cb(event) {
@@ -44,7 +46,7 @@ var LibraryHTML5Audio = {
             fft.maxDecibels = 0;
             fft.minDecibels = -100;
             AUDIO.fft = fft;
-            return 0;
+            return id;
         } catch (e) {
             console.log('Web Audio API is not supported in this browser', e);
             return -1;
@@ -62,7 +64,11 @@ var LibraryHTML5Audio = {
     html5audio_context_spectrum: function (bands, spectrum) {
         AUDIO.fft.fftSize = bands * 2;
         var spectrumArray = Module.HEAPF32.subarray(spectrum >> 2, (spectrum >> 2) + bands);
-        AUDIO.fft.getFloatFrequencyData(spectrumArray);
+        var spectrumArrayCopy = new Float32Array (spectrumArray);
+        AUDIO.fft.getFloatFrequencyData(spectrumArrayCopy);
+        for (let i = 0; i < spectrumArrayCopy.length; i++) {
+            spectrumArray[i] = spectrumArrayCopy[i];
+        }
     },
 
     html5audio_context_samplerate: function () {
@@ -97,7 +103,7 @@ var LibraryHTML5Audio = {
                 var fileSizeInBytes = stats.size;
 
                 var tag = ext; //this covers most types
-                if( ext == mp3 ){
+                if( ext == 'mp3'){
                     tag = 'mpeg';
                 }else if( ext == 'oga'){
                     tag = 'ogg';
@@ -201,48 +207,21 @@ var LibraryHTML5Audio = {
 	}
     },
 
-    html5audio_stream_create: function(bufferSize, inputChannels, outputChannels, inbuffer, outbuffer, callback, userData) {
-            var stream = AUDIO.context.createScriptProcessor(bufferSize, inputChannels, outputChannels);
-            var inbufferArray = Module.HEAPF32.subarray(inbuffer >> 2, (inbuffer >> 2) + bufferSize * inputChannels);
-            var outbufferArray = Module.HEAPF32.subarray(outbuffer >> 2, (outbuffer >> 2) + bufferSize * outputChannels);
-
-            stream.onaudioprocess = function(event) {
-                var i, j, c;
-                if (inputChannels > 0) {
-                    for (c = 0; c < inputChannels; ++c) {
-                        var inChannel = event.inputBuffer.getChannelData(c);
-                        for (i = 0, j = c; i < bufferSize; ++i, j += inputChannels) {
-                            inbufferArray[j] = inChannel[i];
-                        }
-                    }
-                }
-
-                {{{ makeDynCall('viiii', 'callback') }}}(bufferSize, inputChannels, outputChannels, userData);
-
-                if (outputChannels > 0) {
-                    for (c = 0; c < outputChannels; ++c) {
-                        var outChannel = event.outputBuffer.getChannelData(c);
-                        for (i = 0, j = c; i < bufferSize; ++i, j += outputChannels) {
-                            outChannel[i] = outbufferArray[j];
-                        }
-                    }
-                }
-            };
-
-            if (inputChannels > 0) {
-                navigator.mediaDevices.getUserMedia({ audio: true })
-                    .then(function (audioIn) {
-                        var mediaElement = AUDIO.context.createMediaStreamSource(audioIn);
-                        mediaElement.connect(stream);
-                        AUDIO.mediaElement = mediaElement;
-                    })
-                    .catch(function (error) {
-                        console.log("Error creating audio in", error);
-                    });
-            }
-
-            stream.connect(AUDIO.fft);
-        },
+    html5audio_stream_create__deps: ['$emscriptenGetAudioObject'],
+    html5audio_stream_create: function(audioWorkletNode, numInputChannels){
+        var audioWorkletNode = emscriptenGetAudioObject(audioWorkletNode);
+        if(numInputChannels > 0){
+            navigator.mediaDevices.getUserMedia({ audio: true })
+            .then(function (audioIn) {
+                var mediaElement = AUDIO.context.createMediaStreamSource(audioIn);
+                mediaElement.connect(audioWorkletNode);
+            })
+            .catch(function (error) {
+                console.log("Error creating audio in", error);
+            });
+        }
+        audioWorkletNode.connect(AUDIO.fft);
+    },
 
     html5audio_stream_free: function () {
 

addons/ofxEmscripten/src/ofxEmscriptenSoundStream.cpp

@@ -6,17 +6,64 @@
  */
 
 #include "ofxEmscriptenSoundStream.h"
-#include "html5audio.h"
 #include "ofBaseApp.h"
 #include "ofLog.h"
+#include "html5audio.h"
+#include "emscripten/webaudio.h"
 
 using namespace std;
 
+EM_BOOL ProcessAudio(int numInputs, const AudioSampleFrame *inputs, int numOutputs, AudioSampleFrame *outputs, int numParams, const AudioParamFrame *params, void *userData) {
+	ofxEmscriptenSoundStream * stream = (ofxEmscriptenSoundStream *)userData;
+	++stream->audioProcessedCount;
+	if (stream->settings.numInputChannels > 0 && stream->settings.inCallback) {
+		stream->settings.inCallback(stream->inbuffer);
+		for (int o = 0; o < numInputs; ++o) {
+			for (int i = 0; i < 128; ++i) {
+				for (int ch = 0; ch < inputs[o].numberOfChannels; ++ch) {
+					stream->inbuffer[i * inputs[o].numberOfChannels + ch] = inputs[o].data[ch * 128 + i];
+				}
+			}
+		}
+	}
+	if (stream->settings.numOutputChannels > 0 && stream->settings.outCallback) {
+		stream->settings.outCallback(stream->outbuffer);
+		for (int o = 0; o < numOutputs; ++o) {
+			for (int i = 0; i < 128; ++i) {
+				for (int ch = 0; ch < outputs[o].numberOfChannels; ++ch) {
+					outputs[o].data[ch * 128 + i] = stream->outbuffer[i * outputs[o].numberOfChannels + ch];
+				}
+			}
+		}
+	}
+	return EM_TRUE;
+}
+
+void AudioWorkletProcessorCreated(EMSCRIPTEN_WEBAUDIO_T audioContext, EM_BOOL success, void *userData) {
+	if (!success) return;
+	ofxEmscriptenSoundStream * stream = (ofxEmscriptenSoundStream *)userData;
+	int outputChannelCounts[1] = { static_cast<int>(stream->settings.numOutputChannels) };
+	EmscriptenAudioWorkletNodeCreateOptions options = {
+		.numberOfInputs = 1,
+		.numberOfOutputs = 1,
+		.outputChannelCounts = outputChannelCounts
+	};
+	EMSCRIPTEN_AUDIO_WORKLET_NODE_T audioWorklet = emscripten_create_wasm_audio_worklet_node(audioContext, "audio-processor", &options, &ProcessAudio, userData);
+	html5audio_stream_create(audioWorklet, stream->settings.numInputChannels);
+}
+
+void WebAudioWorkletThreadInitialized(EMSCRIPTEN_WEBAUDIO_T audioContext, EM_BOOL success, void *userData) {
+	if (!success) return;
+	WebAudioWorkletProcessorCreateOptions opts = {
+		.name = "audio-processor",
+	};
+	emscripten_create_wasm_audio_worklet_processor_async(audioContext, &opts, AudioWorkletProcessorCreated, userData);
+}
+
 int ofxEmscriptenAudioContext();
 
 ofxEmscriptenSoundStream::ofxEmscriptenSoundStream()
 :context(ofxEmscriptenAudioContext())
-,tickCount(0)
 {
 
 }
@@ -33,8 +80,9 @@ std::vector<ofSoundDevice> ofxEmscriptenSoundStream::getDeviceList(ofSoundDevice
 bool ofxEmscriptenSoundStream::setup(const ofSoundStreamSettings & settings) {
 	inbuffer.allocate(settings.bufferSize, settings.numInputChannels);
 	outbuffer.allocate(settings.bufferSize, settings.numOutputChannels);
+	audioProcessedCount = 0;
 	this->settings = settings;
-	html5audio_stream_create(settings.bufferSize,settings.numInputChannels,settings.numOutputChannels,inbuffer.getBuffer().data(),outbuffer.getBuffer().data(),&audio_cb,this);
+	emscripten_start_wasm_audio_worklet_thread_async(context, wasmAudioWorkletStack, sizeof(wasmAudioWorkletStack), WebAudioWorkletThreadInitialized, this);
 	return true;
 }
 
@@ -67,7 +115,7 @@ void ofxEmscriptenSoundStream::close() {
 }
 
 uint64_t ofxEmscriptenSoundStream::getTickCount() const{
-	return tickCount;
+	return audioProcessedCount;
 }
 
 int ofxEmscriptenSoundStream::getNumInputChannels() const{
@@ -85,14 +133,3 @@ int ofxEmscriptenSoundStream::getSampleRate() const{
 int ofxEmscriptenSoundStream::getBufferSize() const{
 	return settings.bufferSize;
 }
-
-void ofxEmscriptenSoundStream::audio_cb( int bufferSize, int inputChannels, int outputChannels, void * userData){
-	ofxEmscriptenSoundStream * stream = (ofxEmscriptenSoundStream*) userData;
-	stream->audioCB(bufferSize,inputChannels,outputChannels);
-}
-
-void ofxEmscriptenSoundStream::audioCB(int bufferSize, int inputChannels, int outputChannels){
-	if(inputChannels>0 && settings.inCallback) settings.inCallback(inbuffer);
-	if(outputChannels>0 && settings.outCallback) settings.outCallback(outbuffer);
-	tickCount++;
-}

addons/ofxEmscripten/src/ofxEmscriptenSoundStream.h

@@ -32,13 +32,12 @@ class ofxEmscriptenSoundStream: public ofBaseSoundStream {
 	int getNumOutputChannels() const;
 	int getSampleRate() const;
 	int getBufferSize() const;
-
-private:
-	static void audio_cb(int bufferSize, int inputChannels, int outputChannels, void * userData);
-	void audioCB(int bufferSize, int inputChannels, int outputChannels);
-	int context;
-	unsigned long long tickCount;
 	ofSoundStreamSettings settings;
 	ofSoundBuffer inbuffer;
 	ofSoundBuffer outbuffer;
+	int audioProcessedCount;
+
+private:
+	int context;
+	uint8_t wasmAudioWorkletStack[4096 * 4];
 };

examples/sound/audioInputExample/src/ofApp.cpp

@@ -9,7 +9,7 @@ void ofApp::setup(){
 
 	soundStream.printDeviceList();
 
-	int bufferSize = 256;
+	int bufferSize = 128;
 
 	left.assign(bufferSize, 0.0);
 	right.assign(bufferSize, 0.0);
@@ -44,10 +44,11 @@ void ofApp::setup(){
 	settings.sampleRate = 44100;
 	#ifdef TARGET_EMSCRIPTEN
 		settings.numOutputChannels = 2;
+		settings.numInputChannels = 2;
 	#else
 		settings.numOutputChannels = 0;
+		settings.numInputChannels = 1;
 	#endif
-	settings.numInputChannels = 1;
 	settings.bufferSize = bufferSize;
 	soundStream.setup(settings);
 
@@ -92,7 +93,7 @@ void ofApp::draw(){
 
 			ofBeginShape();
 			for (unsigned int i = 0; i < left.size(); i++){
-				ofVertex(i*2, 100 -left[i]*180.0f);
+				ofVertex(i*4, 100 -left[i]*180.0f);
 			}
 			ofEndShape(false);
 
@@ -115,7 +116,7 @@ void ofApp::draw(){
 
 			ofBeginShape();
 			for (unsigned int i = 0; i < right.size(); i++){
-				ofVertex(i*2, 100 -right[i]*180.0f);
+				ofVertex(i*4, 100 -right[i]*180.0f);
 			}
 			ofEndShape(false);
 

examples/sound/audioOutputExample/src/ofApp.cpp

@@ -5,7 +5,7 @@ void ofApp::setup(){
 
 	ofBackground(34, 34, 34);
 
-	int bufferSize		= 512;
+	int bufferSize		= 128;
 	sampleRate 			= 44100;
 	phase 				= 0;
 	phaseAdder 			= 0.0f;

examples/sound/soundBufferExample/src/ofApp.cpp

@@ -12,7 +12,7 @@ void ofApp::setup(){
 	ofSoundStreamSettings settings;
 	settings.numOutputChannels = 2;
 	settings.sampleRate = 44100;
-	settings.bufferSize = 512;
+	settings.bufferSize = 128;
 	settings.numBuffers = 4;
 	settings.setOutListener(this);
 	soundStream.setup(settings);
@@ -24,7 +24,7 @@ void ofApp::update(){
 	// "lastBuffer" is shared between update() and audioOut(), which are called
 	// on two different threads. This lock makes sure we don't use lastBuffer
 	// from both threads simultaneously (see the corresponding lock in audioOut())
-	unique_lock<mutex> lock(audioMutex);
+	std::unique_lock<std::mutex> lock(audioMutex);
 
 	// this loop is building up a polyline representing the audio contained in
 	// the left channel of the buffer
@@ -92,7 +92,7 @@ void ofApp::audioOut(ofSoundBuffer &outBuffer) {
 		pulsePhase += pulsePhaseStep;
 	}
 
-	unique_lock<mutex> lock(audioMutex);
+	std::unique_lock<std::mutex> lock(audioMutex);
 	lastBuffer = outBuffer;
 }
 

libs/openFrameworksCompiled/project/emscripten/config.emscripten.default.mk

@@ -64,8 +64,8 @@ PLATFORM_REQUIRED_ADDONS = ofxEmscripten
 ################################################################################
 
 # Code Generation Option Flags (http://gcc.gnu.org/onlinedocs/gcc/Code-Gen-Options.html)
-PLATFORM_CFLAGS =
-PLATFORM_CXXFLAGS = -Wall -std=c++17 -Wno-warn-absolute-paths
+PLATFORM_CFLAGS = -s USE_PTHREADS=1
+PLATFORM_CXXFLAGS = -Wall -std=c++17 -Wno-warn-absolute-paths -s USE_PTHREADS=1
 
 ################################################################################
 # PLATFORM LDFLAGS
@@ -93,7 +93,7 @@ ifdef USE_CCACHE
 	endif
 endif
 
-PLATFORM_LDFLAGS = -Wl --gc-sections --preload-file bin/data@data --emrun --bind --profiling-funcs -s USE_FREETYPE=1 -s ALLOW_MEMORY_GROWTH=1 -s MAX_WEBGL_VERSION=2 -s WEBGL2_BACKWARDS_COMPATIBILITY_EMULATION=1 -s FULL_ES2 -sFULL_ES3=1
+PLATFORM_LDFLAGS = -Wl --gc-sections --preload-file bin/data@data --emrun --bind --profiling-funcs -s USE_FREETYPE=1 -s ALLOW_MEMORY_GROWTH=1 -s MAX_WEBGL_VERSION=2 -s WEBGL2_BACKWARDS_COMPATIBILITY_EMULATION=1 -s FULL_ES2 -sFULL_ES3=1  -s USE_PTHREADS=1 -s AUDIO_WORKLET=1 -s WASM_WORKERS=1 -sENVIRONMENT="web,worker" -s WEBAUDIO_DEBUG=1
 PLATFORM_LDFLAGS += --js-library $(OF_ADDONS_PATH)/ofxEmscripten/libs/html5video/lib/emscripten/library_html5video.js
 PLATFORM_LDFLAGS += --js-library $(OF_ADDONS_PATH)/ofxEmscripten/libs/html5audio/lib/emscripten/library_html5audio.js