Skip to content

Commit 7964be1

Browse files
fquirinfquirin
committed
experimenting with audio-recorder to fix iOS 12.4 build
1 parent b4f5332 commit 7964be1

File tree

4 files changed

+161
-77
lines changed

4 files changed

+161
-77
lines changed

‎www/scripts/recorder.js‎

Lines changed: 56 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -21,30 +21,24 @@ DEALINGS IN THE SOFTWARE.
2121

2222
(function (window) {
2323

24-
var Recorder = function (source) {
24+
var Recorder = function(source, audioProcessor, startFun, stopFun) {
2525
var bufferLen = 4096;
2626
var websocket;
27-
this.context = source.context;
28-
if (!this.context.createScriptProcessor) {
29-
this.node = this.context.createJavaScriptNode(bufferLen, 2, 2);
30-
} else {
31-
this.node = this.context.createScriptProcessor(bufferLen, 2, 2);
32-
}
27+
var audioContext = source.context;
3328

3429
var recording = false;
35-
36-
this.node.onaudioprocess = function (e) {
30+
31+
function processAudio(inputAudioFrame){
3732
if (!recording) return;
3833

39-
var inputL = e.inputBuffer.getChannelData(0);
40-
var length = Math.floor(inputL.length / 3);
34+
var length = Math.floor(inputAudioFrame.length / 3);
4135
var result = new Float32Array(length);
4236

43-
var index = 0,
44-
inputIndex = 0;
37+
var index = 0;
38+
var inputIndex = 0;
4539

4640
while (index < length) {
47-
result[index++] = inputL[inputIndex];
41+
result[index++] = inputAudioFrame[inputIndex];
4842
inputIndex += 3;
4943
}
5044

@@ -56,32 +50,66 @@ DEALINGS IN THE SOFTWARE.
5650
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
5751
}
5852

59-
//console.log('Recorder onaudioprocess - view: ' + view); //DEBUG
60-
websocket.send(view);
53+
//console.log('Recorder onaudioprocess - view: ' + view); //DEBUG
54+
if (websocket){
55+
websocket.send(view);
56+
}
6157
}
6258

63-
//Move this to 'start'?
64-
source.connect(this.node);
59+
//Custom
60+
if (audioProcessor && startFun && stopFun){
61+
audioProcessor.onaudioprocess = function(inputAudioFrame){
62+
processAudio(inputAudioFrame);
63+
}
64+
//Web-Audio
65+
}else if (audioContext){
66+
let processNode;
67+
if ('createScriptProcessor' in audioContext){
68+
processNode = audioContext.createScriptProcessor(bufferLen, 1, 1);
69+
}else if ('createJavaScriptNode' in audioContext){
70+
processNode = audioContext.createJavaScriptNode(bufferLen, 1, 1);
71+
}else{
72+
console.error("Recorder - cannot create an audio processor!");
73+
return;
74+
}
75+
processNode.onaudioprocess = function(e) {
76+
var inputAudioFrame = e.inputBuffer.getChannelData(0);
77+
processAudio(inputAudioFrame);
78+
}
79+
startFun = function(){
80+
source.connect(processNode);
81+
processNode.connect(audioContext.destination); //if the script node is not connected to an output the "onaudioprocess" event is not triggered in chrome.
82+
}
83+
stopFun = function(){
84+
source.disconnect(processNode);
85+
processNode.disconnect(audioContext.destination);
86+
//processNode.disconnect();
87+
}
88+
//Error
89+
}else{
90+
console.error('Recorder - no valid audio processor found!');
91+
return;
92+
}
6593

6694
//Will be called at beginning of SepiaFW.audioRecorder.start();
67-
this.start = function() {
68-
this.node.connect(this.context.destination); // if the script node is not connected to an output the "onaudioprocess" event is not triggered in chrome.
95+
this.start = function(){
96+
recording = true;
97+
startFun();
6998
}
7099

71100
//Will be called at beginning of SepiaFW.audioRecorder.stop();
72-
this.stop = function () {
101+
this.stop = function(){
73102
recording = false;
74-
this.node.disconnect(0);
103+
stopFun();
75104
}
76105

77-
this.record = function (ws) {
106+
this.connect = function(ws){
78107
websocket = ws;
79-
recording = true;
80108
//console.log('Recorder record - ws: ' + ws); //DEBUG
81-
//console.log(this.node); //DEBUG
109+
//console.log(processNode); //DEBUG
82110
}
83111

84-
this.sendHeader = function (ws) {
112+
this.sendHeader = function(ws){
85113
var sampleLength = 1000000;
86114
var mono = true;
87115
var sampleRate = 16000; //TODO: this might not be true! We cannot guarantee that!
@@ -118,8 +146,8 @@ DEALINGS IN THE SOFTWARE.
118146
//console.log('Recorder sendHeader - view: ' + view); //DEBUG
119147
ws.send(view);
120148
}
121-
function writeString(view, offset, string) {
122-
for (var i = 0; i < string.length; i++) {
149+
function writeString(view, offset, string){
150+
for (var i = 0; i < string.length; i++){
123151
view.setUint8(offset + i, string.charCodeAt(i));
124152
}
125153
}

‎www/scripts/sepiaFW.audioRecorder.js‎

Lines changed: 71 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,12 @@ function sepiaFW_build_audio_recorder(){
4848
AudioRecorder.isStreamRecorderSupported = testStreamRecorderSupport(); //set once on start
4949

5050
//set default parameters for audio recorder
51-
AudioRecorder.setup = function (successCallback, errorCallback){
51+
AudioRecorder.setup = function(successCallback, errorCallback){
5252
//... ?
5353
}
5454

5555
//STOP recorder
56-
AudioRecorder.stop = function (closeAfterStop, successCallback, errorCallback){
56+
AudioRecorder.stop = function(closeAfterStop, successCallback, errorCallback){
5757
if (!AudioRecorder.isRecording){
5858
//clean up?
5959
//...
@@ -90,19 +90,23 @@ function sepiaFW_build_audio_recorder(){
9090
}
9191
}
9292
//We release the audioContext here to be sure
93-
setTimeout(function(){
94-
recorderAudioContext.suspend().then(function() { //Note: a promise that can fail ...
95-
//console.log('SUSPENDED audio-context');
96-
if (closeAfterStop){
97-
closeAudioContext(recorderAudioContext, successCallback, errorCallback);
98-
}else{
99-
if (successCallback) successCallback();
100-
}
101-
}).catch(function(e){
102-
broadcastRecorderError(e);
103-
if (errorCallback) errorCallback(e);
104-
});
105-
},100);
93+
if (recorderAudioContext){
94+
setTimeout(function(){
95+
recorderAudioContext.suspend().then(function() { //Note: a promise that can fail ...
96+
//console.log('SUSPENDED audio-context');
97+
if (closeAfterStop){
98+
closeAudioContext(recorderAudioContext, successCallback, errorCallback);
99+
}else{
100+
if (successCallback) successCallback();
101+
}
102+
}).catch(function(e){
103+
broadcastRecorderError(e);
104+
if (errorCallback) errorCallback(e);
105+
});
106+
},100);
107+
}else{
108+
if (successCallback) successCallback();
109+
}
106110
}, 100);
107111

108112
AudioRecorder.isRecording = false; //TODO: this probably has to wait for callbacks to be safe
@@ -124,7 +128,7 @@ function sepiaFW_build_audio_recorder(){
124128
}
125129

126130
//START recorder
127-
AudioRecorder.start = function (successCallback, errorCallback){
131+
AudioRecorder.start = function(successCallback, errorCallback){
128132
if (AudioRecorder.isRecording){
129133
//clean up?
130134
//...
@@ -166,18 +170,6 @@ function sepiaFW_build_audio_recorder(){
166170

167171
// ---------------- Audio Recorder (Recorder.js) ----------------------
168172

169-
function createOrUpdateAudioContext(){
170-
if (recorderAudioContext){
171-
//TODO: clean up old context and sources?
172-
if (recorderAudioContext.state == "closed"){
173-
recorderAudioContext = new AudioContext();
174-
}
175-
}else{
176-
recorderAudioContext = new AudioContext();
177-
}
178-
return recorderAudioContext;
179-
}
180-
181173
AudioRecorder.getRecorder = function(RecorderInstance, callback, errorCallback){
182174
//Create a new audio recorder.
183175
//NOTE: audioRec is a global variable (inside this scope) because we can't have 2 anyway (I guess..)
@@ -237,13 +229,24 @@ function sepiaFW_build_audio_recorder(){
237229
}
238230

239231
// ---------------- MediaDevices interface stuff ----------------------
232+
233+
function createOrUpdateAudioContext(){
234+
if (recorderAudioContext){
235+
//TODO: clean up old context and sources?
236+
if (recorderAudioContext.state == "closed"){
237+
recorderAudioContext = new AudioContext();
238+
}
239+
}else{
240+
recorderAudioContext = new AudioContext();
241+
}
242+
return recorderAudioContext;
243+
}
240244

241245
function getStreamRecorder(RecorderInstance, stream, callback){
242246
if (!RecorderInstance) RecorderInstance = RecorderJS;
243247
recorderAudioContext = createOrUpdateAudioContext();
244248
recorderAudioSource = stream;
245-
var inputPoint = recorderAudioContext.createGain();
246-
recorderAudioContext.createMediaStreamSource(recorderAudioSource).connect(inputPoint);
249+
var inputPoint = recorderAudioContext.createMediaStreamSource(recorderAudioSource);
247250
audioRec = new RecorderInstance(inputPoint);
248251
if (callback) callback(audioRec);
249252
}
@@ -253,18 +256,31 @@ function sepiaFW_build_audio_recorder(){
253256
var audioInputPluginIsSet = false;
254257
var audioInputPluginHasPermission = false;
255258
var audioInputPluginErrorCallback = undefined; //reset on every call - TODO: lost when 2 errors are queued ...
259+
260+
//AudioProcessor (replacement for scriptProcessor of AudioContext)
261+
function AudioInputPluginProcessor(){
262+
this.onaudioprocess = function(inputAudioFrame){
263+
//to be defined by RecorderInstance
264+
};
265+
this.onaudioreceive = function(evt){
266+
this.onaudioprocess(evt.data);
267+
};
268+
}
269+
256270
//Init
257271
function initAudioinputPlugin(){
258272
if (isCordovaAudioinputSupported){
259273
window.addEventListener('audioinputerror', onAudioInputError, false);
260274
audioInputPluginIsSet = true;
261275
}
262276
}
277+
263278
//Errors
264279
function onAudioInputError(error){
265280
SepiaFW.debug.err("AudioRecorder error (audioinput plugin): " + JSON.stringify(error));
266281
if (audioInputPluginErrorCallback) audioInputPluginErrorCallback(error);
267282
}
283+
268284
//Check permission
269285
function checkAudioinputPermission(successCallback, errorCallback){
270286
//First check whether we already have permission to access the microphone.
@@ -310,11 +326,13 @@ function sepiaFW_build_audio_recorder(){
310326
try {
311327
if (!window.audioinput.isCapturing()){
312328
if (!RecorderInstance) RecorderInstance = RecorderJS;
329+
//--- build audioinput replacement for audio context ---
330+
/* ------ OLD ------
313331
//Reset context?
314332
recorderAudioContext = createOrUpdateAudioContext();
315333
//Start with default values and let the plugin handle conversion from raw data to web audio
316334
if (recorderAudioContext){
317-
window.audioinput.start({
335+
window.audioinput.start({
318336
streamToWebAudio: true,
319337
audioContext: recorderAudioContext
320338
});
@@ -325,10 +343,29 @@ function sepiaFW_build_audio_recorder(){
325343
recorderAudioContext = window.audioinput.getAudioContext();
326344
}
327345
//Get input for the recorder
328-
var inputPoint = window.audioinput.getAudioContext().createGain();
346+
var inputPoint = recorderAudioContext.createGain();
329347
window.audioinput.connect(inputPoint);
330-
audioRec = new RecorderInstance(inputPoint);
331-
//Done! User audioRecorder to continue
348+
*/
349+
var sourceConfig = {
350+
sampleRate: 16000,
351+
bufferSize: 4096,
352+
channels: 1,
353+
format: audioinput.FORMAT.PCM_16BIT,
354+
audioSourceType: audioinput.AUDIOSOURCE_TYPE.UNPROCESSED, //VOICE_COMMUNICATION
355+
normalize: false,
356+
streamToWebAudio: false
357+
};
358+
var audioProc = new AudioInputPluginProcessor();
359+
audioRec = new RecorderInstance(sourceConfig, audioProc, function(){
360+
//start fun (listen to audioinput events)
361+
window.addEventListener("audioinput", audioProc.onaudioreceive, false);
362+
window.audioinput.start(sourceConfig);
363+
}, function(){
364+
//stop fun
365+
window.audioinput.stop();
366+
window.removeEventListener("audioinput", audioProc.onaudioreceive);
367+
});
368+
//--- Done! Use audioRecorder to continue
332369
if (successCallback) successCallback(audioRec);
333370
}else{
334371
SepiaFW.debug.err("AudioRecorder error (audioinput plugin): Tried to capture audio but was already running!");

‎www/scripts/sepiaFW.speechWebSocket.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ function sepiaFW_build_speechWebSocket(){
196196
if (isWaitingToRecord){
197197
isRecording = true; SepiaFW.speech.Interface.isRecognizing(true);
198198
isWaitingToRecord = false;
199-
audioRecorder.record(websocket); //NOTE: websocket is specific for Recorder.js instance
199+
audioRecorder.connect(websocket); //NOTE: websocket is specific for Recorder.js instance
200200
abortRecognition = false;
201201
broadcastAsrMicOpen();
202202
}else{

0 commit comments

Comments
 (0)