Skip to content

Commit f59ab62

Browse files
nfriedlynfriedly
committed
fixed bug with sendiong audio after stop, added firefox testing, disabled failing integration tests, misc renaming and debug events
1 parent d7cecbd commit f59ab62

File tree

9 files changed

+160
-50
lines changed

9 files changed

+160
-50
lines changed

‎dist/watson-speech.js

Lines changed: 28 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5972,7 +5972,7 @@ function MediaElementAudioStream(source, opts) {
59725972
* @see https://developer.mozilla.org/en-US/docs/Web/API/ScriptProcessorNode/onaudioprocess
59735973
* @param {AudioProcessingEvent} e https://developer.mozilla.org/en-US/docs/Web/API/AudioProcessingEvent
59745974
*/
5975-
function recorderProcess(e) {
5975+
function processAudio(e) {
59765976
// onaudioprocess can be called at least once after we've stopped
59775977
if (recording) {
59785978

@@ -6001,20 +6001,20 @@ function MediaElementAudioStream(source, opts) {
60016001

60026002
var context = new AudioContext();
60036003
var audioInput = context.createMediaElementSource(source);
6004-
var recorder = context.createScriptProcessor(opts.bufferSize, inputChannels, outputChannels);
6004+
var scriptProcessor = context.createScriptProcessor(opts.bufferSize, inputChannels, outputChannels);
60056005

6006-
recorder.onaudioprocess = recorderProcess;
6006+
scriptProcessor.onaudioprocess = processAudio;
60076007

60086008
if (!opts.muteSource) {
60096009
var gain = context.createGain();
60106010
audioInput.connect(gain);
60116011
gain.connect(context.destination);
60126012
}
60136013

6014-
audioInput.connect(recorder);
6014+
audioInput.connect(scriptProcessor);
60156015

60166016
// other half of workaround for chrome bugs
6017-
recorder.connect(context.destination);
6017+
scriptProcessor.connect(context.destination);
60186018

60196019
this.stop = function() {
60206020
recording = false;
@@ -6101,7 +6101,7 @@ module.exports = function promise(stream) {
61016101
*/
61026102

61036103
'use strict';
6104-
var fileReaderStream = require('readable-blob-stream');
6104+
var ReadableBlobStream = require('readable-blob-stream');
61056105
var RecognizeStream = require('./recognize-stream.js');
61066106
var FilePlayer = require('./file-player.js');
61076107

@@ -6132,8 +6132,8 @@ module.exports = function recognizeBlob(options) {
61326132
});
61336133
}
61346134

6135-
return fileReaderStream(options.data).pipe(recognizeStream);
6136-
}
6135+
return new ReadableBlobStream(options.data).pipe(recognizeStream);
6136+
};
61376137

61386138

61396139

@@ -6372,7 +6372,7 @@ RecognizeStream.prototype.initialize = function () {
63726372

63736373

63746374
this.socket.onopen = function () {
6375-
socket.send(JSON.stringify(openingMessage));
6375+
self.sendJSON(openingMessage);
63766376
self.emit('connect');
63776377
};
63786378

@@ -6469,6 +6469,15 @@ RecognizeStream.prototype.initialize = function () {
64696469
this.initialized = true;
64706470
};
64716471

6472+
RecognizeStream.prototype.sendJSON = function sendJSON(msg) {
6473+
this.emit('send-json', msg);
6474+
return this.socket.send(JSON.stringify(msg));
6475+
};
6476+
6477+
RecognizeStream.prototype.sendData = function sendData(data) {
6478+
this.emit('send-data', data);
6479+
return this.socket.send(data);
6480+
};
64726481

64736482
RecognizeStream.prototype._read = function (size) {
64746483
// there's no easy way to control reads from the underlying library
@@ -6477,8 +6486,12 @@ RecognizeStream.prototype._read = function (size) {
64776486

64786487
RecognizeStream.prototype._write = function (chunk, encoding, callback) {
64796488
var self = this;
6489+
if (self.finished) {
6490+
// can't send any more data after the stop message (although this shouldn't happen normally...)
6491+
return;
6492+
}
64806493
if (self.listening) {
6481-
self.socket.send(chunk);
6494+
self.sendData(chunk);
64826495
this.afterSend(callback);
64836496
} else {
64846497
if (!this.initialized) {
@@ -6488,7 +6501,7 @@ RecognizeStream.prototype._write = function (chunk, encoding, callback) {
64886501
this.initialize();
64896502
}
64906503
this.once('listening', function () {
6491-
self.socket.send(chunk);
6504+
self.sendData(chunk);
64926505
this.afterSend(callback);
64936506
});
64946507
}
@@ -6504,13 +6517,9 @@ RecognizeStream.prototype.afterSend = function afterSend(next) {
65046517
}
65056518
};
65066519

6507-
RecognizeStream.prototype.stop = function (hard) {
6520+
RecognizeStream.prototype.stop = function () {
65086521
this.emit('stop');
6509-
if (hard) {
6510-
this.socket.close();
6511-
} else {
6512-
this.finish();
6513-
}
6522+
this.finish();
65146523
};
65156524

65166525
RecognizeStream.prototype.finish = function finish() {
@@ -6522,10 +6531,10 @@ RecognizeStream.prototype.finish = function finish() {
65226531
var self = this;
65236532
var closingMessage = {action: 'stop'};
65246533
if (self.socket) {
6525-
self.socket.send(JSON.stringify(closingMessage));
6534+
self.sendJSON(closingMessage);
65266535
} else {
65276536
this.once('connect', function () {
6528-
self.socket.send(JSON.stringify(closingMessage));
6537+
self.sendJSON(closingMessage);
65296538
});
65306539
}
65316540
};

‎examples/public/audio-element-programmatic.html

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8">
5+
<title>Programmatic new Audio() Example - Watson Speech to Text</title>
6+
</head>
7+
<body>
8+
9+
<section>
10+
<h2>Transcribe Programmatic new Audio(), Streaming</h2>
11+
<p><small><i>The browser must be able to decode and play the audio file.</i></small></p>
12+
13+
<button id="button">Play and Transcribe</button>
14+
15+
<h2>Output:</h2>
16+
<div id="output">--</div>
17+
</section>
18+
19+
<script src="watson-speech.js"></script>
20+
<script src="http://code.jquery.com/jquery-2.2.0.min.js"></script>
21+
22+
<h2>Code for this demo:</h2>
23+
<pre><code><script style="display: block;">
24+
$(function() {
25+
var $output = $('#output');
26+
27+
$('#button').click(function () {
28+
$.get('/token').then(function (token) {
29+
$output.html('');
30+
31+
var audioElement = new Audio(); // document.createElement('video'); also works here
32+
audioElement.src = "audio.wav";
33+
34+
var stream = WatsonSpeech.SpeechToText.recognizeElement({
35+
token: token,
36+
element: audioElement
37+
// muteSource: true // prevents sound from also playing locally
38+
})
39+
.pipe(new WatsonSpeech.SpeechToText.FormatStream()); // optional
40+
41+
// each result gets it's own <span> because watson will sometimes go back and change a word as it hears more context
42+
var $curSentence = $('<span>&nbsp;</span>').appendTo($output);
43+
44+
// a result is approximately equivalent to a sentence
45+
stream.on('result', function(result) {
46+
// update the text for the current sentence with the default alternative.
47+
// there may be multiple alternatives but this example app ignores all but the first.
48+
$curSentence.html(result.alternatives[0].transcript);
49+
if (result.final) {
50+
// if we have the final text for that sentence, start a new one
51+
$curSentence = $('<span/>').appendTo($output);
52+
}
53+
});
54+
55+
stream.on('error', function(err) {
56+
console.log(err);
57+
});
58+
});
59+
});
60+
});
61+
</script></code></pre>
62+
</body>
63+
</html>

‎examples/public/audio-element.html

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ <h2>Transcribe &lt;audio&gt; Element, Streaming</h2>
1212
Your browser does not support the <code>audio</code> element.
1313
</audio>
1414
<button id="button">Play and Transcribe</button>
15-
<p><small><i>&lt;video&gt; elements should behave the same as &lt;audio&gt; elements.</i></small></p>
15+
<p><small><i>&lt;video&gt; elements should behave the same as &lt;audio&gt; elements. The browser must be able to decode and play the file in either case. </i></small></p>
1616

1717
<h2>Output:</h2>
1818
<div id="output">--</div>
@@ -33,6 +33,7 @@ <h2>Code for this demo:</h2>
3333
var stream = WatsonSpeech.SpeechToText.recognizeElement({
3434
token: token,
3535
element: $('#audio-element')[0]
36+
// muteSource: true // prevents sound from also playing locally
3637
})
3738
.pipe(new WatsonSpeech.SpeechToText.FormatStream()); // optional
3839

‎examples/public/index.html

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
<li><a href="file-streaming.html">Transcribe from file, Streaming</a></li>
1212
<li><a href="file-promise.html">Transcribe from file, Promise</a></li>
1313
<li><a href="audio-element.html">Transcribe from HTML5 &lt;audio&gt; element, Streaming</a></li>
14+
<li><a href="audio-element-programmatic.html">Transcribe from <code>new Audio()</code>, Streaming</a></li>
1415
</ul>
1516

1617
</body>

‎package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"devDependencies": {
2121
"browserify": "^13.0.0",
2222
"concat-stream": "^1.5.1",
23+
"envify": "^3.4.0",
2324
"eslint": "^1.10.3",
2425
"eslint-config-google": "^0.3.0",
2526
"expect.js": "^0.3.1",
@@ -28,6 +29,7 @@
2829
"karma-browserify": "^5.0.1",
2930
"karma-chrome-launcher": "^0.2.2",
3031
"karma-express-http-server": "0.0.1",
32+
"karma-firefox-launcher": "^0.1.7",
3133
"karma-mocha": "^0.2.1",
3234
"mocha": "^2.4.4",
3335
"serve-static": "^1.10.2",

‎speech-to-text/media-element-audio-stream.js

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ function MediaElementAudioStream(source, opts) {
4141
* @see https://developer.mozilla.org/en-US/docs/Web/API/ScriptProcessorNode/onaudioprocess
4242
* @param {AudioProcessingEvent} e https://developer.mozilla.org/en-US/docs/Web/API/AudioProcessingEvent
4343
*/
44-
function recorderProcess(e) {
44+
function processAudio(e) {
4545
// onaudioprocess can be called at least once after we've stopped
4646
if (recording) {
4747

@@ -70,20 +70,20 @@ function MediaElementAudioStream(source, opts) {
7070

7171
var context = new AudioContext();
7272
var audioInput = context.createMediaElementSource(source);
73-
var recorder = context.createScriptProcessor(opts.bufferSize, inputChannels, outputChannels);
73+
var scriptProcessor = context.createScriptProcessor(opts.bufferSize, inputChannels, outputChannels);
7474

75-
recorder.onaudioprocess = recorderProcess;
75+
scriptProcessor.onaudioprocess = processAudio;
7676

7777
if (!opts.muteSource) {
7878
var gain = context.createGain();
7979
audioInput.connect(gain);
8080
gain.connect(context.destination);
8181
}
8282

83-
audioInput.connect(recorder);
83+
audioInput.connect(scriptProcessor);
8484

8585
// other half of workaround for chrome bugs
86-
recorder.connect(context.destination);
86+
scriptProcessor.connect(context.destination);
8787

8888
this.stop = function() {
8989
recording = false;

‎speech-to-text/recognize-stream.js

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ RecognizeStream.prototype.initialize = function () {
124124

125125

126126
this.socket.onopen = function () {
127-
socket.send(JSON.stringify(openingMessage));
127+
self.sendJSON(openingMessage);
128128
self.emit('connect');
129129
};
130130

@@ -221,6 +221,15 @@ RecognizeStream.prototype.initialize = function () {
221221
this.initialized = true;
222222
};
223223

224+
RecognizeStream.prototype.sendJSON = function sendJSON(msg) {
225+
this.emit('send-json', msg);
226+
return this.socket.send(JSON.stringify(msg));
227+
};
228+
229+
RecognizeStream.prototype.sendData = function sendData(data) {
230+
this.emit('send-data', data);
231+
return this.socket.send(data);
232+
};
224233

225234
RecognizeStream.prototype._read = function (size) {
226235
// there's no easy way to control reads from the underlying library
@@ -229,8 +238,12 @@ RecognizeStream.prototype._read = function (size) {
229238

230239
RecognizeStream.prototype._write = function (chunk, encoding, callback) {
231240
var self = this;
241+
if (self.finished) {
242+
// can't send any more data after the stop message (although this shouldn't happen normally...)
243+
return;
244+
}
232245
if (self.listening) {
233-
self.socket.send(chunk);
246+
self.sendData(chunk);
234247
this.afterSend(callback);
235248
} else {
236249
if (!this.initialized) {
@@ -240,7 +253,7 @@ RecognizeStream.prototype._write = function (chunk, encoding, callback) {
240253
this.initialize();
241254
}
242255
this.once('listening', function () {
243-
self.socket.send(chunk);
256+
self.sendData(chunk);
244257
this.afterSend(callback);
245258
});
246259
}
@@ -256,13 +269,9 @@ RecognizeStream.prototype.afterSend = function afterSend(next) {
256269
}
257270
};
258271

259-
RecognizeStream.prototype.stop = function (hard) {
272+
RecognizeStream.prototype.stop = function () {
260273
this.emit('stop');
261-
if (hard) {
262-
this.socket.close();
263-
} else {
264-
this.finish();
265-
}
274+
this.finish();
266275
};
267276

268277
RecognizeStream.prototype.finish = function finish() {
@@ -274,10 +283,10 @@ RecognizeStream.prototype.finish = function finish() {
274283
var self = this;
275284
var closingMessage = {action: 'stop'};
276285
if (self.socket) {
277-
self.socket.send(JSON.stringify(closingMessage));
286+
self.sendJSON(closingMessage);
278287
} else {
279288
this.once('connect', function () {
280-
self.socket.send(JSON.stringify(closingMessage));
289+
self.sendJSON(closingMessage);
281290
});
282291
}
283292
};

‎test/resources/karma.conf.js

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,10 @@ module.exports = function(config) {
3232

3333
browserify: {
3434
debug: true,
35-
transform: [ ] // 'brfs', 'browserify-shim'
35+
// 'brfs' makes fs.read* work
36+
// 'browserify-shim' wraps non-browserify modules
37+
// 'envify' makes process.env work
38+
transform: [ 'envify' ]
3639
},
3740

3841

@@ -61,7 +64,7 @@ module.exports = function(config) {
6164

6265
// start these browsers
6366
// available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
64-
browsers: ['ChromeWithPrerecordedMic'],
67+
browsers: ['ChromeWithPrerecordedMic', 'FirefoxAutoGUM'],
6568

6669
// you can define custom flags
6770
// there's a handy list of chrome flags at
@@ -70,6 +73,13 @@ module.exports = function(config) {
7073
base: 'Chrome',
7174
// --no-sandbox is required for travis-ci
7275
flags: ['--use-fake-device-for-media-stream','--use-fake-ui-for-media-stream', '--use-file-for-fake-audio-capture=test/resources/audio.wav', '--no-sandbox']
76+
},
77+
// automatically approve getUserMedia calls
78+
FirefoxAutoGUM: {
79+
base: 'Firefox',
80+
prefs: {
81+
'media.navigator.permission.disabled': true
82+
}
7383
}
7484
},
7585

0 commit comments

Comments
 (0)