objectmode works for file playback and transcription

nfriedly · nfriedly · commit e6c4be12a5cb · 2016-02-15T15:21:59.000-05:00
diff --git a/README.md b/README.md
@@ -135,3 +135,11 @@ Inherits `.stop()` method and `result` event from the `RecognizeStream`.
 * more tests in general
 * update node-sdk to use current version of this lib's RecognizeStream (and also provide the FormatStream + anything else that might be handy)
 * improve docs
+* consider a wrapper to match https://dvcs.w3.org/hg/speech-api/raw-file/tip/speechapi.html
+
+
+recognizeBlob -> recognizeFile (?)
+objectMode
+move timing/format stream.stop to wrapper methods
+test promise with objectmode
+consider interim event for recognize/format/timing streams
diff --git a/dist/watson-speech.js b/dist/watson-speech.js
diff --git a/examples/public/audio-element-programmatic.html b/examples/public/audio-element-programmatic.html
@@ -33,15 +33,16 @@ <h2>Code for this demo:</h2>
 
             var stream = WatsonSpeech.SpeechToText.recognizeElement({
                 token: token,
-                element: audioElement
+                element: audioElement,
+                objectMode: true // necessary to receive interim results
                 // muteSource: true // prevents sound from also playing locally
             });
 
             // each result gets it's own <span> because watson will sometimes go back and change a word as it hears more context
             var $curSentence = $('<span>&nbsp;</span>').appendTo($output);
 
             // a result is approximately equivalent to a sentence
-            stream.on('result', function(result) {
+            stream.on('data', function(result) {
                 // update the text for the current sentence with the default alternative.
                 // there may be multiple alternatives but this example app ignores all but the first.
                 $curSentence.html(result.alternatives[0].transcript);
diff --git a/examples/public/audio-element.html b/examples/public/audio-element.html
@@ -33,7 +33,7 @@ <h2>Code for this demo:</h2>
             var stream = WatsonSpeech.SpeechToText.recognizeElement({
                 token: token,
                 element: $('#audio-element')[0],
-                objectMode: true, // necessary to receive interim results
+                objectMode: true // necessary to receive interim results
                 // muteSource: true // prevents sound from also playing locally
             });
 
diff --git a/examples/public/blob-realtime-vs-no-realtime.html b/examples/public/blob-realtime-vs-no-realtime.html
@@ -31,7 +31,7 @@ <h2>Code for this demo:</h2>
     var $curSentence = $('<span>&nbsp;</span>').appendTo($output);
 
     // a result is approximately equivalent to a sentence
-    stream.on('result', function(result) {
+    stream.on('data', function(result) {
         // update the text for the current sentence with the default alternative.
         // there may be multiple alternatives but this example app ignores all but the first.
         $curSentence.html(result.alternatives[0].transcript);
@@ -62,14 +62,15 @@ <h2>Code for this demo:</h2>
                 token: token,
                 data: $('#audiofile')[0].files[0],
                 play: true,
+                objectMode: true,
                 max_alternatives: 1, // default is 3, but only the first one includes word timing data
                 realtime: false // defaults to true, but we're going to turn it off and then manually do it in a moment to show the difference
             });
 
             renderStream(stream, $('#output'));
 
             // now do what the realtime option would have done: pipe through a TimingStream
-            var realtimeStream = stream.pipe(new WatsonSpeech.SpeechToText.TimingStream());
+            var realtimeStream = stream.pipe(new WatsonSpeech.SpeechToText.TimingStream({objectMode: true}));
 
             renderStream(realtimeStream, $('#realtime-output'));
 
diff --git a/examples/public/file-promise.html b/examples/public/file-promise.html
@@ -29,12 +29,7 @@ <h2>Code for this demo:</h2>
 
             WatsonSpeech.SpeechToText.recognizeBlob({
                 token: token,
-                data: $('#audiofile')[0].files[0],
-                // optional: disable unused features to save time/energy/bandwidth
-                interim_results: false,
-                word_confidence: false,
-                timestamps: false,
-                max_alternatives: 1
+                data: $('#audiofile')[0].files[0]
             })
             .promise()
             .then(function(text) {
diff --git a/examples/public/file-streaming.html b/examples/public/file-streaming.html
@@ -32,14 +32,15 @@ <h2>Code for this demo:</h2>
             stream = WatsonSpeech.SpeechToText.recognizeBlob({
                 token: token,
                 data: $('#audiofile')[0].files[0],
+                objectMode: true, // necessary to receive interim results
                 play: true // play the audio out loud
             });
 
             // each result (sentence) gets it's own <span> because watson will sometimes go back and change a word as it hears more context
             var $curSentence = $('<span>&nbsp;</span>').appendTo($output);
 
             // a result is approximately equivalent to a sentence
-            stream.on('result', function(result) {
+            stream.on('data', function(result) {
                 // update the text for the current sentence with the default alternative.
                 // there may be multiple alternatives but this example app ignores all but the first.
                 $curSentence.html(result.alternatives[0].transcript);
diff --git a/examples/public/microphone-streaming-auto-stop.html b/examples/public/microphone-streaming-auto-stop.html
@@ -29,14 +29,15 @@ <h2>Code for this demo:</h2>
         $.get('/token').then(function (token) {
             var stream = WatsonSpeech.SpeechToText.recognizeMicrophone({
                 token: token,
-                continuous: false // false = automatically stop transcription the first time a pause is detected
+                continuous: false, // false = automatically stop transcription the first time a pause is detected
+                objectMode: true // necessary to receive interim results
             });
 
             // each result (sentence) gets it's own <span> because Watson will sometimes go back and change a word as it hears more context
             var $curSentence = $('<span>&nbsp;</span>').appendTo($output);
 
             // a result is approximately equivalent to a sentence
-            stream.on('result', function(result) {
+            stream.on('data', function(result) {
                 // update the text for the current sentence with the default alternative.
                 // there may be multiple alternatives but this example app ignores all but the first.
                 $curSentence.html(result.alternatives[0].transcript);
diff --git a/examples/public/microphone-streaming.html b/examples/public/microphone-streaming.html
@@ -29,14 +29,15 @@ <h2>Code for this demo:</h2>
 
         $.get('/token').then(function (token) {
             var stream = WatsonSpeech.SpeechToText.recognizeMicrophone({
-                token: token
+                token: token,
+                objectMode: true // necessary to receive interim results
             });
 
             // each result (sentence) gets it's own <span> because Watson will sometimes go back and change a word as it hears more context
             var $curSentence = $('<span>&nbsp;</span>').appendTo($output);
 
             // a result is approximately equivalent to a sentence
-            stream.on('result', function(result) {
+            stream.on('data', function(result) {
                 // update the text for the current sentence with the default alternative.
                 // there may be multiple alternatives but this example app ignores all but the first.
                 $curSentence.html(result.alternatives[0].transcript);
diff --git a/examples/token-server.js b/examples/token-server.js
@@ -35,7 +35,7 @@ var config = extend({
 }, vcapServices.getCredentials('speech_to_text'));
 
 // quick hack to make development easier
-try { extend(config, require('../test/resources/auth.json')) } catch (ex) {}
+try { extend(config, require('../test/resources/stt-auth.json')) } catch (ex) {}
 
 var authService = watson.authorization(config);
 
diff --git a/jsdoc/config.json b/jsdoc/config.json
@@ -2,7 +2,7 @@
   "source": {
     "include": [ "./", "speech-to-text/" ]
   },
-  "opts": {
+  "options": {
     "package": "package.json",
     "readme": "README.md",
     "recurse": false,
diff --git a/speech-to-text/format-stream.js b/speech-to-text/format-stream.js
@@ -18,15 +18,14 @@ var clone = require('clone');
  * @constructor
  */
 function FormatStream(opts) {
-  this.opts = util._extend({
+  this.options = util._extend({
     model: '', // some models should have all spaces removed
     hesitation: '\u2026', // ellipsis
-    decodeStrings: true,
-    objectMode: true
+    decodeStrings: true
   }, opts);
   Transform.call(this, opts);
 
-  this.isJaCn = ((this.opts.model.substring(0,5) === 'ja-JP') || (this.opts.model.substring(0,5) === 'zh-CN'));
+  this.isJaCn = ((this.options.model.substring(0,5) === 'ja-JP') || (this.options.model.substring(0,5) === 'zh-CN'));
   this._transform = opts.objectMode ? this.formatResult : this.formatString;
 }
 util.inherits(FormatStream, Transform);
@@ -43,7 +42,7 @@ var reDUnderscoreWords = /D_[^\s]+/g; // replace D_(anything)
  */
 FormatStream.prototype.clean = function clean(text) {
   // clean out "junk"
-  text = text.trim().replace(reHesitation, this.opts.hesitation)
+  text = text.trim().replace(reHesitation, this.options.hesitation)
     .replace(reRepeatedCharacter, '')
     .replace(reDUnderscoreWords,'');
 
@@ -89,7 +88,7 @@ FormatStream.prototype.formatString = function(chunk, encoding, next) {
  *
  * @param result
  */
-FormatStream.prototype.formatResult = function handleResult(result) {
+FormatStream.prototype.formatResult = function formatResult(result, encoding, next) {
   result = clone(result);
   result.alternatives = result.alternatives.map(function(alt) {
     alt.transcript = this.capitalize(this.clean(alt.transcript));
@@ -112,7 +111,8 @@ FormatStream.prototype.formatResult = function handleResult(result) {
     }
     return alt;
   }, this);
-  this.emit('result', result);
+  this.push(result);
+  next();
 };
 
 FormatStream.prototype.promise = require('./promise');
diff --git a/speech-to-text/promise.js b/speech-to-text/promise.js
@@ -14,7 +14,7 @@ module.exports = function promise(stream) {
     .on('data', function (result) {
       results.push(result);
     }).on('end', function () {
-      resolve(results.join(''));
+      resolve(typeof results[0] === 'string' ? results.join('') : results);
     }).on('error', reject);
   });
 };
diff --git a/speech-to-text/recognize-blob.js b/speech-to-text/recognize-blob.js
@@ -20,6 +20,8 @@ var RecognizeStream = require('./recognize-stream.js');
 var FilePlayer = require('./file-player.js');
 var FormatStream = require('./format-stream.js');
 var TimingStream = require('./timing-stream.js');
+var util = require('util');
+
 
 /**
  * Create and return a RecognizeStream from a File or Blob
@@ -39,15 +41,22 @@ module.exports = function recognizeBlob(options) {
     throw new Error("WatsonSpeechToText: missing required parameter: opts.token");
   }
 
-  var recognizeStream = new RecognizeStream(options);
+  var realtime = options.realtime || typeof options.realtime === 'undefined' && options.play;
+
+  var rsOpts = util._extend({
+    // if we're piping through the timing stream, we need objectMode input to get the timing data
+    readableObjectMode: options.objectMode || realtime
+  }, options);
+  delete rsOpts.objectMode;
+
+  var recognizeStream = new RecognizeStream(rsOpts);
   var stream = new BlobStream(options.data).pipe(recognizeStream);
 
   if (options.format !== false) {
     stream = stream.pipe(new FormatStream(options));
   }
-  if (options.realtime || typeof options.realtime === 'undefined' && options.play) {
+  if (realtime) {
     stream = stream.pipe(new TimingStream(options));
-    start = Date.now();
   }
 
   if (options.play) {
diff --git a/speech-to-text/recognize-element.js b/speech-to-text/recognize-element.js
@@ -38,10 +38,15 @@ module.exports = function recognizeElement(options) {
   options['content-type'] = 'audio/l16;rate=16000'; // raw wav audio (no header)
   var recognizeStream = new RecognizeStream(options);
 
-  var sourceStream = new MediaElementAudioStream(options.element , options);
+  var sourceStream = new MediaElementAudioStream(options.element , {
+    objectMode: true,
+    bufferSize: options.bufferSize,
+    muteSource: options.muteSource,
+    autoplay: options.autoPlay
+  });
 
   var stream = sourceStream
-    .pipe(new L16())
+    .pipe(new L16({writableObjectMode: true}))
     .pipe(recognizeStream);
 
   if (options.format !== false) {
diff --git a/speech-to-text/recognize-microphone.js b/speech-to-text/recognize-microphone.js
@@ -47,7 +47,7 @@ module.exports = function recognizeMicrophone(options) {
       bufferSize: options.bufferSize
     });
     micStream
-      .pipe(new L16())
+      .pipe(new L16({writableObjectMode: true, downsample: false}))
       .pipe(recognizeStream);
 
     recognizeStream.on('stop', micStream.stop.bind(micStream));
diff --git a/speech-to-text/recognize-stream.js b/speech-to-text/recognize-stream.js
@@ -47,14 +47,14 @@ var QUERY_PARAMS_ALLOWED = ['model', 'watson-token']; //, 'X-Watson-Learning-Opt
  * @param {Boolean} [options.timestamps=false] - include timestamps with results. Defaults to true when in objectMode.
  * @param {Number} [options.max_alternatives=1] - maximum number of alternative transcriptions to include. Defaults to 3 when in objectMode.
  * @param {Number} [options.inactivity_timeout=30] - how many seconds of silence before automatically closing the stream (even if continuous is true). use -1 for infinity
- * @param {Boolean} [options.objectMode=false] - emit `result` objects instead of string Buffers for the `data` events. Changes several other defaults.
+ * @param {Boolean} [options.objectMode=false] - emit `result` objects instead of string Buffers for the `data` events. Changes several other defaults. (Effectively readableObjectMode
  *
  * //todo: investigate other options at http://www.ibm.com/smarterplanet/us/en/ibmwatson/developercloud/apis/#!/speech-to-text/recognizeSessionless
  *
  * @constructor
  */
 function RecognizeStream(options) {
-  Duplex.call(this, {readableObjectMode: options && (options.objectMode || options.readableObjectMode)});
+  Duplex.call(this, options);
   this.options = options;
   this.listening = false;
   this.initialized = false;
@@ -69,6 +69,11 @@ function RecognizeStream(options) {
         self.on('data', function () {
         }); // todo: is there a better way to put a stream in flowing mode?
       });
+      if (!options.silent) {
+        console.log(new Error('Watson Speech to Text RecognizeStream: the ' + event + ' event is deprecated and will be removed from a future release. ' +
+          'Please set {objectMode: true} and listen for the data event instead. ' +
+          'Pass {silent: true} to disable this message.'));
+      }
     }
   }
   this.on('newListener', flowForResults);
@@ -97,7 +102,7 @@ RecognizeStream.prototype.initialize = function () {
 
   var url = (options.url || "wss://stream.watsonplatform.net/speech-to-text/api").replace(/^http/, 'ws') + '/v1/recognize?' + queryString;
 
-  // turn off all the extras if we're just outputting a string with a single final result
+  // turn off all the extras if we're just outputting text
   var defaults = {
     interim_results: false,
     word_confidence: false,
@@ -120,7 +125,7 @@ RecognizeStream.prototype.initialize = function () {
     max_alternatives: 3,
     inactivity_timeout: 30
   },
-    options.objectMode ? objectModeDefaults : defaults,
+    (options.objectMode || options.readableObjectMode) ? objectModeDefaults : defaults,
     pick(options, OPENING_MESSAGE_PARAMS_ALLOWED)
   );
 
@@ -199,35 +204,37 @@ RecognizeStream.prototype.initialize = function () {
       }
     } else if (data.results) {
       /**
-       * Object with interim or final results, including possible alternatives. May have no results at all for empty audio files.
+       * Object with array of interim or final results, possibly including confidence scores, alternatives, and word timing. May have no results at all for empty audio files.
        * @event RecognizeStream#results
        * @param {Object} results
-       * @deprecated - use objectMode instead
+       * @deprecated - use objectMode and listen for the 'data' event instead
        */
       self.emit('results', data.results);
 
       // note: currently there is always either 0 or 1 entries in the results array. However, this may change in the future.
       data.results.forEach(function (result) {
+        result.index = data.result_index;
         /**
-         * Object with interim or final results, including possible alternatives. May have no results at all for empty audio files.
+         * Object with interim or final results, possibly including confidence scores, alternatives, and word timing.
          * @event RecognizeStream#results
          * @param {Object} results
-         * @deprecated - use objectMode instead
+         * @deprecated - use objectMode and listen for the 'data' event instead
          */
-        result.index = data.result_index;
         self.emit('result', result);
-        if (result.final && result.alternatives) {
+        if (options.objectMode || options.readableObjectMode) {
+          /**
+           * Object with interim or final results, possibly including confidence scores, alternatives, and word timing.
+           * @event RecognizeStream#data
+           * @param {Object} data
+           */
+          self.push(result);
+        } else if (result.final && result.alternatives) {
           /**
            * Finalized text
            * @event RecognizeStream#data
            * @param {String} transcript
            */
-          if (options.objectMode) {
-            self.push(result);
-          } else {
-            self.push(result.alternatives[0].transcript, 'utf8');
-          }
-
+          self.push(result.alternatives[0].transcript, 'utf8');
         }
       });
     } else {
diff --git a/speech-to-text/timing-stream.js b/speech-to-text/timing-stream.js
diff --git a/speech-to-text/webaudio-l16-stream.js b/speech-to-text/webaudio-l16-stream.js
