BOT-2750 added start/end timecodes to STT stream

Author: Florian Treml

frontend/src/stt/google.js

@@ -43,6 +43,7 @@ class GoogleSTT {
 
     const request = {
       config: {
+        enableWordTimeOffsets: true,
         encoding: 'LINEAR16',
         sampleRateHertz: 16000,
         languageCode: language
@@ -74,13 +75,18 @@ class GoogleSTT {
     const events = new EventEmitter()
 
     recognizeStream.on('data', (data) => {
-      const transcription = data.results[0] && data.results[0].alternatives[0] ? data.results[0].alternatives[0].transcript : null
-      if (transcription) {
-        events.emit('data', {
-          text: transcription,
+      const alternative = data.results[0] && data.results[0].alternatives[0]
+      if (alternative && alternative.transcript) {
+        const event = {
+          text: alternative.transcript,
           final: !!data.results[0].isFinal,
           debug: data
-        })
+        }
+        if (alternative.words && alternative.words.length > 0) {
+          event.start = _.round(_.toNumber(alternative.words[0].startTime.seconds) + _.toNumber(alternative.words[0].startTime.nanos) / 1000000000, 3)
+          event.end = _.round(_.toNumber(alternative.words[alternative.words.length - 1].endTime.seconds) + _.toNumber(alternative.words[alternative.words.length - 1].endTime.nanos) / 1000000000, 3)
+        }
+        events.emit('data', event)
       }
     })
     recognizeStream.on('error', (err) => {

frontend/src/stt/ibm.js

@@ -21,7 +21,8 @@ class IbmSTT {
       objectMode: true,
       contentType: 'audio/wav',
       model: language,
-      interimResults: true
+      interimResults: true,
+      timestamps: true
     }
     if (recognizeParams.model.length === 5) {
       recognizeParams.model = `${recognizeParams.model}_BroadbandModel`
@@ -46,11 +47,16 @@ class IbmSTT {
       for (const result of data.results || []) {
         const transcription = result.alternatives[0] ? result.alternatives[0].transcript : null
         if (transcription) {
-          events.emit('data', {
+          const event = {
             text: transcription,
             final: !!result.final,
             debug: result
-          })
+          }
+          if (result.alternatives[0].timestamps && result.alternatives[0].timestamps.length > 0) {
+            event.start = _.round(result.alternatives[0].timestamps[0][1], 3)
+            event.end = _.round(result.alternatives[result.alternatives.length - 1].timestamps[0][2], 3)
+          }
+          events.emit('data', event)
         }
       }
     })

frontend/src/stt/kaldi.js

@@ -19,7 +19,7 @@ class KaldiSTT {
   }
 
   stt_OpenStream (req, { language }) {
-    return new Promise((resolve) => {
+    return new Promise((resolve, reject) => {
       const kaldiUrl = getKaldiUrl(language)
 
       const qs = {
@@ -47,12 +47,19 @@ class KaldiSTT {
         ws.on('message', (data) => {
           try {
             const dj = JSON.parse(data)
-            if (dj.result && dj.result.hypotheses && dj.result.hypotheses.length > 0) {
-              events.emit('data', {
-                text: dj.result.hypotheses[0].transcript,
+            console.log(dj)
+            const hypotheses = dj.result && dj.result.hypotheses && dj.result.hypotheses[0]
+            if (hypotheses && hypotheses.transcript) {
+              const event = {
+                text: hypotheses.transcript,
                 final: !!dj.result.final,
                 debug: dj
-              })
+              }
+              if (dj['total-length']) {
+                event.start = _.round(_.toNumber(dj['total-length']) - _.toNumber(dj['segment-length']), 3)
+                event.end = _.round(_.toNumber(dj['total-length']), 3)
+              }
+              events.emit('data', event)
             }
           } catch (err) {
             debug(`received non JSON content on stream, ignoring. ${err.message}`)
@@ -86,6 +93,7 @@ class KaldiSTT {
           }
         })
       })
+      ws.on('error', (err) => reject(new Error(`Failed to open Kaldi stream to ${wsUri}: ${err.message}`)))
     })
   }