Merge pull request #7 from codeforequity-at/features/calculate-word-error-rate

features/calculate-word-error-rate

Author: Botium

frontend/package.json

@@ -25,10 +25,12 @@
     "nodemon": "^2.0.1",
     "request": "^2.88.0",
     "request-promise-native": "^1.0.8",
+    "sanitize-filename": "^1.6.3",
     "swagger-jsdoc": "^3.5.0",
     "swagger-ui-express": "^4.1.2",
     "uuid": "^3.3.3",
-    "winston": "^3.2.1"
+    "winston": "^3.2.1",
+    "word-error-rate": "0.0.7"
   },
   "devDependencies": {
     "eslint": "^6.7.2",

frontend/src/routes.js

@@ -3,7 +3,9 @@ const path = require('path')
 const mkdirp = require('mkdirp')
 const crypto = require('crypto')
 const express = require('express')
+const sanitize = require('sanitize-filename')
 const { runsox } = require('./convert/sox.js')
+const { wer } = require('./utils')
 const debug = require('debug')('botium-speech-processing-routes')
 
 const cachePathStt = process.env.BOTIUM_SPEECH_CACHE_DIR && path.join(process.env.BOTIUM_SPEECH_CACHE_DIR, 'stt')
@@ -15,9 +17,6 @@ if (cachePathTts) mkdirp.sync(cachePathTts)
 
 const router = express.Router()
 
-const tts = new (require(`./tts/${process.env.BOTIUM_SPEECH_PROVIDER_TTS}`))()
-const stt = new (require(`./stt/${process.env.BOTIUM_SPEECH_PROVIDER_STT}`))()
-
 /**
  * @swagger
  * components:
@@ -66,6 +65,19 @@ router.get('/api/status', (req, res) => {
  *         required: true
  *         schema:
  *           type: string
+ *       - name: hint
+ *         description: Hint text for calculating the Levenshtein edit distance for the result text (word error rate)
+ *         in: query
+ *         required: false
+ *         schema:
+ *           type: string
+ *       - name: stt
+ *         description: Speech-to-text backend
+ *         in: query
+ *         required: false
+ *         schema:
+ *           type: string
+ *           enum: [kaldi, google]
  *     requestBody:
  *       description: Audio file
  *       content:
@@ -97,10 +109,15 @@ router.post('/api/stt/:language', async (req, res, next) => {
       }
     }
     try {
+      const stt = new (require(`./stt/${(req.query.stt && sanitize(req.query.stt)) || process.env.BOTIUM_SPEECH_PROVIDER_STT}`))()
+
       const result = await stt.stt({
         language: req.params.language,
         buffer: req.body
       })
+      if (req.query.hint) {
+        result.wer = await wer(req.query.hint, result.text)
+      }
       res.json(result).end()
 
       if (cachePathStt) {
@@ -137,6 +154,13 @@ router.post('/api/stt/:language', async (req, res, next) => {
  *         required: true
  *         schema:
  *           type: string
+ *       - name: tts
+ *         description: Text-to-speech backend
+ *         in: query
+ *         required: false
+ *         schema:
+ *           type: string
+ *           enum: [marytts, picotts]
  *     responses:
  *       200:
  *         description: Audio file
@@ -169,6 +193,8 @@ router.get('/api/tts/:language', async (req, res, next) => {
       }
     }
     try {
+      const tts = new (require(`./tts/${(req.query.tts && sanitize(req.query.tts)) || process.env.BOTIUM_SPEECH_PROVIDER_TTS}`))()
+
       const { buffer, name } = await tts.tts({
         language: req.params.language,
         text: req.query.text
@@ -249,4 +275,40 @@ router.post('/api/convert/:profile', async (req, res, next) => {
   }
 })
 
+/**
+ * @swagger
+ * /api/wer:
+ *   get:
+ *     description: Calculate Levenshtein edit distance between two strings (word error rate)
+ *     security:
+ *       - ApiKeyAuth: []
+ *     produces:
+ *       - application/json
+ *     parameters:
+ *       - name: text1
+ *         description: Text
+ *         in: query
+ *         required: true
+ *         schema:
+ *           type: string
+ *       - name: text2
+ *         description: Text
+ *         in: query
+ *         required: true
+ *         schema:
+ *           type: string
+ *     responses:
+ *       200:
+ *         description: Levenshtein Edit Distance on word level
+ *         schema:
+ *           properties:
+ *             distance:
+ *               type: integer
+ *             wer:
+ *               type: number
+ */
+router.get('/api/wer', async (req, res) => {
+  res.json(await wer(req.query.text1, req.query.text2))
+})
+
 module.exports = router

frontend/src/stt/kaldi.js

@@ -18,6 +18,7 @@ class Kaldi {
 
     let response
     try {
+      debug(`Calling kaldi url ${requestOptions.uri} ...`)
       response = await request(requestOptions)
     } catch (err) {
       throw new Error(`Calling url ${requestOptions.uri} failed: ${err.message}`)

frontend/src/swagger.json

@@ -56,6 +56,28 @@
             "schema": {
               "type": "string"
             }
+          },
+          {
+            "name": "hint",
+            "description": "Hint text for calculating the Levenshtein edit distance for the result text (word error rate)",
+            "in": "query",
+            "required": false,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "stt",
+            "description": "Speech-to-text backend",
+            "in": "query",
+            "required": false,
+            "schema": {
+              "type": "string",
+              "enum": [
+                "kaldi",
+                "google"
+              ]
+            }
           }
         ],
         "requestBody": {
@@ -112,6 +134,19 @@
             "schema": {
               "type": "string"
             }
+          },
+          {
+            "name": "tts",
+            "description": "Text-to-speech backend",
+            "in": "query",
+            "required": false,
+            "schema": {
+              "type": "string",
+              "enum": [
+                "marytts",
+                "picotts"
+              ]
+            }
           }
         ],
         "responses": {
@@ -176,6 +211,54 @@
           }
         }
       }
+    },
+    "/api/wer": {
+      "get": {
+        "description": "Calculate Levenshtein edit distance between two strings (word error rate)",
+        "security": [
+          {
+            "ApiKeyAuth": []
+          }
+        ],
+        "produces": [
+          "application/json"
+        ],
+        "parameters": [
+          {
+            "name": "text1",
+            "description": "Text",
+            "in": "query",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "text2",
+            "description": "Text",
+            "in": "query",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Levenshtein Edit Distance on word level",
+            "schema": {
+              "properties": {
+                "distance": {
+                  "type": "integer"
+                },
+                "wer": {
+                  "type": "number"
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "components": {

frontend/src/utils.js

@@ -0,0 +1,12 @@
+const speechScorer = require('word-error-rate')
+
+const wer = async (text1, text2) => {
+  return {
+    distance: speechScorer.calculateEditDistance(text1 || '', text2 || ''),
+    wer: speechScorer.wordErrorRate(text1 || '', text2 || '')
+  }
+}
+
+module.exports = {
+  wer
+}