Using reasonable filenames for TTS result. Added some default configurations.

Florian Treml · Florian Treml · commit 5d5592694f27 · 2020-12-18T13:32:22.000+01:00
diff --git a/Makefile b/Makefile
@@ -1,23 +1,39 @@
 TAG_COMMIT := $(shell git rev-list --abbrev-commit --tags --max-count=1)
 VERSION := $(shell git describe --abbrev=0 --tags ${TAG_COMMIT} 2>/dev/null || true)
 
-docker_build:
+docker_build_develop:
+	docker build -t botium/botium-speech-frontend:develop frontend
+	docker build -t botium/botium-speech-watcher:develop watcher
+	docker build -f stt/Dockerfile.kaldi.en -t botium/botium-speech-kaldi-en:develop) stt
+	docker build -f stt/Dockerfile.kaldi.de -t botium/botium-speech-kaldi-de:develop stt
+	docker build -f tts/Dockerfile.marytts -t botium/botium-speech-marytts:develop tts
+	docker build -t botium/botium-speech-dictate:develop dictate
+
+docker_publish_develop:
+  docker push botium/botium-speech-frontend:develop
+  docker push botium/botium-speech-watcher:develop
+  docker push botium/botium-speech-kaldi-en:develop
+  docker push botium/botium-speech-kaldi-de:develop
+  docker push botium/botium-speech-marytts:develop
+  docker push botium/botium-speech-dictate:develop
+
+docker_build_release:
 	docker build -t botium/botium-speech-frontend:$(VERSION) frontend
 	docker build -t botium/botium-speech-watcher:$(VERSION) watcher
 	docker build -f stt/Dockerfile.kaldi.en -t botium/botium-speech-kaldi-en:$(VERSION) stt
 	docker build -f stt/Dockerfile.kaldi.de -t botium/botium-speech-kaldi-de:$(VERSION) stt
 	docker build -f tts/Dockerfile.marytts -t botium/botium-speech-marytts:$(VERSION) tts
 	docker build -t botium/botium-speech-dictate:$(VERSION) dictate
 
-docker_publish:
+docker_publish_release:
   docker push botium/botium-speech-frontend:$(VERSION)
   docker push botium/botium-speech-watcher:$(VERSION)
   docker push botium/botium-speech-kaldi-en:$(VERSION)
   docker push botium/botium-speech-kaldi-de:$(VERSION)
   docker push botium/botium-speech-marytts:$(VERSION)
   docker push botium/botium-speech-dictate:$(VERSION)
 
-docker_latest:
+docker_latest_release:
 	docker tag botium/botium-speech-frontend:$(VERSION) botium/botium-speech-frontend:latest
   docker push botium/botium-speech-frontend:latest
 
diff --git a/README.md b/README.md
@@ -153,6 +153,12 @@ This project is standing on the shoulders of giants.
 
 ## Changelog
 
+### 2020-12-18
+
+* Adding support for Google Text-To-Speech
+* Adding support for listing and using available TTS voices
+* Added sample docker-compose configurations for PicoTTS and Google
+
 ### 2020-03-05
 
 * Optional _start_/_end_ parameters for audio file conversion to trim an audio file by time codes formatted as mm:ss (_01:32_)
diff --git a/dictate/Dockerfile b/dictate/Dockerfile
@@ -11,4 +11,5 @@ RUN curl -L -o dicatejs.zip "https://github.com/Kaljurand/dictate.js/archive/mas
 
 EXPOSE 56100
 
+USER node
 CMD DICTATEDIR=/app/dictate.js-master npm start
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
@@ -10,12 +10,14 @@ services:
   frontend:
     build:
       context: frontend
+    image: botium/botium-speech-frontend:develop
     restart: always
     volumes:
       - "./frontend/resources:/app/resources"
 #  watcher:
 #    build:
 #      context: watcher
+#    image: botium/botium-speech-watcher:develop
 #    restart: always
 #    volumes:
 #      - "./watcher:/app/watch"
@@ -24,22 +26,26 @@ services:
 #    build:
 #      context: stt
 #      dockerfile: Dockerfile.kaldi.en
+#    image: botium/botium-speech-kaldi-en:develop
 #    restart: always
 #    volumes:
 #      - "./logs/stt_en:/opt/logs"
 #  stt_de:
 #    build:
 #      context: stt
 #      dockerfile: Dockerfile.kaldi.de
+#    image: botium/botium-speech-kaldi-de:develop
 #    restart: always
 #    volumes:
 #      - "./logs/stt_de:/opt/logs"
   tts:
     build:
       context: tts
       dockerfile: Dockerfile.marytts
+    image: botium/botium-speech-marytts:develop
     restart: always
 #  dictate:
 #    build:
 #      context: dictate
+#    image: botium/botium-speech-dictate:develop
 #    restart: always
diff --git a/docker-compose-google.yml b/docker-compose-google.yml
@@ -0,0 +1,26 @@
+version: '3'
+services:
+  nginx:
+    image: nginx
+    restart: always
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+    ports:
+      - 80:80
+  frontend:
+    image: botium/botium-speech-frontend:latest
+    restart: always
+    environment:
+      BOTIUM_API_TOKENS: 
+      BOTIUM_SPEECH_PROVIDER_TTS: google
+      BOTIUM_SPEECH_PROVIDER_STT: google
+      BOTIUM_SPEECH_GOOGLE_CLIENT_EMAIL:
+      BOTIUM_SPEECH_GOOGLE_PRIVATE_KEY:
+    volumes:
+      - "./frontend/resources:/app/resources"
+  watcher:
+    image: botium/botium-speech-watcher:latest
+    restart: always
+    volumes:
+      - "./watcher:/app/watch"
+      - "./logs/watcher:/app/logs"
diff --git a/docker-compose-picotts.yml b/docker-compose-picotts.yml
@@ -0,0 +1,37 @@
+version: '3'
+services:
+  nginx:
+    image: nginx
+    restart: always
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+    ports:
+      - 80:80
+  frontend:
+    image: botium/botium-speech-frontend:latest
+    restart: always
+    environment:
+      BOTIUM_API_TOKENS: 
+      BOTIUM_SPEECH_PROVIDER_TTS: picotts
+      BOTIUM_SPEECH_PROVIDER_STT: kaldi
+    volumes:
+      - "./frontend/resources:/app/resources"
+  watcher:
+    image: botium/botium-speech-watcher:latest
+    restart: always
+    volumes:
+      - "./watcher:/app/watch"
+      - "./logs/watcher:/app/logs"
+  stt_en:
+    image: botium/botium-speech-kaldi-en:latest
+    restart: always
+    volumes:
+      - "./logs/stt_en:/opt/logs"
+  stt_de:
+    image: botium/botium-speech-kaldi-de:latest
+    restart: always
+    volumes:
+      - "./logs/stt_de:/opt/logs"
+  dictate:
+    image: botium/botium-speech-dictate:latest
+    restart: always
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -10,6 +10,12 @@ services:
   frontend:
     image: botium/botium-speech-frontend:latest
     restart: always
+    environment:
+      BOTIUM_API_TOKENS: 
+      BOTIUM_SPEECH_PROVIDER_TTS: marytts
+      BOTIUM_SPEECH_PROVIDER_STT: kaldi
+      BOTIUM_SPEECH_GOOGLE_CLIENT_EMAIL:
+      BOTIUM_SPEECH_GOOGLE_PRIVATE_KEY:
     volumes:
       - "./frontend/resources:/app/resources"
   watcher:
diff --git a/frontend/Dockerfile b/frontend/Dockerfile
@@ -4,12 +4,15 @@ RUN apt-get update && apt-get -y install curl gnupg && curl -sL https://deb.node
 RUN apt-get install -y dos2unix sox libsox-fmt-mp3 libttspico-utils
 
 WORKDIR /app
-COPY . /app
-RUN find . -type f -print0 | xargs -0 dos2unix
+COPY ./package.json /app/package.json
 RUN npm install --no-optional --production
+COPY . /app
+RUN find . -type f ! -path '*/node_modules/*' -print0 | xargs -0 dos2unix
 
 VOLUME /app/resources
 
 EXPOSE 56000
 
-CMD npm start
+RUN groupadd --gid 1000 node && useradd --uid 1000 --gid node --shell /bin/bash --create-home node
+USER node
+CMD npm run start-dist
diff --git a/frontend/package.json b/frontend/package.json
@@ -6,6 +6,7 @@
     "eslint": "eslint src",
     "eslint-fix": "eslint --fix src",
     "start": "cross-env DOTENV_FLOW_PATH=./resources nodemon -w ./resources/.env -w ./resources/.env.local -w ./src/ -x \"node -r dotenv-flow/config\" ./src/server.js",
+    "start-dist": "cross-env DOTENV_FLOW_PATH=./resources node -r dotenv-flow/config ./src/server.js",
     "jsdoc": "swagger-jsdoc -d ./src/swaggerDef.json -o ./src/swagger.json src/routes.js"
   },
   "author": "Botium GmbH",
diff --git a/frontend/resources/.env b/frontend/resources/.env
@@ -10,6 +10,9 @@ BOTIUM_SPEECH_UPLOAD_LIMIT=50mb
 # Cache Path
 BOTIUM_SPEECH_CACHE_DIR=./resources/.cache
 
+# Temp Path
+BOTIUM_SPEECH_TMP_DIR=./resources/.tmp
+
 # Provider for TTS (google or marytts or picotts)
 BOTIUM_SPEECH_PROVIDER_TTS=marytts
 
diff --git a/frontend/src/convert/sox.js b/frontend/src/convert/sox.js
@@ -6,7 +6,7 @@ const debug = require('debug')('botium-speech-processing-convert-sox')
 
 const runsox = (cmdLine, { inputBuffer, start, end }) => {
   return new Promise((resolve, reject) => {
-    const output = `/tmp/${uuidv1()}.wav`
+    const output = `${process.env.BOTIUM_SPEECH_TMP_DIR || '/tmp'}/${uuidv1()}.wav`
 
     let cmdLineSox = Mustache.render(cmdLine, { output })
     if (start && end) {
diff --git a/frontend/src/routes.js b/frontend/src/routes.js
@@ -16,6 +16,10 @@ const cacheKeyTts = (data, language, voice, ext) => `${crypto.createHash('md5').
 if (cachePathStt) mkdirp.sync(cachePathStt)
 if (cachePathTts) mkdirp.sync(cachePathTts)
 
+if (process.env.BOTIUM_SPEECH_TMP_DIR) {
+  mkdirp.sync(process.env.BOTIUM_SPEECH_TMP_DIR)
+}
+
 const ttsEngines = {
   google: new (require('./tts/google'))(),
   marytts: new (require('./tts/marytts'))(),
diff --git a/frontend/src/tts/google.js b/frontend/src/tts/google.js
@@ -1,7 +1,7 @@
 const textToSpeech = require('@google-cloud/text-to-speech')
 const debug = require('debug')('botium-speech-processing-google-tts')
 
-const { googleOptions } = require('../utils')
+const { googleOptions, ttsFilename } = require('../utils')
 
 let googleVoices = null
 
@@ -54,7 +54,7 @@ class GoogleTTS {
       const [response] = await client.synthesizeSpeech(request)
       return {
         buffer: response.audioContent,
-        name: 'tts.wav'
+        name: `${ttsFilename(text)}.wav`
       }
     } catch (err) {
       debug(err)
diff --git a/frontend/src/tts/marytts.js b/frontend/src/tts/marytts.js
@@ -2,6 +2,8 @@ const _ = require('lodash')
 const request = require('request-promise-native')
 const debug = require('debug')('botium-speech-processing-marytts')
 
+const { ttsFilename } = require('../utils')
+
 let maryVoices = null
 
 class MaryTTS {
@@ -62,7 +64,7 @@ class MaryTTS {
       debug(`Called url ${requestOptions.uri} success`)
       return {
         buffer: response.body,
-        name: 'tts.wav'
+        name: `${ttsFilename(text)}.wav`
       }
     } else {
       throw new Error(`Calling url ${requestOptions.uri} failed with code ${response.statusCode}: ${response.statusMessage}`)
diff --git a/frontend/src/tts/picotts.js b/frontend/src/tts/picotts.js
@@ -3,6 +3,8 @@ const { spawn } = require('child_process')
 const { v1: uuidv1 } = require('uuid')
 const debug = require('debug')('botium-speech-processing-picotts')
 
+const { ttsFilename } = require('../utils')
+
 const voicesList = [
   {
     name: 'en-EN',
@@ -55,7 +57,7 @@ class PicoTTS {
     if (!picoVoice) throw new Error(`Voice <${voice || 'default'}> for language <${language}> not available`)
 
     return new Promise((resolve, reject) => {
-      const output = `/tmp/${uuidv1()}.wav`
+      const output = `${process.env.BOTIUM_SPEECH_TMP_DIR || '/tmp'}/${uuidv1()}.wav`
 
       const cmdLinePico = `${process.env.BOTIUM_SPEECH_PICO_CMDPREFIX || 'pico2wave'} --lang=${picoVoice.name} --wave=${output}`
       debug(`cmdLinePico: ${cmdLinePico}`)
@@ -70,7 +72,7 @@ class PicoTTS {
             fs.unlinkSync(output)
             resolve({
               buffer: outputBuffer,
-              name: 'tts.wav'
+              name: `${ttsFilename(text)}.wav`
             })
           } catch (err) {
             reject(new Error(`pico2wave process output file ${output} not readable: ${err.message}`))
diff --git a/frontend/src/utils.js b/frontend/src/utils.js
@@ -1,4 +1,6 @@
 const fs = require('fs')
+const _ = require('lodash')
+const sanitize = require('sanitize-filename')
 const speechScorer = require('word-error-rate')
 
 const wer = async (text1, text2) => {
@@ -8,6 +10,11 @@ const wer = async (text1, text2) => {
   }
 }
 
+const ttsFilename = (text) => {
+  const shortenedText = _.truncate(text, { length: 500 })
+  return sanitize(shortenedText)
+}
+
 const cleanEnv = (envName) => {
   return process.env[envName] && process.env[envName].replace(/\\n/g, '\n')
 }
@@ -28,5 +35,6 @@ const googleOptions = () => {
 
 module.exports = {
   wer,
+  ttsFilename,
   googleOptions
 }

Original file line number	Diff line number	Diff line change
`@@ -11,4 +11,5 @@ RUN curl -L -o dicatejs.zip "https://github.com/Kaljurand/dictate.js/archive/mas`
`11`	`11`
`12`	`12`	`EXPOSE 56100`
`13`	`13`
	`14`	`+USER node`
`14`	`15`	`CMD DICTATEDIR=/app/dictate.js-master npm start`