diff --git a/Dockerfile b/Dockerfile index 48216a2..378ddde 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,5 +10,5 @@ RUN yarn build COPY . . -EXPOSE 8080 +EXPOSE 8337 CMD ['node', 'index.js'] \ No newline at end of file diff --git a/bin/patch-deepspeech.sh b/bin/patch-deepspeech.sh new file mode 100755 index 0000000..a315f3f --- /dev/null +++ b/bin/patch-deepspeech.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +DS_PATH="./node_modules/deepspeech/lib/binding/v0.10.0-alpha.3/darwin-x64" + +if [[ ! $OSTYPE == 'darwin'* ]]; then + echo "Patch currently only works on MacOS" + exit 0 +fi + +if [ -f "$DS_PATH/electron-v15.3" ]; then + echo "Deepspeech already fixed" +else + curl -L https://github.com/vanstinator/SubD/files/7532394/electron-v15.3.zip -o "$DS_PATH/electron-v15.3.zip" + mkdir "$DS_PATH/electron-v15.3" + unzip "$DS_PATH/electron-v15.3.zip" -d "$DS_PATH" +fi \ No newline at end of file diff --git a/core/examples/detect-subtitles.ts b/core/examples/detect-subtitles.ts deleted file mode 100644 index 0b61523..0000000 --- a/core/examples/detect-subtitles.ts +++ /dev/null @@ -1,14 +0,0 @@ -import 'config'; -import audioStream from 'core/services/media/audioStream'; -import detectSpeech from 'core/services/speech/detect'; - -const MOVIE_FILE = - 'https://archive.org/download/cartoon-network-21st-century/Cartoon%20Network%20-%2021st%20century.mp4'; - -const stream = audioStream(MOVIE_FILE); -detectSpeech(stream) - .then(text => { - // console.log('speech', text.transcripts?.[0].tokens); - console.log('Detected text:', text); - }) - .catch(console.error); diff --git a/core/main.ts b/core/main.ts index eb37be8..32db0df 100644 --- a/core/main.ts +++ b/core/main.ts @@ -1,13 +1,11 @@ import { app, BrowserWindow } from 'electron'; import registerUpdater from 'core/update'; import createServer from 'core/services/server'; -import ipcListeners from 'core/services/ipc'; +import 'core/services/ipc/listeners'; declare const MAIN_WINDOW_WEBPACK_ENTRY: string; declare const MAIN_WINDOW_PRELOAD_WEBPACK_ENTRY: string; -ipcListeners(); - // modify your existing createWindow() function function createWindow() { const win = new BrowserWindow({ diff --git a/core/services/ipc.ts b/core/services/ipc.ts deleted file mode 100644 index fc9f131..0000000 --- a/core/services/ipc.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { ipcMain } from 'electron'; -import logger from 'electron-log'; -import queue from 'core/services/queue'; -import { - getModelPaths, - verifyModelFiles, - downloadModelFiles, - Downloader, - DownloadProgress -} from 'core/services/speech/download'; - -const log = logger.scope('ipc'); - -let activeDownloader: Downloader; - -export default function ipcListeners() { - ipcMain.on('analyze.start', (event, files: { movie: BasicFile; subtitle: BasicFile }) => { - log.info('analyze.start', files); - queue.queueMedia(files.movie, files.subtitle); - }); - - ipcMain.on('analyze.startupCheck', event => { - const hasModelFiles = verifyModelFiles(); - event.reply('analyze.startupResult', { hasModelFiles }); - }); - ipcMain.on('analyze.download.start', event => { - // start download - activeDownloader = downloadModelFiles(); - activeDownloader.on('progress', progress => { - event.reply('analyze.download.progress', progress); - }); - activeDownloader.on('finish', () => { - log.info('Download finished'); - event.reply('analyze.download.finished'); - }); - activeDownloader.on('error', error => { - log.info('Download error', error); - event.reply('analyze.download.error', error); - }); - activeDownloader.start(); - }); - ipcMain.on('analyze.download.cancel', event => { - log.info('Cancel download'); - activeDownloader.cancel(); - }); -} diff --git a/core/services/ipc/broadcast.ts b/core/services/ipc/broadcast.ts new file mode 100644 index 0000000..43f861c --- /dev/null +++ b/core/services/ipc/broadcast.ts @@ -0,0 +1,7 @@ +import { webContents } from 'electron'; + +export default function broadcastMessage(eventName: string, ...args: any) { + webContents.getAllWebContents().forEach(wc => { + wc.send(eventName, ...args); + }); +} diff --git a/core/services/ipc/listeners/analyze.ts b/core/services/ipc/listeners/analyze.ts new file mode 100644 index 0000000..dd3b5b0 --- /dev/null +++ b/core/services/ipc/listeners/analyze.ts @@ -0,0 +1,10 @@ +import { ipcMain } from 'electron'; +import logger from 'electron-log'; +import queue from 'core/services/queue'; + +const log = logger.scope('ipc_analyze'); + +ipcMain.on('analyze.start', (event, files: { movie: BasicFile; subtitle: BasicFile }) => { + log.info('analyze.start', files); + queue.queueMedia(files.movie, files.subtitle); +}); diff --git a/core/services/ipc/listeners/index.ts b/core/services/ipc/listeners/index.ts new file mode 100644 index 0000000..b924ecf --- /dev/null +++ b/core/services/ipc/listeners/index.ts @@ -0,0 +1,3 @@ +import './analyze'; +import './queue'; +import './setup'; diff --git a/core/services/ipc/listeners/queue.ts b/core/services/ipc/listeners/queue.ts new file mode 100644 index 0000000..769f780 --- /dev/null +++ b/core/services/ipc/listeners/queue.ts @@ -0,0 +1,6 @@ +import { ipcMain } from 'electron'; +import queue from 'core/services/queue'; + +ipcMain.on('queue.get', event => { + event.reply('queue.update', queue.queue); +}); diff --git a/core/services/ipc/listeners/setup.ts b/core/services/ipc/listeners/setup.ts new file mode 100644 index 0000000..b7eeda3 --- /dev/null +++ b/core/services/ipc/listeners/setup.ts @@ -0,0 +1,34 @@ +import { ipcMain } from 'electron'; +import logger from 'electron-log'; +import { verifyModelFiles, downloadModelFiles, Downloader } from 'core/services/speech/download'; + +const log = logger.scope('ipc_analyze'); + +let downloader: Downloader; + +ipcMain.on('setup.startupCheck', event => { + const hasModelFiles = verifyModelFiles(); + event.reply('setup.startupResult', { hasModelFiles }); +}); + +ipcMain.on('setup.download.start', event => { + // start download + downloader = downloadModelFiles(); + downloader.on('progress', progress => { + event.reply('setup.download.progress', progress); + }); + downloader.on('finish', () => { + log.info('Download finished'); + event.reply('setup.download.finished'); + }); + downloader.on('error', error => { + log.info('Download error', error); + event.reply('setup.download.error', error); + }); + downloader.start(); +}); + +ipcMain.on('setup.download.cancel', event => { + log.info('Cancel download'); + downloader.cancel(); +}); diff --git a/core/services/media/audioStream.ts b/core/services/media/audioStream.ts deleted file mode 100644 index 5d50b06..0000000 --- a/core/services/media/audioStream.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { spawn } from 'child_process'; -import pathToFfmpeg from 'ffmpeg-static'; -import { Readable } from 'stream'; -import { getDesiredSampleRate } from 'core/services/speech/detect'; - -export default function extractAudio(inputFile: string): Readable { - const sampleRate = getDesiredSampleRate(); - - const child = spawn(pathToFfmpeg, [ - '-i', - inputFile, - '-hide_banner', - '-loglevel', - 'error', - '-ar', - sampleRate.toString(), - '-ac', - '1', - '-f', - 's16le', - '-acodec', - 'pcm_s16le', - 'pipe:1' - ]); - let ffmpegOutput = ''; - child.stderr.on('data', (data: Buffer) => { - ffmpegOutput += data.toString('utf-8'); - }); - child.on('exit', code => { - if (code !== 0) { - console.error(ffmpegOutput); - throw new Error(`ffmpeg exited with code: ${code}`); - } - }); - return child.stdout; -} diff --git a/core/services/queue/jobs/analyze.ts b/core/services/queue/jobs/analyze.ts new file mode 100644 index 0000000..7f19d67 --- /dev/null +++ b/core/services/queue/jobs/analyze.ts @@ -0,0 +1,18 @@ +import detectSpeech from 'core/services/speech/detect'; +import groupByWord from 'core/services/speech/groupByWord'; +import compareToSubtitle from 'core/services/subtitles/compare'; + +export default async function analyzeJob( + mediaPath: string, + subtitlePath: string, + onProgress: (percent: number) => void +) { + const speech = await detectSpeech(mediaPath, onProgress); + const words = groupByWord(speech.text); + if (words) { + compareToSubtitle(subtitlePath, words); + } + return { + score: 100 + }; +} diff --git a/core/services/queue/queue.ts b/core/services/queue/queue.ts index 54df8ef..ee02fe1 100644 --- a/core/services/queue/queue.ts +++ b/core/services/queue/queue.ts @@ -1,6 +1,11 @@ import { v4 as uuid } from 'uuid'; +import logger from 'electron-log'; +import broadcastMessage from 'core/services/ipc/broadcast'; +import analyzeJob from './jobs/analyze'; -const MAX_SIZE = 100; +const log = logger.scope('queue'); + +const MAX_SIZE = 10; enum QUEUE_STATUS { COMPLETE = 'complete', @@ -10,17 +15,21 @@ enum QUEUE_STATUS { } interface QueueItem { + data?: any; id: string; media: BasicFile; progress: number; status: QUEUE_STATUS; + error?: string; subtitle: BasicFile; } class JobQueue { + activeIndex: number; queue: QueueItem[]; constructor() { + this.activeIndex = -1; this.queue = []; } @@ -35,6 +44,50 @@ class JobQueue { progress: 0, status: QUEUE_STATUS.QUEUED }); + + this.postUpdate(); + + if (this.activeIndex < 0) { + this.start(); + } + } + + postUpdate() { + broadcastMessage('queue.update', this.queue); + } + + async start() { + this.activeIndex = this.queue.findIndex(item => item.status === QUEUE_STATUS.QUEUED); + log.debug('this.activeIndex', this.activeIndex); + if (this.activeIndex < 0) { + log.warn('No jobs in queue'); + return; + } + const activeItem = this.queue[this.activeIndex]; + activeItem.status = QUEUE_STATUS.PROCESSING; + this.postUpdate(); + + try { + await this.runJob(); + activeItem.status = QUEUE_STATUS.COMPLETE; + activeItem.progress = 1; + this.postUpdate(); + } catch (error) { + activeItem.status = QUEUE_STATUS.ERROR; + } + this.postUpdate(); + } + + async runJob() { + // TODO: Edit this to allow for other types of jobs? + this.queue[this.activeIndex].data = await analyzeJob( + this.queue[this.activeIndex].media.path, + this.queue[this.activeIndex].subtitle.path, + percent => { + this.queue[this.activeIndex].progress = percent; + this.postUpdate(); + } + ); } } diff --git a/core/services/server/index.ts b/core/services/server/index.ts index 00354aa..9ad4a95 100644 --- a/core/services/server/index.ts +++ b/core/services/server/index.ts @@ -1,7 +1,7 @@ import express from 'express'; import logger from 'electron-log'; -const PORT = process.env.PORT || 8080; +const PORT = process.env.PORT || 8337; const log = logger.scope('express'); const app = express(); diff --git a/core/services/speech/audio.ts b/core/services/speech/audio.ts new file mode 100644 index 0000000..9dc7bb8 --- /dev/null +++ b/core/services/speech/audio.ts @@ -0,0 +1,51 @@ +import { exec, spawn } from 'child_process'; +import pathToFfmpeg from 'ffmpeg-static'; +import { Readable } from 'stream'; + +const DURATION_REGEX = /Duration:\s(\d{2}):(\d{2}):([\d.]+)/; + +console.log(pathToFfmpeg); + +export function getDuration(inputFile: string): Promise { + return new Promise((resolve, reject) => { + exec(`"${pathToFfmpeg}" -hide_banner -i "${inputFile}"`, (error, stdout, stderr) => { + const matches = stderr.match(DURATION_REGEX); + if (matches?.length === 4) { + const [, hours, minutes, seconds] = matches; + const duration = parseInt(hours, 10) * 3600 + parseInt(minutes, 10) * 60 + parseFloat(seconds); + return resolve(duration); + } + reject('Could not determine media duration'); + }); + }); +} + +export default function extractAudio(inputFile: string, sampleRate: number): Readable { + const child = spawn(pathToFfmpeg, [ + '-i', + inputFile, + '-hide_banner', + '-loglevel', + 'error', + '-ar', + sampleRate.toString(), + '-ac', + '1', + '-f', + 's16le', + '-acodec', + 'pcm_s16le', + 'pipe:1' + ]); + let ffmpegOutput = ''; + child.stderr.on('data', (data: Buffer) => { + ffmpegOutput += data.toString('utf-8'); + }); + child.on('exit', code => { + if (code !== 0) { + console.error(ffmpegOutput); + throw new Error(`ffmpeg exited with code: ${code}`); + } + }); + return child.stdout; +} diff --git a/core/services/speech/detect.ts b/core/services/speech/detect.ts index 5631838..84f48d8 100644 --- a/core/services/speech/detect.ts +++ b/core/services/speech/detect.ts @@ -1,7 +1,11 @@ import path from 'path'; -import { Model } from 'deepspeech'; +import { Model, CandidateTranscript } from 'deepspeech'; +import logger from 'electron-log'; import { Readable } from 'stream'; import { getModelPaths } from './download'; +import createAudioStream, { getDuration } from './audio'; + +const log = logger.scope('detection'); const modelPaths = getModelPaths(); if (!modelPaths.model || !modelPaths.scorer) { @@ -10,23 +14,31 @@ if (!modelPaths.model || !modelPaths.scorer) { const model = new Model(modelPaths.model); model.enableExternalScorer(modelPaths.scorer); -export function getDesiredSampleRate() { - return model.sampleRate(); +interface SpeechResult { + text: { transcripts: CandidateTranscript[] }; + audioDuration: number; } -export default async function detectSpeech(audioStream: Readable) { +export default async function detectSpeech( + mediaPath: string, + onProgress: (percent: number) => void +): Promise { + const sampleRate = model.sampleRate(); + const audioDuration = await getDuration(mediaPath); + log.info(`mediaPath: ${mediaPath}`); + log.info(`audioDuration: ${audioDuration} ${sampleRate}`); + const audioStream = createAudioStream(mediaPath, sampleRate); return new Promise((resolve, reject) => { - let duration = 0; + let currentTime = 0; const modelStream = model.createStream(); audioStream.on('data', chunk => { - duration += (chunk.length / 2) * (1 / getDesiredSampleRate()); - process.stdout.write(`processed ${duration} seconds\r`); + currentTime += (chunk.length / 2) * (1 / sampleRate); + onProgress(currentTime / audioDuration); modelStream.feedAudioContent(chunk); }); audioStream.on('close', () => { - process.stdout.write('\n'); const text = modelStream.finishStreamWithMetadata(); - resolve({ text, duration }); + resolve({ text, audioDuration }); }); }); } diff --git a/core/services/speech/groupByWord.ts b/core/services/speech/groupByWord.ts new file mode 100644 index 0000000..a0a2c76 --- /dev/null +++ b/core/services/speech/groupByWord.ts @@ -0,0 +1,39 @@ +import { Metadata, TokenMetadata } from 'deepspeech'; + +export interface WordGroup { + text: string; + start_time: number; + end_time: number; +} + +export default function groupByWord(data: Metadata): WordGroup[] { + // convert characters to words + console.log('data.transcripts', data.transcripts.length); + const tokens = data.transcripts?.[0].tokens; + const wordGroups = tokens.reduce((prev, token) => { + if (!prev.length && token.text !== ' ') { + prev.push([token]); + return prev; + } + if (token.text === ' ') { + prev.push([]); + } else { + prev[prev.length - 1].push(token); + } + return prev; + }, [] as TokenMetadata[][]); + + return wordGroups.map(group => { + return group.reduce( + (prev, current, index) => { + prev.text += current.text; + return prev; + }, + { + text: '', + end_time: Math.max(...group.map(({ start_time }) => start_time)), + start_time: Math.min(...group.map(({ start_time }) => start_time)) + } + ); + }); +} diff --git a/core/services/subtitles/compare.ts b/core/services/subtitles/compare.ts new file mode 100644 index 0000000..2742299 --- /dev/null +++ b/core/services/subtitles/compare.ts @@ -0,0 +1,84 @@ +import fs from 'fs'; +import { parse, map } from 'subtitle'; +import { WordGroup } from 'core/services/speech/groupByWord'; +import leven from 'leven'; + +const FUZZY_TOLERANCE = 2; + +interface Cue { + startTime: number; + endTime: number; + words: string[]; + text: string; + mapped?: WordGroup[]; +} + +export default function compare(subtitleFile: string, wordGroups: WordGroup[]): Promise { + return new Promise((resolve, reject) => { + const cues: Cue[] = []; + const stream = fs.createReadStream(subtitleFile).pipe(parse()); + stream + .on('data', node => { + if (node.type === 'cue') { + const { end, start, text } = node.data; + const startTime = start / 1000; // ms to seconds + const endTime = end / 1000; // ms to seconds + const words = text + .toLowerCase() + .replace('\n', ' ') + .replace(/[^\w ]+/g, '') + .trim() + .split(' '); + cues.push({ startTime, endTime, words, text }); + } + }) + .on('error', console.error) + .on('finish', () => { + // console.log(wordGroups); + let lastIndex = -1; + console.log('Cuepoint', cues[0]); + const mapped = cues.slice(0, 1).map(subCue => { + const { words } = subCue; + const found = words + .map(cueWord => { + // console.log(`cue: ${cueWord}`); + for (let i = lastIndex + 1; i < wordGroups.length; i++) { + const wg = wordGroups[i]; + const matchScore = leven(wg.text, cueWord); + // console.log('- iter', wg.text, matchScore); + if ( + matchScore <= FUZZY_TOLERANCE || + (cueWord.length > 3 && wg.text.length > 3 && new RegExp(`^${wg.text}|${wg.text}`).test(cueWord)) + ) { + // console.log('- match', cueWord, wg.text, wg.start_time); + lastIndex = i; + return wg; + } + } + }) + .filter((wg): wg is WordGroup => !!wg) + .reduce( + (obj, wg, index) => { + return { + speech: [...obj.speech, wg.text], + start: index === 0 ? wg.start_time : Math.min(obj.start, wg.start_time), + end: index === 0 ? wg.end_time : Math.max(obj.end, wg.end_time) + // end: Math.max(obj.end, wg.end_time) + }; + }, + { + speech: [] as string[], + start: -1, + end: -1 + } + ); + if (found) { + console.log(found); + const accuracy = (cues[0].startTime - found.start) * 1000; + console.log(`SRT Accuracy: ${accuracy} ms`); + } + }); + resolve(); + }); + }); +} diff --git a/forge.config.js b/forge.config.js index d2a8d00..e09569a 100644 --- a/forge.config.js +++ b/forge.config.js @@ -38,7 +38,8 @@ const config = { } ] } - } + }, + ['@electron-forge/plugin-auto-unpack-natives'] ] ], makers: [ diff --git a/package-lock.json b/package-lock.json index 83e851e..d6d0f0c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,17 +14,19 @@ "@mui/icons-material": "^5.0.5", "@mui/material": "^5.0.6", "debounce": "^1.2.1", - "deepspeech": "^0.9.3", + "deepspeech": "^0.10.0-alpha.3", "electron-log": "^4.4.1", "electron-squirrel-startup": "^1.0.0", "express": "^4.17.1", "ffmpeg-static": "^4.4.0", "got": "^11.8.2", "history": "^5.1.0", + "leven": "^4.0.0", "react": "^17.0.2", "react-dom": "^17.0.2", "react-router-dom": "^6.0.1", "react-scripts": "4.0.3", + "subtitle": "^4.1.1", "tar": "^6.1.11", "uuid": "^8.3.2" }, @@ -40,6 +42,7 @@ "@electron-forge/maker-rpm": "^6.0.0-beta.61", "@electron-forge/maker-squirrel": "^6.0.0-beta.61", "@electron-forge/maker-zip": "^6.0.0-beta.61", + "@electron-forge/plugin-auto-unpack-natives": "^6.0.0-beta.61", "@electron-forge/plugin-webpack": "^6.0.0-beta.61", "@marshallofsound/webpack-asset-relocator-loader": "^0.5.0", "@testing-library/jest-dom": "^5.15.0", @@ -2425,6 +2428,19 @@ "node": ">= 12.13.0" } }, + "node_modules/@electron-forge/plugin-auto-unpack-natives": { + "version": "6.0.0-beta.61", + "resolved": "https://registry.npmjs.org/@electron-forge/plugin-auto-unpack-natives/-/plugin-auto-unpack-natives-6.0.0-beta.61.tgz", + "integrity": "sha512-jgkC1E/zkpPJbY8FO7rXFWGWkMTqJNlhMXIY3WbR5EO7Vsni4mC6AVsXjtBSnFnlZXzCvxt7qd1pvc+qUVsMGQ==", + "dev": true, + "dependencies": { + "@electron-forge/plugin-base": "6.0.0-beta.61", + "@electron-forge/shared-types": "6.0.0-beta.61" + }, + "engines": { + "node": ">= 12.13.0" + } + }, "node_modules/@electron-forge/plugin-base": { "version": "6.0.0-beta.61", "dev": true, @@ -5479,6 +5495,14 @@ "@types/node": "*" } }, + "node_modules/@types/multipipe": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/multipipe/-/multipipe-3.0.0.tgz", + "integrity": "sha512-CbhyiQkqlGTacMjyw64y1/jIFBJr0TKPefLyUyXmIhabNv5rA8X1+60ss3TjlLoM3JsK288HVyPwTnO0nHawJA==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/node": { "version": "12.20.36", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.36.tgz", @@ -10056,8 +10080,9 @@ } }, "node_modules/deepspeech": { - "version": "0.9.3", - "license": "MPL-2.0", + "version": "0.10.0-alpha.3", + "resolved": "https://registry.npmjs.org/deepspeech/-/deepspeech-0.10.0-alpha.3.tgz", + "integrity": "sha512-Qt2S7bf2waRfINbOriWiU79xOwRSJ13ph74xFOB6/hc2nHdli0EhHPK/H5rEjuJNgT5+VYau+y7HQeK+crjltw==", "dependencies": { "argparse": "1.0.x", "memory-stream": "1.0.x", @@ -10569,6 +10594,41 @@ "version": "0.1.2", "license": "MIT" }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", @@ -18106,6 +18166,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/jest-validate/node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/jest-validate/node_modules/pretty-format": { "version": "26.6.2", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-26.6.2.tgz", @@ -18545,12 +18614,14 @@ } }, "node_modules/leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true, + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-4.0.0.tgz", + "integrity": "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw==", "engines": { - "node": ">=6" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/levn": { @@ -19595,6 +19666,15 @@ "integrity": "sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE=", "dev": true }, + "node_modules/multipipe": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/multipipe/-/multipipe-4.0.0.tgz", + "integrity": "sha512-jzcEAzFXoWwWwUbvHCNPwBlTz3WCWe/jPcXSmTfbo/VjRwRTfvLZ/bdvtiTdqCe8d4otCSsPCbhGYcX+eggpKQ==", + "dependencies": { + "duplexer2": "^0.1.2", + "object-assign": "^4.1.0" + } + }, "node_modules/murmur-32": { "version": "0.2.0", "dev": true, @@ -26553,6 +26633,14 @@ "node": ">=0.10.0" } }, + "node_modules/split2": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", + "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", + "dependencies": { + "readable-stream": "^3.0.0" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "license": "BSD-3-Clause" @@ -27080,6 +27168,28 @@ "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.0.10.tgz", "integrity": "sha512-m3k+dk7QeJw660eIKRRn3xPF6uuvHs/FFzjX3HQ5ove0qYsiygoAhwn5a3IYKaZPo5LrYD0rfVmtv1gNY1uYwg==" }, + "node_modules/subtitle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/subtitle/-/subtitle-4.1.1.tgz", + "integrity": "sha512-FfRj+xwGOOLSDUMAv9LSvS6rUDvaMmHE7rD6e3p/qrYyxD3+EuTj7cUbDthWwVmbFxVRnGGngqEVVwW9tdTR2g==", + "dependencies": { + "@types/multipipe": "^3.0.0", + "multipipe": "^4.0.0", + "split2": "^3.2.2", + "strip-bom": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/subtitle/node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "engines": { + "node": ">=8" + } + }, "node_modules/sudo-prompt": { "version": "9.2.1", "dev": true, @@ -32117,6 +32227,16 @@ "fs-extra": "^10.0.0" } }, + "@electron-forge/plugin-auto-unpack-natives": { + "version": "6.0.0-beta.61", + "resolved": "https://registry.npmjs.org/@electron-forge/plugin-auto-unpack-natives/-/plugin-auto-unpack-natives-6.0.0-beta.61.tgz", + "integrity": "sha512-jgkC1E/zkpPJbY8FO7rXFWGWkMTqJNlhMXIY3WbR5EO7Vsni4mC6AVsXjtBSnFnlZXzCvxt7qd1pvc+qUVsMGQ==", + "dev": true, + "requires": { + "@electron-forge/plugin-base": "6.0.0-beta.61", + "@electron-forge/shared-types": "6.0.0-beta.61" + } + }, "@electron-forge/plugin-base": { "version": "6.0.0-beta.61", "dev": true, @@ -34408,6 +34528,14 @@ "@types/node": "*" } }, + "@types/multipipe": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/multipipe/-/multipipe-3.0.0.tgz", + "integrity": "sha512-CbhyiQkqlGTacMjyw64y1/jIFBJr0TKPefLyUyXmIhabNv5rA8X1+60ss3TjlLoM3JsK288HVyPwTnO0nHawJA==", + "requires": { + "@types/node": "*" + } + }, "@types/node": { "version": "12.20.36", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.36.tgz", @@ -38003,7 +38131,9 @@ "dev": true }, "deepspeech": { - "version": "0.9.3", + "version": "0.10.0-alpha.3", + "resolved": "https://registry.npmjs.org/deepspeech/-/deepspeech-0.10.0-alpha.3.tgz", + "integrity": "sha512-Qt2S7bf2waRfINbOriWiU79xOwRSJ13ph74xFOB6/hc2nHdli0EhHPK/H5rEjuJNgT5+VYau+y7HQeK+crjltw==", "requires": { "argparse": "1.0.x", "memory-stream": "1.0.x", @@ -38406,6 +38536,43 @@ "duplexer": { "version": "0.1.2" }, + "duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", + "requires": { + "readable-stream": "^2.0.2" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", @@ -44176,6 +44343,12 @@ "integrity": "sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg==", "dev": true }, + "leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true + }, "pretty-format": { "version": "26.6.2", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-26.6.2.tgz", @@ -44525,10 +44698,9 @@ } }, "leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-4.0.0.tgz", + "integrity": "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw==" }, "levn": { "version": "0.4.1", @@ -45323,6 +45495,15 @@ "integrity": "sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE=", "dev": true }, + "multipipe": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/multipipe/-/multipipe-4.0.0.tgz", + "integrity": "sha512-jzcEAzFXoWwWwUbvHCNPwBlTz3WCWe/jPcXSmTfbo/VjRwRTfvLZ/bdvtiTdqCe8d4otCSsPCbhGYcX+eggpKQ==", + "requires": { + "duplexer2": "^0.1.2", + "object-assign": "^4.1.0" + } + }, "murmur-32": { "version": "0.2.0", "dev": true, @@ -50787,6 +50968,14 @@ "extend-shallow": "^3.0.0" } }, + "split2": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", + "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", + "requires": { + "readable-stream": "^3.0.0" + } + }, "sprintf-js": { "version": "1.0.3" }, @@ -51204,6 +51393,24 @@ "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.0.10.tgz", "integrity": "sha512-m3k+dk7QeJw660eIKRRn3xPF6uuvHs/FFzjX3HQ5ove0qYsiygoAhwn5a3IYKaZPo5LrYD0rfVmtv1gNY1uYwg==" }, + "subtitle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/subtitle/-/subtitle-4.1.1.tgz", + "integrity": "sha512-FfRj+xwGOOLSDUMAv9LSvS6rUDvaMmHE7rD6e3p/qrYyxD3+EuTj7cUbDthWwVmbFxVRnGGngqEVVwW9tdTR2g==", + "requires": { + "@types/multipipe": "^3.0.0", + "multipipe": "^4.0.0", + "split2": "^3.2.2", + "strip-bom": "^4.0.0" + }, + "dependencies": { + "strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==" + } + } + }, "sudo-prompt": { "version": "9.2.1", "dev": true diff --git a/package.json b/package.json index b644f6f..cb3501d 100644 --- a/package.json +++ b/package.json @@ -4,11 +4,12 @@ "license": "MIT", "main": "./.webpack/main/index.js", "scripts": { - "start": "electron-forge start", + "start": "electron-forge start --inspect-electron", "package": "electron-forge package", "make": "electron-forge make", "release": "electron-forge publish", - "lint": "eslint . --ext js,ts" + "lint": "eslint . --ext js,ts", + "postinstall": "bin/patch-deepspeech.sh" }, "dependencies": { "@emotion/react": "^11.5.0", @@ -16,17 +17,19 @@ "@mui/icons-material": "^5.0.5", "@mui/material": "^5.0.6", "debounce": "^1.2.1", - "deepspeech": "^0.9.3", + "deepspeech": "^0.10.0-alpha.3", "electron-log": "^4.4.1", "electron-squirrel-startup": "^1.0.0", "express": "^4.17.1", "ffmpeg-static": "^4.4.0", "got": "^11.8.2", "history": "^5.1.0", + "leven": "^4.0.0", "react": "^17.0.2", "react-dom": "^17.0.2", "react-router-dom": "^6.0.1", "react-scripts": "4.0.3", + "subtitle": "^4.1.1", "tar": "^6.1.11", "uuid": "^8.3.2" }, @@ -42,6 +45,7 @@ "@electron-forge/maker-rpm": "^6.0.0-beta.61", "@electron-forge/maker-squirrel": "^6.0.0-beta.61", "@electron-forge/maker-zip": "^6.0.0-beta.61", + "@electron-forge/plugin-auto-unpack-natives": "^6.0.0-beta.61", "@electron-forge/plugin-webpack": "^6.0.0-beta.61", "@marshallofsound/webpack-asset-relocator-loader": "^0.5.0", "@testing-library/jest-dom": "^5.15.0", diff --git a/src/App.tsx b/src/App.tsx index 40569bf..c3c7642 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -27,8 +27,8 @@ function App() { }, []); useEffect(() => { - window.api.ipcSend('analyze.startupCheck'); - window.api.addIpcListener('analyze.startupResult', handleStartupResult); + window.api.ipcSend('setup.startupCheck'); + window.api.addIpcListener('setup.startupResult', handleStartupResult); }, []); return ( diff --git a/src/components/SelectFileButton.tsx b/src/components/SelectFileButton.tsx index bc8ba99..c1b4401 100644 --- a/src/components/SelectFileButton.tsx +++ b/src/components/SelectFileButton.tsx @@ -51,7 +51,9 @@ export default function SelectFileButton({ )} diff --git a/src/components/dialogs/DownloadModelFilesDialog.tsx b/src/components/dialogs/DownloadModelFilesDialog.tsx index 77e5438..747eb75 100644 --- a/src/components/dialogs/DownloadModelFilesDialog.tsx +++ b/src/components/dialogs/DownloadModelFilesDialog.tsx @@ -23,12 +23,12 @@ export default function DownloadModelFilesDialog(props: { open: boolean; onClose const [progress, setProgress] = useState(); const handleDownloadStart = useCallback(() => { - window.api.ipcSend('analyze.download.start'); + window.api.ipcSend('setup.download.start'); setIsDownloadActive(true); }, []); const handleDownloadCancel = useCallback(() => { - window.api.ipcSend('analyze.download.cancel'); + window.api.ipcSend('setup.download.cancel'); }, []); const handleProgress = useCallback((event, progressInfo: DownloadProgress) => { @@ -48,13 +48,13 @@ export default function DownloadModelFilesDialog(props: { open: boolean; onClose }, []); useEffect(() => { - window.api.addIpcListener('analyze.download.progress', handleProgress); - window.api.addIpcListener('analyze.download.finished', handleComplete); - window.api.addIpcListener('analyze.download.error', handleError); + window.api.addIpcListener('setup.download.progress', handleProgress); + window.api.addIpcListener('setup.download.finished', handleComplete); + window.api.addIpcListener('setup.download.error', handleError); return function cleanup() { - window.api.removeIpcListener('analyze.download.progress', handleProgress); - window.api.removeIpcListener('analyze.download.finished', handleComplete); - window.api.removeIpcListener('analyze.download.error', handleError); + window.api.removeIpcListener('setup.download.progress', handleProgress); + window.api.removeIpcListener('setup.download.finished', handleComplete); + window.api.removeIpcListener('setup.download.error', handleError); }; }, [handleComplete, handleError, handleProgress]); diff --git a/src/routes/Analyze.tsx b/src/routes/Analyze.tsx index c599302..de0a1c1 100644 --- a/src/routes/Analyze.tsx +++ b/src/routes/Analyze.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useState } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import Box from '@mui/material/Box'; import Button from '@mui/material/Button'; import Grid from '@mui/material/Grid'; @@ -40,6 +40,18 @@ export default function Analyze() { delete files.subtitle; setFiles(files); }, [files]); + const handleQueueUpdate = useCallback((event, queueData) => { + console.log('queueUpdate', queueData); + }, []); + + useEffect(() => { + window.api.addIpcListener('queue.update', handleQueueUpdate); + window.api.ipcSend('queue.get'); + return function cleanup() { + window.api.addIpcListener('queue.update', handleQueueUpdate); + }; + }, [handleQueueUpdate]); + return ( diff --git a/webpack/main.webpack.js b/webpack/main.webpack.js index 3c31b1a..1ca1ba8 100644 --- a/webpack/main.webpack.js +++ b/webpack/main.webpack.js @@ -2,6 +2,10 @@ module.exports = { resolve: { extensions: ['.ts', '.js'] }, + externals: { + deepspeech: 'commonjs2 deepspeech', + 'ffmpeg-static': 'commonjs2 ffmpeg-static' + }, entry: './core/main.ts', module: { rules: require('./rules.webpack') diff --git a/webpack/renderer.webpack.js b/webpack/renderer.webpack.js index 83157f6..b82fbad 100644 --- a/webpack/renderer.webpack.js +++ b/webpack/renderer.webpack.js @@ -1,8 +1,22 @@ +const webpack = require('webpack'); +const dotenv = require('dotenv'); + +const reactConfig = Object.entries(dotenv.config().parsed) + .filter(([key, value]) => /^REACT_/.test(key)) + .reduce((config, [key, value]) => { + return { ...config, [key]: value }; + }, {}); + module.exports = { resolve: { extensions: ['.ts', '.tsx', '.js'] }, module: { rules: require('./rules.webpack') - } + }, + plugins: [ + new webpack.DefinePlugin({ + 'process.env': JSON.stringify(reactConfig) // it will automatically pick up key values from .env file + }) + ] };