feat: local transcription of audio (#110)

* feat: free alternative for audio transcription

* docs: Updated README.md with info on local transcription

Author: Luisotee

.env.example

@@ -7,9 +7,17 @@ BING_COOKIES="MUID=08AC######6940; MUIDB=08AC#####6940; ...a very long string...
 # Determines whether the bot should detect and convert your voice messages into text
 # Accepted values are "true" or "false"
 TRANSCRIPTION_ENABLED="false"
-# This must be set for voice message transcription functionality
+# There are 2 ways to transcribe audio: using WhisperAI, which costs US$0.06 per 10 minutes of audio, or using WhisperAI locally.
+# Local transcription is slower and provides worse results, but it is free.
+# If you choose to use the local method, you need to install WhisperAI. Refer to the readme.md file for more information.
+TRANSCRIPTION_METHOD="local"  # options are 'local' or 'api'
+# TRANSCRIPTION_LANGUAGE improves the transcription results but is not required. If you only plan to send audio in one language,
+# it is recommended to specify the language. If you will use multiple languages, leave it blank.
+# It's only needed in local mode.
+TRANSCRIPTION_LANGUAGE=""  # "--language pt" (pt = Portuguese), list of languages: https://github.com/openai/whisper/blob/main/whisper/tokenizer.py (OPTIONAL)
+# This must be set if you are going to use the WhisperAI API for voice message transcription functionality.
 OPENAI_API_KEY="sk-90..."
-# Determines whether the bot should reply with the transcribed text from your voice messages
+# Determines whether the bot should reply with the transcribed text from your voice messages.
 # Accepted values are "true" or "false"
 REPLY_TRANSCRIPTION="false"
 

README.md

@@ -117,9 +117,11 @@ Sydney is designed to respond to natural language queries from users. You can as
 
 ### Voice Messages
 
-To utilize voice messages, make sure you have FFMPEG installed on your machine and an OpenAI API key. Please note that using the voice transcription API (Whisper AI) provided by OpenAI comes with a cost of US$0.06 per 10 minutes.
+To utilize voice messages, ensure that FFMPEG is installed on your machine. There are two alternatives for voice transcription: using the Whisper AI API (which incurs a cost of US$0.06 per 10 minutes of audio) or using the local method. The local method is slower and less precise but is free.
 
-Once you have installed FFMPEG and obtained your OpenAI API key, you will need to place the key in the `.env` file under the `OPENAI_API_KEY` variable. Additionally, set `TRANSCRIPTION_ENABLED` to `"TRUE"`. After restarting the bot, you can proceed to utilize it.
+1. WhisperAPI: When using the WhisperAPI, you only need to obtain an OpenAI API key. Place the key in the `.env` file under the `OPENAI_API_KEY` variable. Additionally, set `TRANSCRIPTION_ENABLED` to `"true"` and `TRANSCRIPTION_METHOD` to `"api"`. After restarting the bot, you can proceed to utilize it.
+
+2. Local mode: To use the local mode, you need to install Python and Whisper. Follow the setup instructions [here](https://github.com/openai/whisper#setup). Then, make some changes in the `.env` file. Change `TRANSCRIPTION_ENABLED` to `"true"` and `TRANSCRIPTION_METHOD` to `"local"`. After restarting the bot, you can proceed to utilize it.
 
 ### Group Chat
 

package.json

@@ -11,9 +11,9 @@
   "author": "Matheus Veiga <[email protected]>",
   "license": "MIT",
   "devDependencies": {
+    "@types/common-tags": "^1.8.1",
     "@types/fluent-ffmpeg": "^2.1.21",
     "@types/node": "^20.3.2",
-    "@types/common-tags": "^1.8.1",
     "@types/node-schedule": "^2.1.0",
     "@types/uuid": "^9.0.1",
     "nodemon": "^2.0.22",
@@ -26,11 +26,15 @@
     "@keyv/sqlite": "^3.6.5",
     "@types/qrcode-terminal": "^0.12.0",
     "@waylaidwanderer/chatgpt-api": "1.37.0",
+    "common-tags": "^1.8.2",
+    "crypto": "^1.0.1",
     "fluent-ffmpeg": "^2.1.2",
+    "fs": "^0.0.1-security",
     "keyv": "^4.5.2",
-    "openai": "^3.2.1",
-    "common-tags": "^1.8.2",
     "node-schedule": "^2.1.1",
+    "openai": "^3.2.1",
+    "os": "^0.1.2",
+    "path": "^0.12.7",
     "qrcode-terminal": "^0.12.0",
     "serialize-error": "^11.0.0",
     "uuid": "^9.0.0",

src/handlers/audio-transcription-local.ts

@@ -0,0 +1,67 @@
+import { execSync } from "child_process";
+import { randomUUID } from "crypto";
+import fs from "fs";
+import path from "path";
+import { convertOggToWav } from "./audio-transcription";
+
+export async function transcribeAudioLocal(
+  audioBuffer: Buffer
+): Promise<string> {
+  const folderPath = "./"; // Change this to specify the project folder
+  const filename = randomUUID();
+
+  const oggPath = path.join(folderPath, filename + ".ogg");
+  fs.writeFileSync(oggPath, audioBuffer);
+
+  const wavPath = path.join(folderPath, filename + ".wav");
+  await convertOggToWav(oggPath, wavPath);
+
+  // Deletes the ogg file as it is no longer needed
+  fs.unlinkSync(oggPath);
+
+  const output = execSync(
+    `whisper ${wavPath} ${process.env.TRANSCRIPTION_LANGUAGE}`,
+    {
+      encoding: "utf-8",
+    }
+  );
+
+  // Delete tmp file
+  fs.unlinkSync(wavPath);
+
+  // Delete whisper created tmp files
+  const extensions = [
+    ".srt",
+    ".txt",
+    ".vtt",
+    ".json",
+    ".srt.json",
+    ".tsv",
+    ".srt.efb",
+  ];
+
+  const parsedText = parseTextAfterTimeFrame(output);
+  for (const extension of extensions) {
+    const filePath = path.join(folderPath, `${filename}${extension}`);
+    if (fs.existsSync(filePath)) {
+      fs.unlinkSync(filePath);
+    }
+  }
+  if (parsedText) {
+    return parsedText;
+  } else {
+    return "[System] User tried to send an audio message, but the transcription failed. Please ask them to either write it in text or try again.";
+  }
+}
+
+function parseTextAfterTimeFrame(text: string) {
+  const textMatch = text.match(
+    /\[(\d{2}:\d{2}\.\d{3})\s-->\s(\d{2}:\d{2}\.\d{3})\]\s(.+)/
+  ); // Extract the text
+
+  if (textMatch) {
+    return textMatch[3].trim();
+  }
+
+  return null; // Return null if match is not found
+}

src/handlers/audio-transcription.ts

@@ -5,7 +5,7 @@ import os from "os";
 import path from "path";
 import { openai } from "../clients/openai";
 
-function convertOggToWav(oggPath: string, wavPath: string) {
+export function convertOggToWav(oggPath: string, wavPath: string) {
   return new Promise((resolve, reject) => {
     ffmpeg(oggPath)
       .toFormat("wav")

src/handlers/message.ts

@@ -10,6 +10,7 @@ import { counterRequests } from "./requests-counter";
 import { jsonSafeParse, react } from "../utils";
 import { scheduleReminder } from "./reminder";
 import { reminderSchema } from "../schemas/reminder";
+import { transcribeAudioLocal } from "./audio-transcription-local";
 
 function appendSources(sources: SourceAttribution[]) {
   let sourcesString = "\n\n";
@@ -84,9 +85,18 @@ async function handleAudioMessage(message: Message, media: MessageMedia) {
     const audioBuffer = Buffer.from(media.data, "base64");
 
     try {
-      const transcription = await transcribeAudio(audioBuffer);
+      let transcription;
+      if (process.env.TRANSCRIPTION_METHOD === "local") {
+        transcription = await transcribeAudioLocal(audioBuffer);
+      } else if (process.env.TRANSCRIPTION_METHOD === "api") {
+        transcription = await transcribeAudio(audioBuffer);
+      } else {
+        await message.reply(
+          "There was a problem in the transcription of the message, the problem is related to the TRANSCRIPTION_METHOD in the .env file."
+        );
+        return;
+      }
       message.body = transcription;
-
       if (process.env.REPLY_TRANSCRIPTION === "true")
         await message.reply(`Transcription:\n\n${transcription}`);
 

yarn.lock

@@ -828,6 +828,11 @@ cross-spawn@^7.0.3:
     shebang-command "^2.0.0"
     which "^2.0.1"
 
+crypto@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/crypto/-/crypto-1.0.1.tgz#2af1b7cad8175d24c8a1b0778255794a21803037"
+  integrity sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==
+
 debug@4, [email protected], debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.3, debug@^4.3.4:
   version "4.3.4"
   resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz"
@@ -1171,6 +1176,11 @@ fs.realpath@^1.0.0:
   resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz"
   integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
 
+fs@^0.0.1-security:
+  version "0.0.1-security"
+  resolved "https://registry.yarnpkg.com/fs/-/fs-0.0.1-security.tgz#8a7bd37186b6dddf3813f23858b57ecaaf5e41d4"
+  integrity sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==
+
 fsevents@~2.3.2:
   version "2.3.2"
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
@@ -1365,6 +1375,11 @@ inherits@2, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.3:
   resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
 
+[email protected]:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
+  integrity sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==
+
 inquirer-autocomplete-prompt@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/inquirer-autocomplete-prompt/-/inquirer-autocomplete-prompt-3.0.0.tgz#b00478882feb326b34e0754a1695d912e387a63b"
@@ -2016,6 +2031,11 @@ os-tmpdir@~1.0.2:
   resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
   integrity sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==
 
+os@^0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/os/-/os-0.1.2.tgz#f29a50c62908516ba42652de42f7038600cadbc2"
+  integrity sha512-ZoXJkvAnljwvc56MbvhtKVWmSkzV712k42Is2mA0+0KTSRakq5XXuXpjZjgAt9ctzl51ojhQWakQQpmOvXWfjQ==
+
 p-defer@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/p-defer/-/p-defer-3.0.0.tgz#d1dceb4ee9b2b604b1d94ffec83760175d4e6f83"
@@ -2070,6 +2090,14 @@ path-key@^3.0.0, path-key@^3.1.0:
   resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"
   integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==
 
+path@^0.12.7:
+  version "0.12.7"
+  resolved "https://registry.yarnpkg.com/path/-/path-0.12.7.tgz#d4dc2a506c4ce2197eb481ebfcd5b36c0140b10f"
+  integrity sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==
+  dependencies:
+    process "^0.11.1"
+    util "^0.10.3"
+
 pend@~1.2.0:
   version "1.2.0"
   resolved "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz"
@@ -2137,7 +2165,7 @@ process-warning@^2.0.0, process-warning@^2.2.0:
   resolved "https://registry.yarnpkg.com/process-warning/-/process-warning-2.2.0.tgz#008ec76b579820a8e5c35d81960525ca64feb626"
   integrity sha512-/1WZ8+VQjR6avWOgHeEPd7SDQmFQ1B5mC1eRXsCm5TarlNmx/wCsa5GEaxGm05BORRtyG/Ex/3xq3TuRvq57qg==
 
-process@^0.11.10:
+process@^0.11.1, process@^0.11.10:
   version "0.11.10"
   resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
   integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==
@@ -2801,6 +2829,13 @@ util-deprecate@^1.0.1, util-deprecate@~1.0.1:
   resolved "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz"
   integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==
 
+util@^0.10.3:
+  version "0.10.4"
+  resolved "https://registry.yarnpkg.com/util/-/util-0.10.4.tgz#3aa0125bfe668a4672de58857d3ace27ecb76901"
+  integrity sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==
+  dependencies:
+    inherits "2.0.3"
+
 uuid@^9.0.0:
   version "9.0.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5"