upgrade sherpa-onnx

Author: Sg4Dylan

CMakeLists.txt

@@ -1,8 +1,9 @@
 cmake_minimum_required(VERSION 3.28)
-set(VCPKG_TARGET_TRIPLET x64-windows)
+set(VCPKG_TARGET_TRIPLET x64-windows-static)
 project(realtime-bilingual-asr)
 
 set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SUBSYSTEM:WINDOWS /ENTRY:mainCRTStartup")
 add_compile_options("$<$<C_COMPILER_ID:MSVC>:/utf-8>")
@@ -13,6 +14,7 @@ IF (NOT CMAKE_BUILD_TYPE STREQUAL Debug)
 ENDIF ()
 
 include_directories(${CMAKE_SOURCE_DIR}/wtfdanmaku/include)
+include_directories(${CMAKE_SOURCE_DIR}/sherpa-onnx/include)
 
 find_package(ixwebsocket REQUIRED)
 find_package(nlohmann_json REQUIRED)
@@ -33,8 +35,9 @@ target_link_libraries(${PROJECT_NAME}
         mfplat
         mf
         wmcodecdspuuid
-        "${CMAKE_SOURCE_DIR}/sherpa-onnx-c-api.lib"
         bcrypt
+        "${CMAKE_SOURCE_DIR}/sherpa-onnx/lib/sherpa-onnx-c-api.lib"
+        "${CMAKE_SOURCE_DIR}/sherpa-onnx/lib/sherpa-onnx-cxx-api.lib"
 )
 IF (NOT CMAKE_BUILD_TYPE STREQUAL Debug)
     target_link_libraries(${PROJECT_NAME}

README.md

@@ -26,6 +26,7 @@ JikkyoSubtitle: Real-time transcription and translation of system audio into Chi
 
 **Prerequisites:**
 
+0. **sherpa-onnx:** Obtain the pre-built shared libraries in the [Releases](https://github.com/k2-fsa/sherpa-onnx/releases/tag/v1.12.11) page.
 1. **Visual Studio with the "Desktop development with C++" workload:**  Ensure it includes the MSVC compiler and CMake.
 2. **vcpkg:** Installed and integrated with your system (you've likely already done this if you have a vcpkg.json). Make sure `VCPKG_ROOT` environment variable is set and `vcpkg integrate install` has been run.
 3. **Git:** For fetching the project (if applicable).

SpeechRecognition.cpp

@@ -33,23 +33,35 @@ void SpeechRecognition::init() {
     configFile >> config;
 
     // 初始化加载 ASR 模型
-    auto asr_model_path = config["asr"]["sense_voice"]["model_path"].get<std::string>();
-    auto asr_model_lang = config["asr"]["sense_voice"]["language"].get<std::string>();
-    SherpaOnnxOfflineSenseVoiceModelConfig sense_voice_config{
-        asr_model_path.c_str(),
-        asr_model_lang.c_str(),
-        config["asr"]["sense_voice"]["num_threads"].get<int>()
-    };
     // Offline model config
     SherpaOnnxOfflineModelConfig offline_model_config;
     memset(&offline_model_config, 0, sizeof(offline_model_config));
     offline_model_config.debug = 0;
     offline_model_config.num_threads = config["asr"]["num_threads"].get<int>();;
     auto asr_model_provider = config["asr"]["provider"].get<std::string>();
     offline_model_config.provider = asr_model_provider.c_str();
-    auto asr_model_token_path = config["asr"]["token_path"].get<std::string>();
-    offline_model_config.tokens = asr_model_token_path.c_str();
-    offline_model_config.sense_voice = sense_voice_config;
+    if (config["asr"].contains("sense_voice")) {
+        auto asr_model_path = config["asr"]["sense_voice"]["model_path"].get<std::string>();
+        auto asr_model_lang = config["asr"]["sense_voice"]["language"].get<std::string>();
+        auto asr_model_token_path = config["asr"]["sense_voice"]["token_path"].get<std::string>();
+        offline_model_config.tokens = asr_model_token_path.c_str();
+        SherpaOnnxOfflineSenseVoiceModelConfig sense_voice_config{
+            asr_model_path.c_str(),
+            asr_model_lang.c_str(),
+            config["asr"]["num_threads"].get<int>()
+        };
+        offline_model_config.sense_voice = sense_voice_config;
+    } else if (config["asr"].contains("dolphin")) {
+        auto asr_model_path = config["asr"]["dolphin"]["model_path"].get<std::string>();
+        auto asr_model_token_path = config["asr"]["dolphin"]["token_path"].get<std::string>();
+        offline_model_config.tokens = asr_model_token_path.c_str();
+        SherpaOnnxOfflineDolphinModelConfig dolphin_config{
+            asr_model_path.c_str(),
+        };
+        offline_model_config.dolphin = dolphin_config;
+    } else {
+        throw std::runtime_error("Failed to init config, unsupported ASR model type.");
+    }
 
     // Recognizer config
     SherpaOnnxOfflineRecognizerConfig recognizer_config;
@@ -58,46 +70,54 @@ void SpeechRecognition::init() {
     recognizer_config.model_config = offline_model_config;
 
     recognizer = SherpaOnnxCreateOfflineRecognizer(&recognizer_config);
-
     if (recognizer == nullptr) {
         throw std::runtime_error("Please check your recognizer config!\n");
     }
 
     // VAD Config
     SherpaOnnxVadModelConfig vadConfig;
     memset(&vadConfig, 0, sizeof(vadConfig));
-    auto vad_model_path = config["vad"]["silero_vad"]["model"].get<std::string>();
-    vadConfig.silero_vad.model = vad_model_path.c_str();
-    vadConfig.interrupt_threshold = config["vad"]["interrupt_threshold"].get<float>();;
-    vadConfig.silero_vad.threshold = config["vad"]["silero_vad"]["threshold"].get<float>();;
-    vadConfig.silero_vad.min_silence_duration = config["vad"]["silero_vad"]["min_silence_duration"].get<float>();
-    vadConfig.silero_vad.min_speech_duration = config["vad"]["silero_vad"]["min_speech_duration"].get<float>();;
-    vadConfig.silero_vad.max_speech_duration = config["vad"]["silero_vad"]["max_speech_duration"].get<float>();;
-    vadConfig.silero_vad.window_size = 512;
     vadConfig.sample_rate = modelSampleRate;
-    vadConfig.num_threads =
-    vadConfig.num_threads = config["vad"]["num_threads"].get<int>();;
+    vadConfig.num_threads = vadConfig.num_threads = config["vad"]["num_threads"].get<int>();;
     vadConfig.debug = 0;
     vadConfig.provider = "cpu";
+    if (config["vad"].contains("silero_vad")) {
+        auto vad_model_path = config["vad"]["silero_vad"]["model"].get<std::string>();
+        vadConfig.silero_vad.model = vad_model_path.c_str();
+        vadConfig.silero_vad.threshold = config["vad"]["silero_vad"]["threshold"].get<float>();;
+        vadConfig.silero_vad.min_silence_duration = config["vad"]["silero_vad"]["min_silence_duration"].get<float>();
+        vadConfig.silero_vad.min_speech_duration = config["vad"]["silero_vad"]["min_speech_duration"].get<float>();;
+        vadConfig.silero_vad.max_speech_duration = config["vad"]["silero_vad"]["max_speech_duration"].get<float>();;
+        vadConfig.silero_vad.window_size = 512;
+    } else if (config["vad"].contains("ten_vad")) {
+        auto vad_model_path = config["vad"]["ten_vad"]["model"].get<std::string>();
+        vadConfig.ten_vad.model = vad_model_path.c_str();
+        vadConfig.ten_vad.threshold = config["vad"]["ten_vad"]["threshold"].get<float>();;
+        vadConfig.ten_vad.min_silence_duration = config["vad"]["ten_vad"]["min_silence_duration"].get<float>();
+        vadConfig.ten_vad.min_speech_duration = config["vad"]["ten_vad"]["min_speech_duration"].get<float>();;
+        vadConfig.ten_vad.max_speech_duration = config["vad"]["ten_vad"]["max_speech_duration"].get<float>();;
+        vadConfig.ten_vad.window_size = 256;
+    } else {
+        throw std::runtime_error("Failed to init config, unsupported VAD model type.");
+    }
 
     vad = SherpaOnnxCreateVoiceActivityDetector(&vadConfig, 30);
-
     if (vad == nullptr) {
         SherpaOnnxDestroyOfflineRecognizer(recognizer);
         throw std::runtime_error("Please check your vad config!\n");
     }
     std::cout << "VAD & ASR model loaded" << std::endl;
 
     // LLM Params
-    promptTemplate = config["llm"]["prompt_template"].get<std::string>();
-    modelName = config["llm"]["model_name"].get<std::string>();
-    modelAuth = config["llm"]["auth_key"].get<std::string>();
-    llmServer = config["llm"]["api_base"].get<std::string>();
-    isLlamaCpp = config["llm"]["is_llama_cpp"].get<bool>();
-    isSakuraLLM = config["llm"]["is_sakura_llm"].get<bool>();
-    modelMaxTokens = config["llm"]["max_tokens"].get<int>();
-    modelTemperature = config["llm"]["temperature"].get<float>();
-    modelTopP = config["llm"]["top_p"].get<float>();
+    remoteLLMConfig.apiBaseUrl = config["llm"]["api_base"].get<std::string>();
+    remoteLLMConfig.apiToken = config["llm"]["api_token"].get<std::string>();
+    remoteLLMConfig.modelName = config["llm"]["model_name"].get<std::string>();
+    remoteLLMConfig.isSakuraLLM = config["llm"].value("is_sakura_llm", false);
+    remoteLLMConfig.promptTemplate = config["llm"]["prompt_template"].get<std::string>();
+    remoteLLMConfig.engineType = config["llm"].value("is_llama_cpp", false) ? LLaMA_CPP : StandardOpenAI;
+    remoteLLMConfig.samplingConfig.maxTokens = config["llm"].value("max_tokens", 512);
+    remoteLLMConfig.samplingConfig.temperature = config["llm"].value("temperature", 0.1);
+    remoteLLMConfig.samplingConfig.topP = config["llm"].value("topP", 0.3);
 
     // Init ASR Handler
     asrCallback = [this](short *input, int32_t n_samples,
@@ -124,9 +144,9 @@ void SpeechRecognition::init() {
 
             const SherpaOnnxOfflineStream *stream = SherpaOnnxCreateOfflineStream(recognizer);
 
-            SherpaOnnxAcceptWaveformOffline(stream, modelSampleRate, tail_paddings, 4800);
+            SherpaOnnxAcceptWaveformOffline(stream, modelSampleRate, tailPaddings, 4800);
             SherpaOnnxAcceptWaveformOffline(stream, modelSampleRate, segment->samples, segment->n);
-            SherpaOnnxAcceptWaveformOffline(stream, modelSampleRate, tail_paddings, 4800);
+            SherpaOnnxAcceptWaveformOffline(stream, modelSampleRate, tailPaddings, 4800);
 
             SherpaOnnxDecodeOfflineStream(recognizer, stream);
 
@@ -146,7 +166,6 @@ void SpeechRecognition::init() {
                 subtitles.emplace(text, result->lang);
             }
 
-
             SherpaOnnxDestroyOfflineRecognizerResult(result);
             SherpaOnnxDestroyOfflineStream(stream);
             SherpaOnnxDestroySpeechSegment(segment);
@@ -169,35 +188,35 @@ std::string SpeechRecognition::getTranslate(const std::string &text) {
         nonSpaceText.erase(std::ranges::remove_if(nonSpaceText,
                                                   [](unsigned char c) { return std::isspace(c); }).begin(),
                            nonSpaceText.end());
-        ss << llmServer << "/v1/completions";
+        ss << remoteLLMConfig.apiBaseUrl << "/v1/completions";
         const std::string url = ss.str();
         ix::HttpRequestArgsPtr args = httpClient.createRequest();
         ix::WebSocketHttpHeaders headers;
-        headers["Authorization"] = "Bearer " + modelAuth;
+        headers["Authorization"] = "Bearer " + remoteLLMConfig.apiToken;
         headers["content-type"] = "application/json";
         args->extraHeaders = headers;
         json payload = {
-            {"model", modelName},
-            {"max_tokens", modelMaxTokens},
-            {"temperature", modelTemperature},
-            {"top_p", modelTopP}
+            {"model", remoteLLMConfig.modelName},
+            {"max_tokens", remoteLLMConfig.samplingConfig.maxTokens},
+            {"temperature", remoteLLMConfig.samplingConfig.temperature},
+            {"top_p", remoteLLMConfig.samplingConfig.topP}
         };
 
         // Use promptTemplate and format it
-        std::string prompt = promptTemplate;
+        std::string prompt = remoteLLMConfig.promptTemplate;
         size_t pos = prompt.find("%TEXT%");
         if (pos != std::string::npos) {
             prompt.replace(pos, 6, nonSpaceText);
         }
-        if (isSakuraLLM) {
+        if (remoteLLMConfig.isSakuraLLM) {
             payload["stop"] = {"<|im_end|>", "<|im_start|>"};
         }
         payload["prompt"] = prompt;
 
         ix::HttpResponsePtr out = httpClient.post(url, payload.dump(), args);
         if (out->errorCode == ix::HttpErrorCode::Ok) {
             json llm_result = json::parse(out->body);
-            if (isLlamaCpp) {
+            if (remoteLLMConfig.engineType == LLaMA_CPP) {
                 return llm_result["content"];
             }
             return llm_result["choices"][0]["text"];

SpeechRecognition.h

@@ -13,7 +13,28 @@
 #include <thread>
 
 #include "AudioCapture.h"
-#include "c-api.h"
+#include "sherpa-onnx/c-api/c-api.h"
+
+enum LLMEngineType {
+    StandardOpenAI = 0,
+    LLaMA_CPP = 1
+};
+
+struct SamplingConfig {
+    int maxTokens;
+    float temperature;
+    float topP;
+};
+
+struct RemoteLLMConfig {
+    std::string apiToken;
+    std::string apiBaseUrl;
+    std::string modelName;
+    bool isSakuraLLM;
+    std::string promptTemplate;
+    LLMEngineType engineType = StandardOpenAI;
+    SamplingConfig samplingConfig;
+};
 
 class SpeechSubtitle {
 public:
@@ -62,40 +83,34 @@ class SpeechRecognition {
 private:
     void capture();
 
-    // Loads configuration from the JSON file
     void loadConfig();
 
     static bool initNetSystem();
 
-    ix::HttpClient httpClient;
-    AudioCapture audioCapture;
-
-    std::string configFilePath; // Path to the configuration file
+    // Global config / status
+    std::string configFilePath;
+    bool running = false;
+    std::mutex subtitlesMutex;
 
-    // Configuration members loaded from the JSON file
-    SherpaOnnxVadModelConfig vadConfig;
-    std::string promptTemplate;
-    std::string modelName;
-    std::string modelAuth;
-    int modelMaxTokens = 512;
-    float modelTemperature = 0.1;
-    float modelTopP = 0.3;
-    std::string llmServer;
-    bool isLlamaCpp = false;
-    bool isSakuraLLM = false;
+    // Audio capture pipeline
+    AudioCapture audioCapture;
+    std::thread captureThread;
 
+    // ASR pipeline
     int modelSampleRate = 16000;
-    float tail_paddings[4800] = {0.}; // 0.3 seconds at 16 kHz sample rate
-    const SherpaOnnxOfflineRecognizer *recognizer = nullptr;
+    float tailPaddings[4800] = {0.}; // 0.3 seconds at 16 kHz sample rate
     SherpaOnnxOfflineRecognizerConfig recognizer_config;
-    SherpaOnnxOfflineModelConfig offline_model_config;
-    SherpaOnnxVoiceActivityDetector *vad = nullptr;
+    const SherpaOnnxOfflineRecognizer *recognizer = nullptr;
+    SherpaOnnxVadModelConfig vadConfig;
+    const SherpaOnnxVoiceActivityDetector *vad = nullptr;
     std::function<void(short *, int32_t, int32_t)> asrCallback;
 
-    bool running = false;
+    // LLM based translate pipeline
+    ix::HttpClient httpClient;
+    RemoteLLMConfig remoteLLMConfig;
+
+    // Render queue
     std::queue<SpeechSubtitle> subtitles;
-    std::mutex subtitlesMutex;
-    std::thread captureThread;
 };
 
 #endif