Add support for JA in runtime, and add C API for more portability (#303)

* add jpn and C API

* simplify

* Update README.md

---------

Co-authored-by: qwaqrm <[email protected]>

Author: tempstudio

runtime/README.md

@@ -7,6 +7,12 @@ $ cmake -B build -DCMAKE_BUILD_TYPE=Release
 $ cmake --build build
 ```
 
+On Windows:
+``` bash
+$ cmake -DCMAKE_BUILD_TYPE=Release -B build -G "Visual Studio 17 2022" -DBUILD_SHARED_LIBS=0 -DCMAKE_CXX_FLAGS="/ZI"
+$ cmake --build build
+```
+
 2. How to use
 
 ``` bash

runtime/processor/CMakeLists.txt

@@ -11,3 +11,12 @@ else()
     target_link_libraries(wetext_processor PUBLIC dl fst wetext_utils)
   endif()
 endif()
+
+# ----------------------------------------------------------------------------
+# C API shared library (wetext_processor_c)
+# ----------------------------------------------------------------------------
+add_library(wetext_processor_c SHARED
+  wetext_processor_c_api.cc
+)
+
+target_link_libraries(wetext_processor_c PUBLIC wetext_processor)

runtime/processor/wetext_processor.cc

@@ -30,6 +30,8 @@ Processor::Processor(const std::string& tagger_path,
     parse_type_ = ParseType::kZH_ITN;
   } else if (tagger_path.find("en_tn_") != tagger_path.npos) {
     parse_type_ = ParseType::kEN_TN;
+  } else if (tagger_path.find("ja_tn_") != tagger_path.npos) {
+    parse_type_ = ParseType::kJA_TN;
   } else {
     LOG(FATAL) << "Invalid fst prefix, prefix should contain"
                << " either \"_tn_\" or \"_itn_\".";

runtime/processor/wetext_processor_c_api.cc

@@ -0,0 +1,64 @@
+#include "processor/wetext_processor_c_api.h"
+
+#include <cstring>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "processor/wetext_processor.h"
+
+using wetext::Processor;
+
+// Utility ------------------------------------------------------------------
+namespace {
+// Copies an std::string into a newly allocated C string that the caller must
+// free via wetext_free_string().
+const char* CopyToCString(const std::string& str) {
+  char* out = new char[str.size() + 1];
+  std::memcpy(out, str.c_str(), str.size() + 1);
+  return out;
+}
+}  // namespace
+
+// Public API ---------------------------------------------------------------
+
+WetextProcessorHandle wetext_create_processor(const char* tagger_path,
+                                             const char* verbalizer_path) {
+  if (!tagger_path || !verbalizer_path) {
+    return nullptr;
+  }
+  try {
+    Processor* proc = new Processor(tagger_path, verbalizer_path);
+    return static_cast<WetextProcessorHandle>(proc);
+  } catch (...) {
+    return nullptr;
+  }
+}
+
+void wetext_destroy_processor(WetextProcessorHandle handle) {
+  if (!handle) return;
+  Processor* proc = static_cast<Processor*>(handle);
+  delete proc;
+}
+
+#define WETEXT_RETURN_STRING(expr)                    \
+  if (!handle || !input) return nullptr;              \
+  Processor* proc = static_cast<Processor*>(handle);  \
+  std::string result = (expr);                        \
+  return CopyToCString(result);
+
+const char* wetext_tag(WetextProcessorHandle handle, const char* input) {
+  WETEXT_RETURN_STRING(proc->Tag(input));
+}
+
+const char* wetext_verbalize(WetextProcessorHandle handle, const char* input) {
+  WETEXT_RETURN_STRING(proc->Verbalize(input));
+}
+
+const char* wetext_normalize(WetextProcessorHandle handle, const char* input) {
+  WETEXT_RETURN_STRING(proc->Normalize(input));
+}
+
+void wetext_free_string(const char* str) {
+  delete[] str;
+}

runtime/processor/wetext_processor_c_api.h

@@ -0,0 +1,46 @@
+#ifndef WETEXT_PROCESSOR_C_API_H_
+#define WETEXT_PROCESSOR_C_API_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Symbol visibility
+#if defined(_WIN32) || defined(_WIN64)
+  // Symbols are auto-exported on Windows because CMake sets
+  // `CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON` for this target. We therefore
+  // leave `WETEXT_API` empty to avoid the usual dllexport/dllimport
+  // clutter while still allowing the same header to compile elsewhere.
+  #define WETEXT_API 
+#else
+  #define WETEXT_API __attribute__((visibility("default")))
+#endif
+
+// Opaque handle to the underlying wetext::Processor C++ object
+typedef void* WetextProcessorHandle;
+
+// Create / destroy ---------------------------------------------------------
+
+// Creates a new Processor instance. Returns nullptr on failure.
+WETEXT_API WetextProcessorHandle wetext_create_processor(const char* tagger_path,
+                                                        const char* verbalizer_path);
+
+// Destroys a Processor instance obtained via wetext_create_processor().
+WETEXT_API void wetext_destroy_processor(WetextProcessorHandle handle);
+
+// Processing APIs ----------------------------------------------------------
+
+// The returned C-string is heap allocated and must be released with
+// wetext_free_string() once you are done with it.
+WETEXT_API const char* wetext_tag(WetextProcessorHandle handle, const char* input);
+WETEXT_API const char* wetext_verbalize(WetextProcessorHandle handle, const char* input);
+WETEXT_API const char* wetext_normalize(WetextProcessorHandle handle, const char* input);
+
+// Frees a string returned by any of the processing APIs above.
+WETEXT_API void wetext_free_string(const char* str);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
+#endif  // WETEXT_PROCESSOR_C_API_H_

runtime/processor/wetext_token_parser.cc

@@ -32,6 +32,10 @@ const std::unordered_map<std::string, std::vector<std::string>> ZH_TN_ORDERS = {
     {"measure", {"denominator", "numerator", "value"}},
     {"money", {"value", "currency"}},
     {"time", {"noon", "hour", "minute", "second"}}};
+const std::unordered_map<std::string, std::vector<std::string>> JA_TN_ORDERS = {
+    {"date", {"year", "month", "day"}},
+    {"money", {"value", "currency"}}};
+
 const std::unordered_map<std::string, std::vector<std::string>> EN_TN_ORDERS = {
     {"date", {"preserve_order", "text", "day", "month", "year"}},
     {"money", {"integer_part", "fractional_part", "quantity", "currency_maj"}}};
@@ -49,6 +53,8 @@ TokenParser::TokenParser(ParseType type) {
     orders_ = ZH_ITN_ORDERS;
   } else if (type == ParseType::kEN_TN) {
     orders_ = EN_TN_ORDERS;
+  } else if (type == ParseType::kJA_TN) {
+    orders_ = JA_TN_ORDERS;
   } else {
     LOG(FATAL) << "Invalid order";
   }

runtime/processor/wetext_token_parser.h

@@ -64,7 +64,9 @@ struct Token {
 enum ParseType {
   kZH_TN = 0x00,   // Chinese Text Normalization
   kZH_ITN = 0x01,  // Chinese Inverse Text Normalization
-  kEN_TN = 0x02    // English Text Normalization
+  kEN_TN = 0x02,   // English Text Normalization
+  kEN_ITN = 0x03,  // English Inverse Text Normalization (Unsupported)
+  kJA_TN = 0x04    // Japanese Text Normalization
 };
 
 class TokenParser {