Add Kotlin and Java API for online punctuation models (#1936)

csukuangfj · web-flow · commit f5dfcf8d2f99 · 2025-02-27T16:52:36.000+08:00
diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml
@@ -105,6 +105,22 @@ jobs:
             make -j4
             ls -lh lib
 
+      - name:  Run java test (Online add punctuations)
+        shell: bash
+        run: |
+          cd ./java-api-examples
+          ./run-online-add-punctuation-zh-en.sh
+          # Delete model files to save space
+          rm -rf sherpa-onnx-online-*
+
+      - name:  Run java test (Offline add punctuations)
+        shell: bash
+        run: |
+          cd ./java-api-examples
+          ./run-offline-add-punctuation-zh-en.sh
+          # Delete model files to save space
+          rm -rf sherpa-onnx-punct-*
+
       - name:  Run java test (Non-Streaming ASR)
         shell: bash
         run: |
@@ -196,13 +212,6 @@ jobs:
           ./run-audio-tagging-ced-from-file.sh
           rm -rf sherpa-onnx-ced-*
 
-      - name:  Run java test (add punctuations)
-        shell: bash
-        run: |
-          cd ./java-api-examples
-          ./run-add-punctuation-zh-en.sh
-          # Delete model files to save space
-          rm -rf sherpa-onnx-punct-*
 
       - name:  Run java test (Spoken language identification)
         shell: bash
diff --git a/java-api-examples/OfflineAddPunctuation.java b/java-api-examples/OfflineAddPunctuation.java
@@ -5,7 +5,7 @@
 // The model supports both English and Chinese.
 import com.k2fsa.sherpa.onnx.*;
 
-public class AddPunctuation {
+public class OfflineAddPunctuation {
   public static void main(String[] args) {
     // please download the model from
     // https://github.com/k2-fsa/sherpa-onnx/releases/tag/punctuation-models
diff --git a/java-api-examples/OnlineAddPunctuation.java b/java-api-examples/OnlineAddPunctuation.java
@@ -0,0 +1,41 @@
+// Copyright 2025 Xiaomi Corporation
+
+// This file shows how to use a punctuation model to add punctuations to text.
+//
+// The model supports ONLY English.
+import com.k2fsa.sherpa.onnx.*;
+
+public class OnlineAddPunctuation {
+  public static void main(String[] args) {
+    // please download the model from
+    // https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+    String model = "./sherpa-onnx-online-punct-en-2024-08-06/model.int8.onnx";
+    String bpeVocab = "./sherpa-onnx-online-punct-en-2024-08-06/bpe.vocab";
+    OnlinePunctuationModelConfig modelConfig =
+        OnlinePunctuationModelConfig.builder()
+            .setCnnBilstm(model)
+            .setBpeVocab(bpeVocab)
+            .setNumThreads(1)
+            .setDebug(true)
+            .build();
+    OnlinePunctuationConfig config =
+        OnlinePunctuationConfig.builder().setModel(modelConfig).build();
+
+    OnlinePunctuation punct = new OnlinePunctuation(config);
+
+    String[] sentences =
+        new String[] {
+          "how are you doing fantastic thank you how about you",
+          "The African blogosphere is rapidly expanding bringing more voices online in the form of"
+              + " commentaries opinions analyses rants and poetry",
+        };
+
+    System.out.println("---");
+    for (String text : sentences) {
+      String out = punct.addPunctuation(text);
+      System.out.printf("Input: %s\n", text);
+      System.out.printf("Output: %s\n", out);
+      System.out.println("---");
+    }
+  }
+}
diff --git a/java-api-examples/run-offline-add-punctuation-zh-en.sh b/java-api-examples/run-offline-add-punctuation-zh-en.sh
@@ -34,4 +34,4 @@ fi
 java \
   -Djava.library.path=$PWD/../build/lib \
   -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \
-  ./AddPunctuation.java
+  ./OfflineAddPunctuation.java
diff --git a/java-api-examples/run-online-add-punctuation-zh-en.sh b/java-api-examples/run-online-add-punctuation-zh-en.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib  && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then
+  mkdir -p ../build
+  pushd ../build
+  cmake \
+    -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
+    -DSHERPA_ONNX_ENABLE_TESTS=OFF \
+    -DSHERPA_ONNX_ENABLE_CHECK=OFF \
+    -DBUILD_SHARED_LIBS=ON \
+    -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
+    -DSHERPA_ONNX_ENABLE_JNI=ON \
+    ..
+
+  make -j4
+  ls -lh lib
+  popd
+fi
+
+if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then
+  pushd ../sherpa-onnx/java-api
+  make
+  popd
+fi
+
+if [ ! -f ./sherpa-onnx-online-punct-en-2024-08-06/model.int8.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+  tar xvf sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+  rm sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+fi
+
+java \
+  -Djava.library.path=$PWD/../build/lib \
+  -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \
+  ./OnlineAddPunctuation.java
diff --git a/kotlin-api-examples/OnlinePunctuation.kt b/kotlin-api-examples/OnlinePunctuation.kt
@@ -0,0 +1 @@
+../sherpa-onnx/kotlin-api/OnlinePunctuation.kt
diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh
@@ -302,16 +302,16 @@ function testInverseTextNormalizationOnlineAsr() {
   java -Djava.library.path=../build/lib -jar $out_filename
 }
 
-function testPunctuation() {
+function testOfflinePunctuation() {
   if [ ! -f ./sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12/model.onnx ]; then
     curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
     tar xvf sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
     rm sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
   fi
 
-  out_filename=test_punctuation.jar
+  out_filename=test_offline_punctuation.jar
   kotlinc-jvm -include-runtime -d $out_filename \
-    ./test_punctuation.kt \
+    ./test_offline_punctuation.kt \
     ./OfflinePunctuation.kt \
     faked-asset-manager.kt \
     faked-log.kt
@@ -321,6 +321,25 @@ function testPunctuation() {
   java -Djava.library.path=../build/lib -jar $out_filename
 }
 
+function testOnlinePunctuation() {
+  if [ ! -f ./sherpa-onnx-online-punct-en-2024-08-06/model.int8.onnx ]; then
+    curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+    tar xvf sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+    rm sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+  fi
+
+  out_filename=test_online_punctuation.jar
+  kotlinc-jvm -include-runtime -d $out_filename \
+    ./test_online_punctuation.kt \
+    ./OnlinePunctuation.kt \
+    faked-asset-manager.kt \
+    faked-log.kt
+
+  ls -lh $out_filename
+
+  java -Djava.library.path=../build/lib -jar $out_filename
+}
+
 function testOfflineSpeakerDiarization() {
   if [ ! -f ./sherpa-onnx-pyannote-segmentation-3-0/model.onnx ]; then
     curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2
@@ -359,6 +378,7 @@ testTts
 testAudioTagging
 testSpokenLanguageIdentification
 testOfflineAsr
-testPunctuation
+testOfflinePunctuation
+testOnlinePunctuation
 testInverseTextNormalizationOfflineAsr
 testInverseTextNormalizationOnlineAsr
diff --git a/kotlin-api-examples/test_offline_punctuation.kt b/kotlin-api-examples/test_offline_punctuation.kt
diff --git a/kotlin-api-examples/test_online_punctuation.kt b/kotlin-api-examples/test_online_punctuation.kt
@@ -0,0 +1,30 @@
+package com.k2fsa.sherpa.onnx
+
+fun main() {
+  testPunctuation()
+}
+
+// https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+fun testPunctuation() {
+  val config = OnlinePunctuationConfig(
+      model=OnlinePunctuationModelConfig(
+          cnnBilstm="./sherpa-onnx-online-punct-en-2024-08-06/model.int8.onnx",
+          bpeVocab="./sherpa-onnx-online-punct-en-2024-08-06/bpe.vocab",
+          numThreads=1,
+          debug=true,
+          provider="cpu",
+      )
+  )
+  val punct = OnlinePunctuation(config = config)
+  val sentences = arrayOf(
+        "how are you doing fantastic thank you what is about you",
+        "The African blogosphere is rapidly expanding bringing more voices online in the form of commentaries opinions analyses rants and poetry",
+  )
+  println("---")
+  for (text in sentences) {
+    val out = punct.addPunctuation(text)
+    println("Input: $text")
+    println("Output: $out")
+    println("---")
+  }
+}
diff --git a/sherpa-onnx/java-api/Makefile b/sherpa-onnx/java-api/Makefile
@@ -53,6 +53,10 @@ java_files += OfflinePunctuationModelConfig.java
 java_files += OfflinePunctuationConfig.java
 java_files += OfflinePunctuation.java
 
+java_files += OnlinePunctuationModelConfig.java
+java_files += OnlinePunctuationConfig.java
+java_files += OnlinePunctuation.java
+
 java_files += OfflineZipformerAudioTaggingModelConfig.java
 java_files += AudioTaggingModelConfig.java
 java_files += AudioTaggingConfig.java
diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlinePunctuation.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlinePunctuation.java
@@ -0,0 +1,39 @@
+// Copyright 2024 Xiaomi Corporation
+
+package com.k2fsa.sherpa.onnx;
+
+public class OnlinePunctuation {
+    static {
+        System.loadLibrary("sherpa-onnx-jni");
+    }
+
+    private long ptr = 0;
+
+    public OnlinePunctuation(OnlinePunctuationConfig config) {
+        ptr = newFromFile(config);
+    }
+
+    public String addPunctuation(String text) {
+        return addPunctuation(ptr, text);
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+        release();
+    }
+
+    // You'd better call it manually if it is not used anymore
+    public void release() {
+        if (this.ptr == 0) {
+            return;
+        }
+        delete(this.ptr);
+        this.ptr = 0;
+    }
+
+    private native void delete(long ptr);
+
+    private native long newFromFile(OnlinePunctuationConfig config);
+
+    private native String addPunctuation(long ptr, String text);
+}
diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlinePunctuationConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlinePunctuationConfig.java
@@ -0,0 +1,33 @@
+// Copyright 2024 Xiaomi Corporation
+
+package com.k2fsa.sherpa.onnx;
+
+public class OnlinePunctuationConfig {
+    private final OnlinePunctuationModelConfig model;
+
+    private OnlinePunctuationConfig(Builder builder) {
+        this.model = builder.model;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public OnlinePunctuationModelConfig getModel() {
+        return model;
+    }
+
+
+    public static class Builder {
+        private OnlinePunctuationModelConfig model = OnlinePunctuationModelConfig.builder().build();
+
+        public OnlinePunctuationConfig build() {
+            return new OnlinePunctuationConfig(this);
+        }
+
+        public Builder setModel(OnlinePunctuationModelConfig model) {
+            this.model = model;
+            return this;
+        }
+    }
+}
diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlinePunctuationModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlinePunctuationModelConfig.java
@@ -0,0 +1,68 @@
+// Copyright 2024 Xiaomi Corporation
+
+package com.k2fsa.sherpa.onnx;
+
+public class OnlinePunctuationModelConfig {
+    private final String cnnBilstm;
+    private final String bpeVocab;
+    private final int numThreads;
+    private final boolean debug;
+    private final String provider;
+
+    private OnlinePunctuationModelConfig(Builder builder) {
+        this.cnnBilstm = builder.cnnBilstm;
+        this.bpeVocab = builder.bpeVocab;
+        this.numThreads = builder.numThreads;
+        this.debug = builder.debug;
+        this.provider = builder.provider;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public String getCnnBilstm() {
+        return cnnBilstm;
+    }
+
+    public String getBpeVocab() {
+        return bpeVocab;
+    }
+
+    public static class Builder {
+        private String cnnBilstm = "";
+        private String bpeVocab = "";
+        private int numThreads = 1;
+        private boolean debug = true;
+        private String provider = "cpu";
+
+        public OnlinePunctuationModelConfig build() {
+            return new OnlinePunctuationModelConfig(this);
+        }
+
+        public Builder setCnnBilstm(String cnnBilstm) {
+            this.cnnBilstm = cnnBilstm;
+            return this;
+        }
+
+        public Builder setBpeVocab(String bpeVocab) {
+            this.bpeVocab = bpeVocab;
+            return this;
+        }
+
+        public Builder setNumThreads(int numThreads) {
+            this.numThreads = numThreads;
+            return this;
+        }
+
+        public Builder setDebug(boolean debug) {
+            this.debug = debug;
+            return this;
+        }
+
+        public Builder setProvider(String provider) {
+            this.provider = provider;
+            return this;
+        }
+    }
+}
diff --git a/sherpa-onnx/jni/CMakeLists.txt b/sherpa-onnx/jni/CMakeLists.txt
@@ -17,6 +17,7 @@ set(sources
   offline-punctuation.cc
   offline-recognizer.cc
   offline-stream.cc
+  online-punctuation.cc
   online-recognizer.cc
   online-stream.cc
   speaker-embedding-extractor.cc
diff --git a/sherpa-onnx/jni/online-punctuation.cc b/sherpa-onnx/jni/online-punctuation.cc
diff --git a/sherpa-onnx/kotlin-api/OnlinePunctuation.kt b/sherpa-onnx/kotlin-api/OnlinePunctuation.kt

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+../sherpa-onnx/kotlin-api/OnlinePunctuation.kt`