bindings.java : enable copyLibs task [no ci] (#2949)

* bindings.java : enable copyLibs task [no ci]

This commit adds a dependency on the copyLibs task to the sourcesJar and
jar tasks. This ensures that the libwhisper.so file is copied to the
correct location before the jar is built.

It also sets the executable bit on the gradlew file.

* bindings.java : add copyLibs dep for processResources [no ci]

This will otherwise cause builds to fail after doing an initial build.

* bindings.java : pass structs by value to native code

This commit refactors the code to pass the structs by value to the
native code. This is done by creating a ByValue class for each struct
and using it in the Java code.

The motivation for this change is that without this application crashes
due to what I believe was memory mis-alignement. When the structs were
passed to the native code they would be att different memory locations.
Passing by value overcomes this issue and considering that the structs
hold parementers (context and full params) it might be alright do to
this. These changes allow all the tests to pass.

* bindings.java : fix javadoc warnings [no ci]

* bindings.java : fix libwhisper.dylib path in build.gradle [no ci]

This commit fixes the copyLibwhisperDynlib task in the build.gradle file
to copy the correct libwhisper.dylib file from build/src.

Author: danbev

bindings/java/build.gradle

@@ -25,13 +25,13 @@ sourceSets {
 }
 
 tasks.register('copyLibwhisperDynlib', Copy) {
-    from '../../build'
-    include 'libwhisper.dynlib'
+    from '../../build/src'
+    include 'libwhisper.dylib'
     into 'build/generated/resources/main/darwin'
 }
 
 tasks.register('copyLibwhisperSo', Copy) {
-    from '../../build'
+    from '../../build/src'
     include 'libwhisper.so'
     into 'build/generated/resources/main/linux-x86-64'
 }
@@ -55,7 +55,12 @@ java {
     withJavadocJar()
 }
 
+sourcesJar() {
+    dependsOn copyLibs
+}
+
 jar {
+    dependsOn copyLibs
     exclude '**/whisper_java.exp', '**/whisper_java.lib'
 }
 
@@ -67,6 +72,9 @@ tasks.withType(Test) {
     useJUnitPlatform()
 }
 
+test.dependsOn copyLibs
+processResources.dependsOn copyLibs
+
 dependencies {
     implementation "net.java.dev.jna:jna:5.13.0"
     testImplementation "org.junit.jupiter:junit-jupiter:5.9.2"

bindings/java/gradlew

@@ -0,0 +1,24 @@
+package io.github.ggerganov.whispercpp;
+
+/**
+ * Presets for alignment heads in DTW token timestamps
+ */
+public class WhisperConstants {
+    // Alignment heads presets
+    public static final int WHISPER_AHEADS_NONE = 0;
+    public static final int WHISPER_AHEADS_TINY_EN = 1;
+    public static final int WHISPER_AHEADS_TINY = 2;
+    public static final int WHISPER_AHEADS_BASE_EN = 3;
+    public static final int WHISPER_AHEADS_BASE = 4;
+    public static final int WHISPER_AHEADS_SMALL_EN = 5;
+    public static final int WHISPER_AHEADS_SMALL = 6;
+    public static final int WHISPER_AHEADS_MEDIUM_EN = 7;
+    public static final int WHISPER_AHEADS_MEDIUM = 8;
+    public static final int WHISPER_AHEADS_LARGE_V1 = 9;
+    public static final int WHISPER_AHEADS_LARGE_V2 = 10;
+    public static final int WHISPER_AHEADS_LARGE_V3 = 11;
+    public static final int WHISPER_AHEADS_LARGE_V3_TURBO = 12;
+    public static final int WHISPER_AHEADS_CUSTOM = 13;
+    public static final int WHISPER_AHEADS_N_TOP_MOST = 14;
+    public static final int WHISPER_AHEADS_COUNT = 15;
+}

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperConstants.java

@@ -1,41 +1,36 @@
 package io.github.ggerganov.whispercpp;
 
+import com.sun.jna.NativeLong;
 import com.sun.jna.Structure;
 import com.sun.jna.ptr.PointerByReference;
+import com.sun.jna.Pointer;
 import io.github.ggerganov.whispercpp.ggml.GgmlType;
 import io.github.ggerganov.whispercpp.WhisperModel;
 import io.github.ggerganov.whispercpp.params.WhisperContextParams;
 
 import java.util.List;
 
 public class WhisperContext extends Structure {
-    int t_load_us = 0;
-    int t_start_us = 0;
+    public NativeLong t_load_us;
+    public NativeLong t_start_us;
 
     /** weight type (FP32 / FP16 / QX) */
-    GgmlType wtype = GgmlType.GGML_TYPE_F16;
+    public GgmlType wtype = GgmlType.GGML_TYPE_F16;
     /** intermediate type (FP32 or FP16) */
-    GgmlType itype = GgmlType.GGML_TYPE_F16;
+    public GgmlType itype = GgmlType.GGML_TYPE_F16;
 
-//    WhisperModel model;
-    public PointerByReference model;
-//    whisper_vocab vocab;
-//    whisper_state * state = nullptr;
-    public PointerByReference vocab;
-    public PointerByReference state;
+    public WhisperContextParams.ByValue params;
+
+    public Pointer model;
+    public Pointer vocab;
+    public Pointer state;
 
     /** populated by whisper_init_from_file_with_params() */
-    String path_model;
-    WhisperContextParams params;
-
-//    public static class ByReference extends WhisperContext implements Structure.ByReference {
-//    }
-//
-//    public static class ByValue extends WhisperContext implements Structure.ByValue {
-//    }
-//
-//    @Override
-//    protected List<String> getFieldOrder() {
-//        return List.of("t_load_us", "t_start_us", "wtype", "itype", "model", "vocab", "state", "path_model");
-//    }
+    public Pointer path_model;
+
+    @Override
+    protected List<String> getFieldOrder() {
+        return List.of("t_load_us", "t_start_us", "wtype", "itype",
+                "params", "model", "vocab", "state", "path_model");
+    }
 }

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperContext.java

@@ -43,11 +43,11 @@ public void initContext(String modelPath) throws FileNotFoundException {
      * @param modelPath - absolute path, or just the name (eg: "base", "base-en" or "base.en")
      * @param params - params to use when initialising the context
      */
-    public void initContext(String modelPath, WhisperContextParams params) throws FileNotFoundException {
+    public void initContext(String modelPath, WhisperContextParams.ByValue params) throws FileNotFoundException {
         initContextImpl(modelPath, params);
     }
 
-    private void initContextImpl(String modelPath, WhisperContextParams params) throws FileNotFoundException {
+    private void initContextImpl(String modelPath, WhisperContextParams.ByValue params) throws FileNotFoundException {
         if (ctx != null) {
             lib.whisper_free(ctx);
         }
@@ -69,15 +69,13 @@ private void initContextImpl(String modelPath, WhisperContextParams params) thro
 
     /**
      * Provides default params which can be used with `whisper_init_from_file_with_params()` etc.
-     * Because this function allocates memory for the params, the caller must call either:
-     * - call `whisper_free_context_params()`
-     * - `Native.free(Pointer.nativeValue(pointer));`
+     * Returns a ByValue instance to ensure proper parameter passing to native code.
      */
-    public WhisperContextParams getContextDefaultParams() {
-        paramsPointer = lib.whisper_context_default_params_by_ref();
-        WhisperContextParams params = new WhisperContextParams(paramsPointer);
-        params.read();
-        return params;
+    public WhisperContextParams.ByValue getContextDefaultParams() {
+        WhisperContextParams.ByValue valueParams = new WhisperContextParams.ByValue(
+            lib.whisper_context_default_params_by_ref());
+        valueParams.read();
+        return valueParams;
     }
 
     /**
@@ -88,7 +86,7 @@ public WhisperContextParams getContextDefaultParams() {
      *
      * @param strategy - GREEDY
      */
-    public WhisperFullParams getFullDefaultParams(WhisperSamplingStrategy strategy) {
+    public WhisperFullParams.ByValue getFullDefaultParams(WhisperSamplingStrategy strategy) {
         Pointer pointer;
 
         // whisper_full_default_params_by_ref allocates memory which we need to delete, so only create max 1 pointer for each strategy.
@@ -104,7 +102,7 @@ public WhisperFullParams getFullDefaultParams(WhisperSamplingStrategy strategy)
             pointer = beamParamsPointer;
         }
 
-        WhisperFullParams params = new WhisperFullParams(pointer);
+        WhisperFullParams.ByValue params = new WhisperFullParams.ByValue(pointer);
         params.read();
         return params;
     }
@@ -138,15 +136,21 @@ private void freeParams() {
     }
 
     /**
-     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text.
+     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text.
      * Not thread safe for same context
      * Uses the specified decoding strategy to obtain the text.
      */
-    public String fullTranscribe(WhisperFullParams whisperParams, float[] audioData) throws IOException {
+    public String fullTranscribe(WhisperFullParams.ByValue whisperParams, float[] audioData) throws IOException {
         if (ctx == null) {
             throw new IllegalStateException("Model not initialised");
         }
 
+        /*
+        WhisperFullParams.ByValue valueParams = new WhisperFullParams.ByValue(
+            lib.whisper_full_default_params_by_ref(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH.ordinal()));
+        valueParams.read();
+        */
+
         if (lib.whisper_full(ctx, whisperParams, audioData, audioData.length) != 0) {
             throw new IOException("Failed to process audio");
         }
@@ -163,12 +167,17 @@ public String fullTranscribe(WhisperFullParams whisperParams, float[] audioData)
 
         return str.toString().trim();
     }
+
     public List<WhisperSegment> fullTranscribeWithTime(WhisperFullParams whisperParams, float[] audioData) throws IOException {
         if (ctx == null) {
             throw new IllegalStateException("Model not initialised");
         }
 
-        if (lib.whisper_full(ctx, whisperParams, audioData, audioData.length) != 0) {
+        WhisperFullParams.ByValue valueParams = new WhisperFullParams.ByValue(
+            lib.whisper_full_default_params_by_ref(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH.ordinal()));
+        valueParams.read();
+
+        if (lib.whisper_full(ctx, valueParams, audioData, audioData.length) != 0) {
             throw new IOException("Failed to process audio");
         }
 

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperCpp.java

@@ -38,7 +38,7 @@ public interface WhisperCppJnaLibrary extends Library {
      * @param params     Pointer to whisper_context_params
      * @return Whisper context on success, null on failure
      */
-    Pointer whisper_init_from_file_with_params(String path_model, WhisperContextParams params);
+    Pointer whisper_init_from_file_with_params(String path_model, WhisperContextParams.ByValue params);
 
     /**
      * Allocate (almost) all memory needed for the model by loading from a buffer.
@@ -180,12 +180,12 @@ public interface WhisperCppJnaLibrary extends Library {
     /**
      * @return the id of the specified language, returns -1 if not found.
      * Examples:
-     *   "de" -> 2
-     *   "german" -> 2
+     *   "de" -> 2
+     *   "german" -> 2
      */
     int whisper_lang_id(String lang);
 
-    /** @return the short string of the specified language id (e.g. 2 -> "de"), returns nullptr if not found */
+    /** @return the short string of the specified language id (e.g. 2 -> "de"), returns nullptr if not found */
     String whisper_lang_str(int id);
 
     /**
@@ -268,20 +268,21 @@ public interface WhisperCppJnaLibrary extends Library {
     void whisper_free_params(Pointer params);
 
     /**
-     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text
+     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text
      * Not thread safe for same context
      * Uses the specified decoding strategy to obtain the text.
      */
-    int whisper_full(Pointer ctx, WhisperFullParams params, final float[] samples, int n_samples);
+    int whisper_full(Pointer ctx, WhisperFullParams.ByValue params, final float[] samples, int n_samples);
 
-    int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams params, final float[] samples, int n_samples);
+    public int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams.ByValue params, float[] samples, int n_samples);
+    //int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams params, final float[] samples, int n_samples);
 
     // Split the input audio in chunks and process each chunk separately using whisper_full_with_state()
     // Result is stored in the default state of the context
     // Not thread safe if executed in parallel on the same context.
     // It seems this approach can offer some speedup in some cases.
     // However, the transcription accuracy can be worse at the beginning and end of each chunk.
-    int whisper_full_parallel(Pointer ctx, WhisperFullParams params, final float[] samples, int n_samples, int n_processors);
+    int whisper_full_parallel(Pointer ctx, WhisperFullParams.ByValue params, final float[] samples, int n_samples, int n_processors);
 
     /**
      * Number of generated text segments.

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperCppJnaLibrary.java

@@ -0,0 +1,17 @@
+package io.github.ggerganov.whispercpp.callbacks;
+
+import com.sun.jna.Callback;
+
+/**
+ * Callback for aborting GGML computation
+ * Maps to the C typedef: bool (*ggml_abort_callback)(void * data)
+ */
+public interface GgmlAbortCallback extends Callback {
+    /**
+     * Return true to abort the computation, false to continue
+     *
+     * @param data User data passed to the callback
+     * @return true to abort, false to continue
+     */
+    boolean invoke(com.sun.jna.Pointer data);
+}

bindings/java/src/main/java/io/github/ggerganov/whispercpp/callbacks/GgmlAbortCallback.java

@@ -0,0 +1,30 @@
+package io.github.ggerganov.whispercpp.params;
+import com.sun.jna.*;
+import java.util.Arrays;
+import java.util.List;
+
+public class WhisperAhead extends Structure {
+
+    public int n_text_layer;
+
+    public int n_head;
+
+    public WhisperAhead() {
+        super();
+    }
+
+    public WhisperAhead(int textLayer, int head) {
+        super();
+        this.n_text_layer = textLayer;
+        this.n_head = head;
+    }
+
+    @Override
+    protected List<String> getFieldOrder() {
+        return Arrays.asList("n_text_layer", "n_head");
+    }
+
+    public static class ByReference extends WhisperAhead implements Structure.ByReference {}
+
+    public static class ByValue extends WhisperAhead implements Structure.ByValue {}
+}

bindings/java/src/main/java/io/github/ggerganov/whispercpp/params/WhisperAhead.java

@@ -0,0 +1,41 @@
+package io.github.ggerganov.whispercpp.params;
+import com.sun.jna.*;
+import java.util.Arrays;
+import java.util.List;
+
+public class WhisperAheads extends Structure {
+    public NativeLong n_heads;
+
+    public Pointer heads;
+
+    public WhisperAheads() {
+        super();
+    }
+
+    /**
+     * Create alignment heads from an array of WhisperAhead objects
+     */
+    public void setHeads(WhisperAhead[] aheadsArray) {
+        this.n_heads = new NativeLong(aheadsArray.length);
+
+        int structSize = aheadsArray[0].size();
+        Memory mem = new Memory(structSize * aheadsArray.length);
+
+        for (int i = 0; i < aheadsArray.length; i++) {
+            aheadsArray[i].write();
+            byte[] buffer = aheadsArray[i].getPointer().getByteArray(0, structSize);
+            mem.write(i * structSize, buffer, 0, buffer.length);
+        }
+
+        this.heads = mem;
+    }
+
+    @Override
+    protected List<String> getFieldOrder() {
+        return Arrays.asList("n_heads", "heads");
+    }
+
+    public static class ByReference extends WhisperAheads implements Structure.ByReference {}
+
+    public static class ByValue extends WhisperAheads implements Structure.ByValue {}
+}