Merge branch 'ggerganov:master' into feat/detectedLanguage

Author: aderbedr

README.md

@@ -427,7 +427,8 @@ For detailed instructions on how to use Conan, please refer to the [Conan docume
 
 This is a naive example of performing real-time inference on audio from your microphone.
 The [stream](examples/stream) tool samples the audio every half a second and runs the transcription continuously.
-More info is available in [issue #10](https://github.com/ggerganov/whisper.cpp/issues/10).
+More info is available in [issue #10](https://github.com/ggerganov/whisper.cpp/issues/10). 
+You will need to have [sdl2](https://wiki.libsdl.org/SDL2/Installation) installed for it to work properly. 
 
 ```bash
 cmake -B build -DWHISPER_SDL2=ON

bindings/go/Makefile

@@ -11,11 +11,11 @@ UNAME_M := $(shell uname -m)
 endif
 
 GGML_METAL_PATH_RESOURCES := $(abspath ../..)
-BUILD_DIR := build
+BUILD_DIR := build_go
 MODELS_DIR := models
 EXAMPLES_DIR := $(wildcard examples/*)
 INCLUDE_PATH := $(abspath ../../include):$(abspath ../../ggml/include)
-LIBRARY_PATH := $(abspath ../..)
+LIBRARY_PATH := $(abspath ../../${BUILD_DIR}/src:$(abspath ../../${BUILD_DIR}/ggml/src))
 
 ifeq ($(GGML_CUDA),1)
 	LIBRARY_PATH := $(LIBRARY_PATH):$(CUDA_PATH)/targets/$(UNAME_M)-linux/lib/
@@ -29,8 +29,10 @@ endif
 all: clean whisper examples
 
 whisper: mkdir
-	@echo Build whisper
-	@${MAKE} -C ../.. libwhisper.a
+	cmake -S ../.. -B ../../${BUILD_DIR} \
+		-DCMAKE_BUILD_TYPE=Release \
+		-DBUILD_SHARED_LIBS=OFF
+	cmake --build ../../${BUILD_DIR} --target whisper
 
 test: model-small whisper modtidy
 ifeq ($(UNAME_S),Darwin)

bindings/go/whisper.go

@@ -9,7 +9,7 @@ import (
 // CGO
 
 /*
-#cgo LDFLAGS: -lwhisper -lm -lstdc++ -fopenmp
+#cgo LDFLAGS: -lwhisper -lggml -lggml-base -lggml-cpu  -lm -lstdc++ -fopenmp
 #cgo darwin LDFLAGS: -framework Accelerate -framework Metal -framework Foundation -framework CoreGraphics
 #include <whisper.h>
 #include <stdlib.h>

bindings/java/build.gradle

@@ -25,13 +25,13 @@ sourceSets {
 }
 
 tasks.register('copyLibwhisperDynlib', Copy) {
-    from '../../build'
-    include 'libwhisper.dynlib'
+    from '../../build/src'
+    include 'libwhisper.dylib'
     into 'build/generated/resources/main/darwin'
 }
 
 tasks.register('copyLibwhisperSo', Copy) {
-    from '../../build'
+    from '../../build/src'
     include 'libwhisper.so'
     into 'build/generated/resources/main/linux-x86-64'
 }
@@ -55,7 +55,12 @@ java {
     withJavadocJar()
 }
 
+sourcesJar() {
+    dependsOn copyLibs
+}
+
 jar {
+    dependsOn copyLibs
     exclude '**/whisper_java.exp', '**/whisper_java.lib'
 }
 
@@ -67,6 +72,9 @@ tasks.withType(Test) {
     useJUnitPlatform()
 }
 
+test.dependsOn copyLibs
+processResources.dependsOn copyLibs
+
 dependencies {
     implementation "net.java.dev.jna:jna:5.13.0"
     testImplementation "org.junit.jupiter:junit-jupiter:5.9.2"

bindings/java/gradlew

@@ -0,0 +1,24 @@
+package io.github.ggerganov.whispercpp;
+
+/**
+ * Presets for alignment heads in DTW token timestamps
+ */
+public class WhisperConstants {
+    // Alignment heads presets
+    public static final int WHISPER_AHEADS_NONE = 0;
+    public static final int WHISPER_AHEADS_TINY_EN = 1;
+    public static final int WHISPER_AHEADS_TINY = 2;
+    public static final int WHISPER_AHEADS_BASE_EN = 3;
+    public static final int WHISPER_AHEADS_BASE = 4;
+    public static final int WHISPER_AHEADS_SMALL_EN = 5;
+    public static final int WHISPER_AHEADS_SMALL = 6;
+    public static final int WHISPER_AHEADS_MEDIUM_EN = 7;
+    public static final int WHISPER_AHEADS_MEDIUM = 8;
+    public static final int WHISPER_AHEADS_LARGE_V1 = 9;
+    public static final int WHISPER_AHEADS_LARGE_V2 = 10;
+    public static final int WHISPER_AHEADS_LARGE_V3 = 11;
+    public static final int WHISPER_AHEADS_LARGE_V3_TURBO = 12;
+    public static final int WHISPER_AHEADS_CUSTOM = 13;
+    public static final int WHISPER_AHEADS_N_TOP_MOST = 14;
+    public static final int WHISPER_AHEADS_COUNT = 15;
+}

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperConstants.java

@@ -1,41 +1,36 @@
 package io.github.ggerganov.whispercpp;
 
+import com.sun.jna.NativeLong;
 import com.sun.jna.Structure;
 import com.sun.jna.ptr.PointerByReference;
+import com.sun.jna.Pointer;
 import io.github.ggerganov.whispercpp.ggml.GgmlType;
 import io.github.ggerganov.whispercpp.WhisperModel;
 import io.github.ggerganov.whispercpp.params.WhisperContextParams;
 
 import java.util.List;
 
 public class WhisperContext extends Structure {
-    int t_load_us = 0;
-    int t_start_us = 0;
+    public NativeLong t_load_us;
+    public NativeLong t_start_us;
 
     /** weight type (FP32 / FP16 / QX) */
-    GgmlType wtype = GgmlType.GGML_TYPE_F16;
+    public GgmlType wtype = GgmlType.GGML_TYPE_F16;
     /** intermediate type (FP32 or FP16) */
-    GgmlType itype = GgmlType.GGML_TYPE_F16;
+    public GgmlType itype = GgmlType.GGML_TYPE_F16;
 
-//    WhisperModel model;
-    public PointerByReference model;
-//    whisper_vocab vocab;
-//    whisper_state * state = nullptr;
-    public PointerByReference vocab;
-    public PointerByReference state;
+    public WhisperContextParams.ByValue params;
+
+    public Pointer model;
+    public Pointer vocab;
+    public Pointer state;
 
     /** populated by whisper_init_from_file_with_params() */
-    String path_model;
-    WhisperContextParams params;
-
-//    public static class ByReference extends WhisperContext implements Structure.ByReference {
-//    }
-//
-//    public static class ByValue extends WhisperContext implements Structure.ByValue {
-//    }
-//
-//    @Override
-//    protected List<String> getFieldOrder() {
-//        return List.of("t_load_us", "t_start_us", "wtype", "itype", "model", "vocab", "state", "path_model");
-//    }
+    public Pointer path_model;
+
+    @Override
+    protected List<String> getFieldOrder() {
+        return List.of("t_load_us", "t_start_us", "wtype", "itype",
+                "params", "model", "vocab", "state", "path_model");
+    }
 }

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperContext.java

@@ -43,11 +43,11 @@ public void initContext(String modelPath) throws FileNotFoundException {
      * @param modelPath - absolute path, or just the name (eg: "base", "base-en" or "base.en")
      * @param params - params to use when initialising the context
      */
-    public void initContext(String modelPath, WhisperContextParams params) throws FileNotFoundException {
+    public void initContext(String modelPath, WhisperContextParams.ByValue params) throws FileNotFoundException {
         initContextImpl(modelPath, params);
     }
 
-    private void initContextImpl(String modelPath, WhisperContextParams params) throws FileNotFoundException {
+    private void initContextImpl(String modelPath, WhisperContextParams.ByValue params) throws FileNotFoundException {
         if (ctx != null) {
             lib.whisper_free(ctx);
         }
@@ -69,15 +69,13 @@ private void initContextImpl(String modelPath, WhisperContextParams params) thro
 
     /**
      * Provides default params which can be used with `whisper_init_from_file_with_params()` etc.
-     * Because this function allocates memory for the params, the caller must call either:
-     * - call `whisper_free_context_params()`
-     * - `Native.free(Pointer.nativeValue(pointer));`
+     * Returns a ByValue instance to ensure proper parameter passing to native code.
      */
-    public WhisperContextParams getContextDefaultParams() {
-        paramsPointer = lib.whisper_context_default_params_by_ref();
-        WhisperContextParams params = new WhisperContextParams(paramsPointer);
-        params.read();
-        return params;
+    public WhisperContextParams.ByValue getContextDefaultParams() {
+        WhisperContextParams.ByValue valueParams = new WhisperContextParams.ByValue(
+            lib.whisper_context_default_params_by_ref());
+        valueParams.read();
+        return valueParams;
     }
 
     /**
@@ -88,7 +86,7 @@ public WhisperContextParams getContextDefaultParams() {
      *
      * @param strategy - GREEDY
      */
-    public WhisperFullParams getFullDefaultParams(WhisperSamplingStrategy strategy) {
+    public WhisperFullParams.ByValue getFullDefaultParams(WhisperSamplingStrategy strategy) {
         Pointer pointer;
 
         // whisper_full_default_params_by_ref allocates memory which we need to delete, so only create max 1 pointer for each strategy.
@@ -104,7 +102,7 @@ public WhisperFullParams getFullDefaultParams(WhisperSamplingStrategy strategy)
             pointer = beamParamsPointer;
         }
 
-        WhisperFullParams params = new WhisperFullParams(pointer);
+        WhisperFullParams.ByValue params = new WhisperFullParams.ByValue(pointer);
         params.read();
         return params;
     }
@@ -138,15 +136,21 @@ private void freeParams() {
     }
 
     /**
-     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text.
+     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text.
      * Not thread safe for same context
      * Uses the specified decoding strategy to obtain the text.
      */
-    public String fullTranscribe(WhisperFullParams whisperParams, float[] audioData) throws IOException {
+    public String fullTranscribe(WhisperFullParams.ByValue whisperParams, float[] audioData) throws IOException {
         if (ctx == null) {
             throw new IllegalStateException("Model not initialised");
         }
 
+        /*
+        WhisperFullParams.ByValue valueParams = new WhisperFullParams.ByValue(
+            lib.whisper_full_default_params_by_ref(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH.ordinal()));
+        valueParams.read();
+        */
+
         if (lib.whisper_full(ctx, whisperParams, audioData, audioData.length) != 0) {
             throw new IOException("Failed to process audio");
         }
@@ -163,12 +167,17 @@ public String fullTranscribe(WhisperFullParams whisperParams, float[] audioData)
 
         return str.toString().trim();
     }
+
     public List<WhisperSegment> fullTranscribeWithTime(WhisperFullParams whisperParams, float[] audioData) throws IOException {
         if (ctx == null) {
             throw new IllegalStateException("Model not initialised");
         }
 
-        if (lib.whisper_full(ctx, whisperParams, audioData, audioData.length) != 0) {
+        WhisperFullParams.ByValue valueParams = new WhisperFullParams.ByValue(
+            lib.whisper_full_default_params_by_ref(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH.ordinal()));
+        valueParams.read();
+
+        if (lib.whisper_full(ctx, valueParams, audioData, audioData.length) != 0) {
             throw new IOException("Failed to process audio");
         }
 

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperCpp.java

@@ -38,7 +38,7 @@ public interface WhisperCppJnaLibrary extends Library {
      * @param params     Pointer to whisper_context_params
      * @return Whisper context on success, null on failure
      */
-    Pointer whisper_init_from_file_with_params(String path_model, WhisperContextParams params);
+    Pointer whisper_init_from_file_with_params(String path_model, WhisperContextParams.ByValue params);
 
     /**
      * Allocate (almost) all memory needed for the model by loading from a buffer.
@@ -180,12 +180,12 @@ public interface WhisperCppJnaLibrary extends Library {
     /**
      * @return the id of the specified language, returns -1 if not found.
      * Examples:
-     *   "de" -> 2
-     *   "german" -> 2
+     *   "de" -> 2
+     *   "german" -> 2
      */
     int whisper_lang_id(String lang);
 
-    /** @return the short string of the specified language id (e.g. 2 -> "de"), returns nullptr if not found */
+    /** @return the short string of the specified language id (e.g. 2 -> "de"), returns nullptr if not found */
     String whisper_lang_str(int id);
 
     /**
@@ -268,20 +268,21 @@ public interface WhisperCppJnaLibrary extends Library {
     void whisper_free_params(Pointer params);
 
     /**
-     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text
+     * Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text
      * Not thread safe for same context
      * Uses the specified decoding strategy to obtain the text.
      */
-    int whisper_full(Pointer ctx, WhisperFullParams params, final float[] samples, int n_samples);
+    int whisper_full(Pointer ctx, WhisperFullParams.ByValue params, final float[] samples, int n_samples);
 
-    int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams params, final float[] samples, int n_samples);
+    public int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams.ByValue params, float[] samples, int n_samples);
+    //int whisper_full_with_state(Pointer ctx, Pointer state, WhisperFullParams params, final float[] samples, int n_samples);
 
     // Split the input audio in chunks and process each chunk separately using whisper_full_with_state()
     // Result is stored in the default state of the context
     // Not thread safe if executed in parallel on the same context.
     // It seems this approach can offer some speedup in some cases.
     // However, the transcription accuracy can be worse at the beginning and end of each chunk.
-    int whisper_full_parallel(Pointer ctx, WhisperFullParams params, final float[] samples, int n_samples, int n_processors);
+    int whisper_full_parallel(Pointer ctx, WhisperFullParams.ByValue params, final float[] samples, int n_samples, int n_processors);
 
     /**
      * Number of generated text segments.

bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperCppJnaLibrary.java

@@ -0,0 +1,17 @@
+package io.github.ggerganov.whispercpp.callbacks;
+
+import com.sun.jna.Callback;
+
+/**
+ * Callback for aborting GGML computation
+ * Maps to the C typedef: bool (*ggml_abort_callback)(void * data)
+ */
+public interface GgmlAbortCallback extends Callback {
+    /**
+     * Return true to abort the computation, false to continue
+     *
+     * @param data User data passed to the callback
+     * @return true to abort, false to continue
+     */
+    boolean invoke(com.sun.jna.Pointer data);
+}