Merge branch 'layla-build' into merge

Author: l3utterfly

.gitignore

@@ -77,7 +77,6 @@ autogen-*.md
 !.github/workflows/*.yml
 
 # Models
-
 models/*
 models-mnt
 !models/.editorconfig

common/common.cpp

@@ -1040,6 +1040,9 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
     auto mparams = llama_model_default_params();
 
     if (!params.devices.empty()) {
+        // add nullptr to the end just in case
+        params.devices.push_back(nullptr);
+
         mparams.devices = params.devices.data();
     }
 

common/minja/chat-template.hpp

@@ -9,7 +9,6 @@
 #pragma once
 
 #include "minja.hpp"
-
 #include <chrono>
 #include <cstddef>
 #include <cstdio>
@@ -19,7 +18,6 @@
 #include <sstream>
 #include <string>
 #include <vector>
-
 #include <json.hpp>
 
 using json = nlohmann::ordered_json;

common/minja/minja.hpp

@@ -27,7 +27,6 @@
 #include <unordered_set>
 #include <utility>
 #include <vector>
-
 #include <json.hpp>
 
 using json = nlohmann::ordered_json;

common/sampling.cpp

@@ -61,6 +61,17 @@ struct ring_buffer {
         return value;
     }
 
+    T pop_back() {
+        if (sz == 0) {
+            throw std::runtime_error("ring buffer is empty");
+        }
+        // Move pos backwards, wrapping around if necessary
+        pos = (pos == 0) ? capacity - 1 : pos - 1;
+        T value = data[pos];
+        sz--;
+        return value;
+    }
+
     const T & rat(size_t i) const {
         if (i >= sz) {
             throw std::runtime_error("ring buffer: index out of bounds");
@@ -316,6 +327,12 @@ void common_sampler_reset(struct common_sampler * gsmpl) {
     llama_sampler_reset(gsmpl->chain);
 }
 
+void common_sampler_reinit_grammar(struct common_sampler * gsmpl, const struct llama_model * model, const char * grammar) {
+    llama_sampler_reset(gsmpl->grmr);
+
+    gsmpl->grmr = llama_sampler_init_grammar(llama_model_get_vocab(model), grammar, "root");
+}
+
 struct common_sampler * common_sampler_clone(common_sampler * gsmpl) {
     return new common_sampler {
         /* .params = */ gsmpl->params,
@@ -469,6 +486,21 @@ std::string common_sampler_prev_str(common_sampler * gsmpl, llama_context * ctx_
     return result;
 }
 
+const std::vector<llama_token> common_sampler_prev(common_sampler * gsmpl) {
+    return gsmpl->prev.to_vector();
+}
+
+void common_sampler_rollback(common_sampler * gsmpl, int rollback_num) {
+    if(rollback_num > gsmpl->prev.size()) {
+        rollback_num = gsmpl->prev.size();
+    }
+
+    // continuously pop the last token
+    for(int i = 0; i < rollback_num; i++) {
+        gsmpl->prev.pop_back();
+    }
+}
+
 char common_sampler_type_to_chr(enum common_sampler_type cnstr) {
     switch (cnstr) {
         case COMMON_SAMPLER_TYPE_DRY:         return 'd';

common/sampling.h

@@ -43,6 +43,7 @@ void common_sampler_free(struct common_sampler * gsmpl);
 // if accept_grammar is true, the token is accepted both by the sampling chain and the grammar
 void                    common_sampler_accept(struct common_sampler * gsmpl, llama_token token, bool accept_grammar);
 void                    common_sampler_reset (struct common_sampler * gsmpl);
+void                    common_sampler_reinit_grammar(struct common_sampler * gsmpl, const struct llama_model * model, const char * grammar);
 struct common_sampler * common_sampler_clone (struct common_sampler * gsmpl);
 
 // arguments can be nullptr to skip printing
@@ -96,6 +97,8 @@ std::string common_sampler_print(const struct common_sampler * gsmpl);
 
 // get a string representation of the last accepted tokens
 std::string common_sampler_prev_str(common_sampler * gsmpl, llama_context * ctx, int n);
+const std::vector<llama_token> common_sampler_prev(common_sampler * gsmpl);
+void common_sampler_rollback(common_sampler * gsmpl, int rollback_num);
 
 char        common_sampler_type_to_chr(enum common_sampler_type cnstr);
 std::string common_sampler_type_to_str(enum common_sampler_type cnstr);

examples/llava/llava.cpp

@@ -16,6 +16,12 @@
 #   define LOG_WRN(...)
 #   define LOG_ERR(...)
 #   define LOG_DBG(...)
+#elif defined(ANDROID)
+#   include "llama-impl.h"
+#   define LOG_INF(...) LLAMA_LOG_INFO(__VA_ARGS__)
+#   define LOG_WRN(...) LLAMA_LOG_WARN(__VA_ARGS__)
+#   define LOG_ERR(...) LLAMA_LOG_ERROR(__VA_ARGS__)
+#   define LOG_DBG(...) LLAMA_LOG_DEBUG(__VA_ARGS__)
 #else // defined(LLAVA_LOG_OFF)
 #   define LOG_INF(...) do { fprintf(stdout, __VA_ARGS__); } while (0)
 #   define LOG_WRN(...) do { fprintf(stderr, __VA_ARGS__); } while (0)

ggml/CMakeLists.txt

@@ -272,6 +272,13 @@ set(GGML_PUBLIC_HEADERS
     include/gguf.h)
 
 set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")
+
+# link android log library
+if(ANDROID)
+    find_library(log-lib log)
+    target_link_libraries(ggml PRIVATE ${log-lib})
+endif()
+
 #if (GGML_METAL)
 #    set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal")
 #endif()

ggml/include/ggml-backend.h

@@ -202,6 +202,7 @@ extern "C" {
     //
     // Backend registry
     //
+    GGML_API void               ggml_backend_reg_layla(bool useVulkan, bool useOpenCL);
 
     GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);
 

ggml/src/CMakeLists.txt

@@ -203,6 +203,14 @@ add_library(ggml-base
             ggml-quants.h
             gguf.cpp)
 
+# Search for the 'log' library on Android
+if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android")
+    find_library(log-lib log)
+    set(GGML_EXTRA_LIBS     ${GGML_EXTRA_LIBS}     ${log-lib})
+
+    target_link_libraries(ggml-base PUBLIC ${GGML_EXTRA_LIBS})
+endif()
+
 target_include_directories(ggml-base PRIVATE .)
 if (GGML_BACKEND_DL)
     target_compile_definitions(ggml-base PUBLIC GGML_BACKEND_DL)