Refactor and enhance ggml backend implementations

- Updated the AArch64 CPU backend buffer type to use a static structure for better memory management.
- Improved the initialization logic for CPU backend devices, ensuring proper context handling and initialization checks.
- Modified Hexagon backend CMake configuration to remove unnecessary dependencies on external libraries and streamline the build process.
- Added support for dynamic loading of libcdsprpc.so in the Hexagon backend, enhancing compatibility with various environments.
- Introduced new grammar definitions for Chinese, Korean, and scheduling formats to improve language processing capabilities.
- Implemented logging enhancements for Android, allowing for better error tracking and debugging.
- Added Vulkan backend support with pre-generated shader files for cross-compilation, improving performance on Android platforms.
- Cleaned up various source files by removing redundant code and ensuring consistent formatting.

Author: l3utterfly

.gitignore

@@ -77,7 +77,6 @@ autogen-*.md
 !.github/workflows/*.yml
 
 # Models
-
 models/*
 models-mnt
 !models/.editorconfig

common/common.cpp

@@ -1074,6 +1074,9 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
     auto mparams = llama_model_default_params();
 
     if (!params.devices.empty()) {
+        // add nullptr to the end just in case
+        params.devices.push_back(nullptr);
+
         mparams.devices = params.devices.data();
     }
 

common/minja/chat-template.hpp

@@ -9,7 +9,6 @@
 #pragma once
 
 #include "minja.hpp"
-
 #include <chrono>
 #include <cstddef>
 #include <cstdio>
@@ -21,7 +20,6 @@
 #include <stdexcept>
 #include <string>
 #include <vector>
-
 #include <json.hpp>
 
 using json = nlohmann::ordered_json;

common/minja/minja.hpp

@@ -28,7 +28,6 @@
 #include <unordered_set>
 #include <utility>
 #include <vector>
-
 #include <json.hpp>
 
 using json = nlohmann::ordered_json;

common/sampling.cpp

@@ -62,6 +62,17 @@ struct ring_buffer {
         return value;
     }
 
+    T pop_back() {
+        if (sz == 0) {
+            throw std::runtime_error("ring buffer is empty");
+        }
+        // Move pos backwards, wrapping around if necessary
+        pos = (pos == 0) ? capacity - 1 : pos - 1;
+        T value = data[pos];
+        sz--;
+        return value;
+    }
+
     const T & rat(size_t i) const {
         if (i >= sz) {
             throw std::runtime_error("ring buffer: index out of bounds");
@@ -314,6 +325,12 @@ void common_sampler_reset(struct common_sampler * gsmpl) {
     llama_sampler_reset(gsmpl->chain);
 }
 
+void common_sampler_reinit_grammar(struct common_sampler * gsmpl, const struct llama_model * model, const char * grammar) {
+    llama_sampler_reset(gsmpl->grmr);
+
+    gsmpl->grmr = llama_sampler_init_grammar(llama_model_get_vocab(model), grammar, "root");
+}
+
 struct common_sampler * common_sampler_clone(common_sampler * gsmpl) {
     return new common_sampler {
         /* .params = */ gsmpl->params,
@@ -467,6 +484,21 @@ std::string common_sampler_prev_str(common_sampler * gsmpl, llama_context * ctx_
     return result;
 }
 
+const std::vector<llama_token> common_sampler_prev(common_sampler * gsmpl) {
+    return gsmpl->prev.to_vector();
+}
+
+void common_sampler_rollback(common_sampler * gsmpl, int rollback_num) {
+    if(rollback_num > gsmpl->prev.size()) {
+        rollback_num = gsmpl->prev.size();
+    }
+
+    // continuously pop the last token
+    for(int i = 0; i < rollback_num; i++) {
+        gsmpl->prev.pop_back();
+    }
+}
+
 char common_sampler_type_to_chr(enum common_sampler_type cnstr) {
     switch (cnstr) {
         case COMMON_SAMPLER_TYPE_DRY:         return 'd';

common/sampling.h

@@ -43,6 +43,7 @@ void common_sampler_free(struct common_sampler * gsmpl);
 // if accept_grammar is true, the token is accepted both by the sampling chain and the grammar
 void                    common_sampler_accept(struct common_sampler * gsmpl, llama_token token, bool accept_grammar);
 void                    common_sampler_reset (struct common_sampler * gsmpl);
+void                    common_sampler_reinit_grammar(struct common_sampler * gsmpl, const struct llama_model * model, const char * grammar);
 struct common_sampler * common_sampler_clone (struct common_sampler * gsmpl);
 
 // arguments can be nullptr to skip printing
@@ -96,6 +97,8 @@ std::string common_sampler_print(const struct common_sampler * gsmpl);
 
 // get a string representation of the last accepted tokens
 std::string common_sampler_prev_str(common_sampler * gsmpl, llama_context * ctx, int n);
+const std::vector<llama_token> common_sampler_prev(common_sampler * gsmpl);
+void common_sampler_rollback(common_sampler * gsmpl, int rollback_num);
 
 char        common_sampler_type_to_chr(enum common_sampler_type cnstr);
 std::string common_sampler_type_to_str(enum common_sampler_type cnstr);

ggml/CMakeLists.txt

@@ -275,6 +275,13 @@ set(GGML_PUBLIC_HEADERS
     include/gguf.h)
 
 set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")
+
+# link android log library
+if(ANDROID)
+    find_library(log-lib log)
+    target_link_libraries(ggml PRIVATE ${log-lib})
+endif()
+
 #if (GGML_METAL)
 #    set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal")
 #endif()

ggml/include/ggml-backend.h

@@ -202,6 +202,7 @@ extern "C" {
     //
     // Backend registry
     //
+    GGML_API void               ggml_backend_reg_layla(bool useVulkan, bool useOpenCL, bool useHexagon);
 
     GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);
 

ggml/src/CMakeLists.txt

@@ -203,6 +203,14 @@ add_library(ggml-base
             ggml-quants.h
             gguf.cpp)
 
+# Search for the 'log' library on Android
+if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android")
+    find_library(log-lib log)
+    set(GGML_EXTRA_LIBS     ${GGML_EXTRA_LIBS}     ${log-lib})
+
+    target_link_libraries(ggml-base PUBLIC ${GGML_EXTRA_LIBS})
+endif()
+
 target_include_directories(ggml-base PRIVATE .)
 if (GGML_BACKEND_DL)
     target_compile_definitions(ggml-base PUBLIC GGML_BACKEND_DL)

ggml/src/ggml-backend-reg.cpp

@@ -159,6 +159,10 @@ struct ggml_backend_reg_entry {
     dl_handle_ptr handle;
 };
 
+static bool laylaUseVulkan = false;
+static bool laylaUseOpenCL = false;
+static bool laylaUseHexagon = false;
+
 struct ggml_backend_registry {
     std::vector<ggml_backend_reg_entry> backends;
     std::vector<ggml_backend_dev_t> devices;
@@ -174,10 +178,14 @@ struct ggml_backend_registry {
         register_backend(ggml_backend_sycl_reg());
 #endif
 #ifdef GGML_USE_VULKAN
-        register_backend(ggml_backend_vk_reg());
+        if(laylaUseVulkan) {
+            register_backend(ggml_backend_vk_reg());
+        }
 #endif
 #ifdef GGML_USE_OPENCL
-        register_backend(ggml_backend_opencl_reg());
+        if(laylaUseOpenCL) {
+            register_backend(ggml_backend_opencl_reg());
+        }
 #endif
 #ifdef GGML_USE_CANN
         register_backend(ggml_backend_cann_reg());
@@ -192,7 +200,9 @@ struct ggml_backend_registry {
         register_backend(ggml_backend_kompute_reg());
 #endif
 #ifdef GGML_USE_HEXAGON
-        register_backend(ggml_backend_hexagon_reg());
+        if(laylaUseHexagon) {
+            register_backend(ggml_backend_hexagon_reg());
+        }
 #endif
 #ifdef GGML_USE_CPU
         register_backend(ggml_backend_cpu_reg());
@@ -303,6 +313,12 @@ struct ggml_backend_registry {
     }
 };
 
+void ggml_backend_reg_layla(bool useVulkan, bool useOpenCL, bool useHexagon) {
+    laylaUseVulkan = useVulkan;
+    laylaUseOpenCL = useOpenCL;
+    laylaUseHexagon = useHexagon;
+}
+
 static ggml_backend_registry & get_reg() {
     static ggml_backend_registry reg;
     return reg;