Enhance ggml_backend_reg_layla to support Hexagon backend and update CMake configuration for Hexagon SDK integration

l3utterfly · l3utterfly · commit 28565d1d8b18 · 2025-04-30T13:48:01.000+08:00
diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h
@@ -202,7 +202,7 @@ extern "C" {
     //
     // Backend registry
     //
-    GGML_API void               ggml_backend_reg_layla(bool useVulkan, bool useOpenCL);
+    GGML_API void               ggml_backend_reg_layla(bool useVulkan, bool useOpenCL, bool useHexagon);
 
     GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);
 
diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp
@@ -161,6 +161,7 @@ struct ggml_backend_reg_entry {
 
 static bool laylaUseVulkan = false;
 static bool laylaUseOpenCL = false;
+static bool laylaUseHexagon = false;
 
 struct ggml_backend_registry {
     std::vector<ggml_backend_reg_entry> backends;
@@ -199,7 +200,9 @@ struct ggml_backend_registry {
         register_backend(ggml_backend_kompute_reg());
 #endif
 #ifdef GGML_USE_HEXAGON
-        register_backend(ggml_backend_hexagon_reg());
+        if(laylaUseHexagon) {
+            register_backend(ggml_backend_hexagon_reg());
+        }
 #endif
 #ifdef GGML_USE_CPU
         register_backend(ggml_backend_cpu_reg());
@@ -310,9 +313,10 @@ struct ggml_backend_registry {
     }
 };
 
-void ggml_backend_reg_layla(bool useVulkan, bool useOpenCL) {
+void ggml_backend_reg_layla(bool useVulkan, bool useOpenCL, bool useHexagon) {
     laylaUseVulkan = useVulkan;
     laylaUseOpenCL = useOpenCL;
+    laylaUseHexagon = useHexagon;
 }
 
 static ggml_backend_registry & get_reg() {
diff --git a/ggml/src/ggml-hexagon/CMakeLists.txt b/ggml/src/ggml-hexagon/CMakeLists.txt
@@ -33,42 +33,45 @@ endif()
 #v73 --- Snapdragon 8 Gen2
 #v75 --- Snapdragon 8 Gen3
 #v79 --- Snapdragon 8 Elite(aka Gen4)
-if(NOT DEFINED HTP_ARCH_VERSION)
-    message(FATAL_ERROR "HTP_ARCH_VERSION not defined, valid htp arch: v68,v69,v73,v75,v79")
-endif()
+# we do not use HTP_ARCH_VERSION right now because we don't use raw cdsp calls
+#if(NOT DEFINED HTP_ARCH_VERSION)
+#    message(FATAL_ERROR "HTP_ARCH_VERSION not defined, valid htp arch: v68,v69,v73,v75,v79")
+#endif()
 
 #check whether user's specified htp arch is valid
-set(CHECK_HTP_ARCH "WRONG")
-foreach (feat v68 v69 v73 v75 v79)
-    if (${feat} STREQUAL ${HTP_ARCH_VERSION})
-        set(CHECK_HTP_ARCH "GOOD")
-    endif()
-endforeach()
-if (${CHECK_HTP_ARCH} STREQUAL "WRONG")
-    message(FATAL_ERROR "ggml-hexagon backend only support htp arch v68,v69,v73,v75,v79")
-endif()
+#set(CHECK_HTP_ARCH "WRONG")
+#foreach (feat v68 v69 v73 v75 v79)
+#    if (${feat} STREQUAL ${HTP_ARCH_VERSION})
+#        set(CHECK_HTP_ARCH "GOOD")
+#    endif()
+#endforeach()
+#if (${CHECK_HTP_ARCH} STREQUAL "WRONG")
+#    message(FATAL_ERROR "ggml-hexagon backend only support htp arch v68,v69,v73,v75,v79")
+#endif()
 
 #check optimization flags
 set(OPT_FLAG " ")
-if (${HTP_ARCH_VERSION} STREQUAL "v75" OR ${HTP_ARCH_VERSION} STREQUAL "v79")
+#if (${HTP_ARCH_VERSION} STREQUAL "v75" OR ${HTP_ARCH_VERSION} STREQUAL "v79")
     #works fine on Snapdragon 8Gen3&8Elite with 1.5x - 3x performance gains with the default ggml backend
-    set(OPT_FLAG " -O3 -march=armv8.7-a -mcpu=cortex-x1 -mtune=cortex-x1 -flto -D_GNU_SOURCE -fvectorize -ffp-model=fast -fno-finite-math-only")
-endif()
-message("OPT_FLAG:${OPT_FLAG}")
+#    set(OPT_FLAG " -O3 -march=armv8.7-a -mcpu=cortex-x1 -mtune=cortex-x1 -flto -D_GNU_SOURCE -fvectorize -ffp-model=fast -fno-finite-math-only")
+#endif()
+#message("OPT_FLAG:${OPT_FLAG}")
 
 if(CMAKE_SYSTEM_NAME STREQUAL "Android")
     find_library(LOG_LIB log)
 
-    add_library(cdsprpc
-        SHARED
-        IMPORTED)
-    set_target_properties(cdsprpc
-        PROPERTIES
-        IMPORTED_LOCATION
-        ${HEXAGON_SDK_PATH}/ipc/fastrpc/remote/ship/android_aarch64/libcdsprpc.so)
+    # we do not use libcdsprpc.so provided in the Hexagon SDK, we will look for the one installed by the user's phone vendor
+    #add_library(cdsprpc
+    #    SHARED
+    #    IMPORTED)
+    #set_target_properties(cdsprpc
+    #    PROPERTIES
+    #    IMPORTED_LOCATION
+    #    ${HEXAGON_SDK_PATH}/ipc/fastrpc/remote/ship/android_aarch64/libcdsprpc.so)
 
-    set(QNN_LINK_LIBRARIES ${LOG_LIB} cdsprpc)
-    set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend")
+    #set(QNN_LINK_LIBRARIES ${LOG_LIB} cdsprpc)
+    set(QNN_LINK_LIBRARIES ${LOG_LIB})
+    #set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend")
 
     include_directories(${HEXAGON_SDK_PATH}/incs)
     include_directories(${HEXAGON_SDK_PATH}/incs/stddef)
@@ -129,5 +132,6 @@ function(ggml_hexagon_setup_cfg KNAME)
     )
 endfunction()
 
-ggml_hexagon_build_kernel("cdsp")
-ggml_hexagon_setup_cfg("ggml-hexagon.cfg")
+# we do not build cdsp kernels directly in CMake
+#ggml_hexagon_build_kernel("cdsp")
+#ggml_hexagon_setup_cfg("ggml-hexagon.cfg")
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -391,7 +391,7 @@ static struct hexagon_appcfg_t g_hexagon_appcfg = {
     #if defined(STANDARD_ANDROID_APP)
         .runtime_libpath        = "/data/data/com.kantvai.kantvplayer/",
     #else
-        .runtime_libpath        = "/data/local/tmp/",
+        .runtime_libpath        = "/data/data/com.layla/files/app-data/qnn-inference/",
     #endif
 #elif defined(__linux__)
         .qnn_runtimelib_path    = "/tmp/",
@@ -1829,24 +1829,31 @@ static void ggmlhexagon_set_runtime_path(size_t device, const std::string & path
     if ((HEXAGON_BACKEND_QNNNPU == device) || (HWACCEL_CDSP == g_hexagon_appcfg.hwaccel_approach)) {
         std::string lib_runtime_path = path + ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images";
         if (0 == setenv("LD_LIBRARY_PATH", lib_runtime_path.c_str(), 1)) {
-            GGMLHEXAGON_LOG_DEBUG("setenv LD_LIBRARY_PATH %s successfully", lib_runtime_path.c_str());
+            GGMLHEXAGON_LOG_INFO("setenv LD_LIBRARY_PATH %s successfully", lib_runtime_path.c_str());
         } else {
             GGMLHEXAGON_LOG_ERROR("setenv LD_LIBRARY_PATH %s failure", lib_runtime_path.c_str());
         }
 
         std::string adsp_runtime_path = path + ";/vendor/dsp/cdsp;/vendor/lib/rfsa/adsp;/system/lib/rfsa/adsp;/vendor/dsp/dsp;/vendor/dsp/images;/dsp";
         if (0 == setenv("ADSP_LIBRARY_PATH", adsp_runtime_path.c_str(), 1)) {
-            GGMLHEXAGON_LOG_DEBUG("setenv ADSP_LIBRARY_PATH %s successfully", adsp_runtime_path.c_str());
+            GGMLHEXAGON_LOG_INFO("setenv ADSP_LIBRARY_PATH %s successfully", adsp_runtime_path.c_str());
         } else {
             GGMLHEXAGON_LOG_ERROR("setenv ADSP_LIBRARY_PATH %s failure", adsp_runtime_path.c_str());
         }
+
+        std::string dsp_runtime_path = path;
+        if (0 == setenv("DSP_LIBRARY_PATH", dsp_runtime_path.c_str(), 1)) {
+            GGMLHEXAGON_LOG_INFO("setenv DSP_LIBRARY_PATH %s successfully", dsp_runtime_path.c_str());
+        } else {
+            GGMLHEXAGON_LOG_ERROR("setenv DSP_LIBRARY_PATH %s failure", dsp_runtime_path.c_str());
+        }
     } else {
         if (0 == setenv("LD_LIBRARY_PATH",
                         (path +
                          ":/vendor/dsp/cdsp:/vendor/lib64:/vendor/dsp/dsp:/vendor/dsp/images").c_str(),
                         1)) {
             GGMLHEXAGON_LOG_DEBUG("%s backend setenv successfully\n",
-                                 ggml_backend_hexagon_get_devname(device));
+                                  ggml_backend_hexagon_get_devname(device));
         } else {
             GGMLHEXAGON_LOG_ERROR("%s backend setenv failure\n",
                                   ggml_backend_hexagon_get_devname(device));
@@ -3375,10 +3382,10 @@ int qnn_instance::qnn_init(const QnnSaver_Config_t ** saver_config) {
 
 #if defined(__ANDROID__) || defined(__linux__)
     std::filesystem::path full_path(std::string(g_hexagon_appcfg.runtime_libpath) + "libcdsprpc.so");
-    full_path /= std::filesystem::path("libcdsprpc.so").filename();
+    //full_path /= std::filesystem::path("libcdsprpc.so").filename();
     _rpc_lib_handle = dlopen(full_path.string().c_str(), RTLD_NOW | RTLD_LOCAL);
     if (nullptr == _rpc_lib_handle) {
-        GGMLHEXAGON_LOG_WARN("failed to load %s\n", full_path.c_str());
+        GGMLHEXAGON_LOG_WARN("failed to load %s from local file, trying to find in system libraries\n", full_path.c_str());
         _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL);
     }
 #else
@@ -5185,6 +5192,8 @@ static int ggmlhexagon_request_status_notifications(int domain_id, void * contex
 }
 
 static int ggmlhexagon_init_rpcmempool(ggml_backend_hexagon_context * ctx) {
+    throw std::runtime_error("Not implemented. Directly initialising RPC memory pool is not supported right now.");
+
     size_t candidate_size   = 0;
     uint8_t * rpc_buffer    = nullptr;
     size_t probe_slots[]    = {1024, 1536, 2000, 2048};
@@ -5232,6 +5241,8 @@ static int ggmlhexagon_init_rpcmempool(ggml_backend_hexagon_context * ctx) {
 }
 
 static void ggmlhexagon_deinit_rpcmempool(ggml_backend_hexagon_context * ctx) {
+    throw std::runtime_error("Not implemented. Directly initialising RPC memory pool is not supported right now.");
+
     if ((g_hexagon_appcfg.hwaccel_approach == HWACCEL_CDSP) && (1 == g_hexagon_appcfg.enable_rpc_ion_mempool)) {
         if (ctx->rpc_mempool) {
             //deregister rpc memory pool
@@ -6233,7 +6244,11 @@ static ggml_backend_buffer_type_t ggml_backend_hexagon_buffer_type(size_t device
         //cover following special case:
         //      toggle backend and forth between cDSP and ggml in a standard Android APP or in
         //      a same running process
-        g_hexagon_appcfg.hexagon_backend = device_index;
+
+        // TODO: not sure why we need to update the global setting here in the original code
+        // it seems this code is reached when we allocate buffers for all devices (including the qnn-cpu device)
+        // so if it reaches this code, then it won't use the NPU anymore since the backend config will be updated to use the cpu device
+        // g_hexagon_appcfg.hexagon_backend = device_index;
     }
 
     static struct ggml_backend_buffer_type ggml_backend_hexagon_buffer_types[GGML_HEXAGON_MAX_DEVICES];
@@ -6284,6 +6299,10 @@ static const char * ggml_backend_hexagon_host_buffer_name(ggml_backend_buffer_t
 }
 
 static void ggml_backend_hexagon_host_buffer_free(ggml_backend_buffer_t buffer) {
+    // always use ggml memory management for now
+    ggml_aligned_free(buffer->context, 0);
+    return;
+
     if (0 == g_hexagon_appcfg.enable_pinned_memory) {
         ggml_aligned_free(buffer->context, 0);
     } else {
@@ -6292,6 +6311,9 @@ static void ggml_backend_hexagon_host_buffer_free(ggml_backend_buffer_t buffer)
 }
 
 static void * ggml_hexagon_host_malloc(ggml_backend_buffer_type_t buft, size_t size) {
+    // we always use ggml malloc right now
+    return ggml_aligned_malloc(size);
+
     if (0 == g_hexagon_appcfg.enable_pinned_memory) {
         return ggml_aligned_malloc(size);
     } else {
@@ -6664,6 +6686,9 @@ ggml_backend_t ggml_backend_hexagon_init(size_t device, const char * runtime_lib
         ggmlhexagon_set_runtime_path(device, runtime_libpath);
     }
 
+    // the condition above never be true because our hardcoded runtime_libpath is always the same as the config, so we manually set the library paths here
+    ggmlhexagon_set_runtime_path(g_hexagon_appcfg.hexagon_backend, g_hexagon_appcfg.runtime_libpath);
+
     if (nullptr != g_hexagon_mgr[device].backend) {
         GGMLHEXAGON_LOG_DEBUG("backend %d(%s) already loaded", device,
                          ggml_backend_hexagon_get_devname(device));
diff --git a/ggml/src/ggml-hexagon/kernels/stub.c b/ggml/src/ggml-hexagon/kernels/stub.c
@@ -291,10 +291,12 @@ __QAIC_SLIM_EXPORT const Interface __QAIC_SLIM(ggmlop_slim) = {8,&(methodArrays[
 extern "C" {
 #endif
 __QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_dsp_open)(const char* uri, remote_handle64* h) __QAIC_STUB_ATTRIBUTE {
-   return __QAIC_REMOTE(remote_handle64_open)(uri, h);
+    return -1;  // don't support direct dsp calls yet
+   //return __QAIC_REMOTE(remote_handle64_open)(uri, h);
 }
 __QAIC_STUB_EXPORT int __QAIC_STUB(ggmlop_dsp_close)(remote_handle64 h) __QAIC_STUB_ATTRIBUTE {
-   return __QAIC_REMOTE(remote_handle64_close)(h);
+    return -1;  // don't support direct dsp calls yet
+    //return __QAIC_REMOTE(remote_handle64_close)(h);
 }
 static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint32_t _in0[1], uint32_t _in1[1], uint32_t _in2[1], uint32_t _in3[1]) {
    remote_arg _pra[1] = {0};
@@ -306,7 +308,8 @@ static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint32_
    _COPY(_primIn, 4, _in1, 0, 4);
    _COPY(_primIn, 8, _in2, 0, 4);
    _COPY(_primIn, 12,_in3, 0, 4);
-   _TRY_FARF(_nErr, __QAIC_REMOTE(remote_handle64_invoke)(_handle, REMOTE_SCALARS_MAKEX(0, _mid, 1, 0, 0, 0), _pra));
+   // TODO: we don't support direct dsp calls yet
+   //_TRY_FARF(_nErr, __QAIC_REMOTE(remote_handle64_invoke)(_handle, REMOTE_SCALARS_MAKEX(0, _mid, 1, 0, 0, 0), _pra));
    _CATCH_FARF(_nErr) {
       _QAIC_FARF(RUNTIME_ERROR, "ERROR 0x%x: handle=0x%"PRIx64", scalar=0x%x, method ID=%d: %s failed\n", _nErr , _handle, REMOTE_SCALARS_MAKEX(0, _mid, 1, 0, 0, 0), _mid, __func__);
    }
@@ -432,7 +435,8 @@ static __inline int _stub_method_1(remote_handle64 _handle, uint32_t _mid, uintp
    _TRY(_nErr, _stub_pack(_al, (_praIn + 0), _ppraIn, (_praROut + 0), _ppraROut, _praHIn, _ppraHIn, _praHROut, _ppraHROut, ((char*)_primIn + 224), ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint32_t*)&(((uint32_t*)_rout2)[1]), (uint32_t*)&(((uint32_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[9]), (uint32_t*)&(((uint32_t*)_rout2)[10]), (uint32_t*)&(((uint32_t*)_rout2)[26]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[27]), (char**)&(((uint64_t*)_rout2)[14])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[28]), (uint32_t*)&(((uint32_t*)_rout2)[30]))));
    _QAIC_ASSERT(_nErr, (_numInH[0] + 0) <= 15);
    _QAIC_ASSERT(_nErr, (_numROutH[0] + 0) <= 15);
-   _TRY_FARF(_nErr, __QAIC_REMOTE(remote_handle64_invoke)(_handle, REMOTE_SCALARS_MAKEX(0, _mid, (_numIn[0] + 1), (_numROut[0] + 1), (_numInH[0] + 0), (_numROutH[0] + 0)), _pra));
+   // TODO: we don't support direct dsp calls yet
+   //_TRY_FARF(_nErr, __QAIC_REMOTE(remote_handle64_invoke)(_handle, REMOTE_SCALARS_MAKEX(0, _mid, (_numIn[0] + 1), (_numROut[0] + 1), (_numInH[0] + 0), (_numROutH[0] + 0)), _pra));
    _TRY(_nErr, _stub_unpack((_praROutPost + 0), _ppraROutPost, ((char*)_primROut + 0), (uint32_t*)&(((uint32_t*)_rout2)[0]), (uint32_t*)&(((uint32_t*)_rout2)[1]), (uint32_t*)&(((uint32_t*)_rout2)[5]), (uint32_t*)&(((uint32_t*)_rout2)[9]), (uint32_t*)&(((uint32_t*)_rout2)[10]), (uint32_t*)&(((uint32_t*)_rout2)[26]), SLIM_IFPTR32((char**)&(((uint32_t*)_rout2)[27]), (char**)&(((uint64_t*)_rout2)[14])), SLIM_IFPTR32((uint32_t*)&(((uint32_t*)_rout2)[28]), (uint32_t*)&(((uint32_t*)_rout2)[30]))));
    _QAIC_CATCH(_nErr) {}
    _CATCH_FARF(_nErr) {
