ggml-hexagon: add mulmat_algotype for further usage

Author: jeffzhou2000

ggml/include/ggml-hexagon.h

@@ -38,10 +38,11 @@ GGML_BACKEND_API int                ggml_backend_hexagon_get_device_count(void);
 
 GGML_BACKEND_API ggml_backend_reg_t ggml_backend_hexagon_reg(void);
 
-//attention:the following APIs are only available in the ggml-hexagon backend
 GGML_BACKEND_API const char *       ggml_backend_hexagon_get_devname(size_t dev_num);
 
-GGML_BACKEND_API void               ggml_backend_set_hexagon_cfg(int new_hexagon_backend, int new_hwaccel_approach);
+GGML_BACKEND_API void               ggml_backend_hexagon_set_cfg(int new_hexagon_backend, int new_hwaccel_approach);
+
+GGML_BACKEND_API int                ggml_backend_hexagon_get_mulmat_algotype(void);
 
 #ifdef __cplusplus
 }

ggml/src/ggml-hexagon/ggml-hexagon.cpp

@@ -342,6 +342,7 @@ struct hexagon_appcfg_t {
     int profiler_duration;      // threshold of duration in profiler, per seconds
     int profiler_counts;        // threshold of counts in profiler
     int thread_counts;          // thread_counts on cDSP side
+    int mulmat_algotype;        // algorithm type of mulmat on cDSP side
     const char * cfgfilename;
     const char * runtime_libpath;
     char ggml_hexagon_version[GGMLHEXAGON_TMPBUF_LEN];
@@ -367,6 +368,7 @@ static struct hexagon_appcfg_t g_hexagon_appcfg = {
         .profiler_duration      = 5,    //seconds
         .profiler_counts        = 100,
         .thread_counts          = 4,
+        .mulmat_algotype        = 0,
         .cfgfilename            = "ggml-hexagon.cfg",
 #if defined(__ANDROID__)
     #if defined(STANDARD_ANDROID_APP)
@@ -379,7 +381,7 @@ static struct hexagon_appcfg_t g_hexagon_appcfg = {
 #elif defined(_WIN32)
         .qnn_runtimelib_path    = "C:\\",
 #endif
-        .ggml_hexagon_version   = {"1.11"},
+        .ggml_hexagon_version   = {"1.12"},
         .ggml_dsp_version       = {"0.63"},
 };
 
@@ -1322,11 +1324,9 @@ class hexagon_perf {
         // had to expose two public function in hexagon_profiler class
         if (g_hexagon_profiler.profiler_get_frame_index() <= g_hexagon_profiler.profiler_get_threshold_count()) {
             const char * devname = ggml_backend_hexagon_get_devname(g_hexagon_appcfg.hexagon_backend);
+            //the logic here is make sense because already checked in ggml_backend_hexagon_device_init_backend
             if (g_hexagon_appcfg.hexagon_backend != HEXAGON_BACKEND_GGML) {
-                //add this check for a special scenario: an invalid value passed from user's program
-                if (0 != memcmp(devname, "unknown", strlen("unknown"))) {
-                    devname += 16;
-                }
+                devname += 16;
             }
             GGMLHEXAGON_LOG_VERBOSE("inference duration of %s through %s: %lld microseconds",
                                     _perf_name.c_str(), devname, _duration);
@@ -2006,6 +2006,7 @@ static void ggmlhexagon_load_cfg() {
     hexagoncfg_instance.get_intvalue("cdsp", "enable_rpc_ion_mempool", g_hexagon_appcfg.enable_rpc_ion_mempool, 0);
     hexagoncfg_instance.get_intvalue("cdsp", "enable_all_q_mulmat", g_hexagon_appcfg.enable_all_q_mulmat, 0);
     hexagoncfg_instance.get_intvalue("cdsp", "thread_counts", g_hexagon_appcfg.thread_counts, 4);
+    hexagoncfg_instance.get_intvalue("cdsp", "mulmat_algotype", g_hexagon_appcfg.mulmat_algotype, 0);
 
     memcpy(g_hexagon_appcfg.ggml_dsp_version, ggmldsp_version.c_str(), strlen(ggmldsp_version.c_str()));
 
@@ -2053,7 +2054,7 @@ static void ggmlhexagon_load_cfg() {
     initialized = true;
 }
 
-void ggml_backend_set_hexagon_cfg(int new_hexagon_backend, int new_hwaccel_approach) {
+void ggml_backend_hexagon_set_cfg(int new_hexagon_backend, int new_hwaccel_approach) {
     std::string cfg_filename = std::string(g_hexagon_appcfg.runtime_libpath) + std::string(g_hexagon_appcfg.cfgfilename);
     GGMLHEXAGON_LOG_VERBOSE("load hexagon appcfg from %s", cfg_filename.c_str());
     hexagon_appcfg hexagoncfg_instance;
@@ -2063,14 +2064,25 @@ void ggml_backend_set_hexagon_cfg(int new_hexagon_backend, int new_hwaccel_appro
     hexagoncfg_instance.dump([](const std::string & section, const std::string & key, const std::string value) {
         std::ostringstream  tmposs;
         tmposs << "section[" << std::setw(10) << std::left << section << "],[" << std::setw(25) << std::left << key << "] = [" << value << "]";
+#if 0
         if (ggmlhexagon_is_llamabench_running()) {
             GGMLHEXAGON_LOG_VERBOSE("%s", tmposs.str().c_str());
         } else {
             GGMLHEXAGON_LOG_INFO("%s", tmposs.str().c_str());
         }
+#endif
+        GGMLHEXAGON_LOG_VERBOSE("%s", tmposs.str().c_str());
     });
 }
 
+int ggml_backend_hexagon_get_mulmat_algotype() {
+    std::string cfg_filename = std::string(g_hexagon_appcfg.runtime_libpath) + std::string(g_hexagon_appcfg.cfgfilename);
+    hexagon_appcfg hexagoncfg_instance;
+    hexagoncfg_instance.load(cfg_filename);
+    hexagoncfg_instance.get_intvalue("cdsp", "mulmat_algotype", g_hexagon_appcfg.mulmat_algotype, 0);
+    return g_hexagon_appcfg.mulmat_algotype;
+}
+
 static bool ggmlhexagon_check_valid_appcfg() {
     bool is_valid_appcfg = true;
 
@@ -5641,7 +5653,9 @@ static int ggmlhexagon_init_dsp(ggml_backend_hexagon_context * ctx) {
         }
         ggmlhexagon_probe_dspinfo(ctx);
         //FIXME: re-use this function to pass thread_counts info to code on cDSP side before fully understand qidl mechanism
-        ggmlop_dsp_setclocks(ctx->ggmlop_handle, HAP_DCVS_VCORNER_TURBO_PLUS, 40, 1, g_hexagon_appcfg.thread_counts);
+        //ggmlop_dsp_setclocks(ctx->ggmlop_handle, HAP_DCVS_VCORNER_TURBO_PLUS, 40, 1, g_hexagon_appcfg.thread_counts);
+        //backward compatible with previous codes on cDSP side
+        ggmlop_dsp_setclocks(ctx->ggmlop_handle, HAP_DCVS_VCORNER_TURBO_PLUS, 40, g_hexagon_appcfg.mulmat_algotype, g_hexagon_appcfg.thread_counts);
         ggmlhexagon_set_rpc_latency(ctx->ggmlop_handle, RPC_POLL_QOS, 100);
         int result = ggmlhexagon_init_rpcmempool(ctx);
         if (0 != result) {
@@ -6427,7 +6441,10 @@ static ggml_backend_t ggml_backend_hexagon_device_init_backend(ggml_backend_dev_
         if (dev_index < 0) {
             GGMLHEXAGON_LOG_VERBOSE("it shouldn't happend\n");
             //test-thread-safety might-be running at the moment or an invalid value passed from user's program
-            dev_index = 0;
+            dev_index = HEXAGON_BACKEND_QNNCPU; //0
+        }
+        if (dev_index > GGML_HEXAGON_MAX_DEVICES) {
+            dev_index = HEXAGON_BACKEND_GGML;   //4
         }
         g_hexagon_appcfg.hexagon_backend = dev_index;
         GGMLHEXAGON_LOG_VERBOSE("program specified dev_index %d\n", dev_index);

ggml/src/ggml-hexagon/kernels/skel.h

@@ -272,7 +272,7 @@ __QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_open)(const char* uri, remote_
     * @retval, 0 on success, should always succeed
     */
 __QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_close)(remote_handle64 h) __QAIC_HEADER_ATTRIBUTE;
-__QAIC_HEADER_EXPORT AEEResult __QAIC_HEADER(ggmlop_dsp_setclocks)(remote_handle64 _h, int32 power_level, int32 latency, int32 dcvs_enable, int32 threads) __QAIC_HEADER_ATTRIBUTE;
+__QAIC_HEADER_EXPORT AEEResult __QAIC_HEADER(ggmlop_dsp_setclocks)(remote_handle64 _h, int32 power_level, int32 latency, int32 mulmat_algotype, int32 thread_counts) __QAIC_HEADER_ATTRIBUTE;
 __QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_add)(remote_handle64 _h, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_HEADER_ATTRIBUTE;
 __QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_mulmat)(remote_handle64 _h, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_HEADER_ATTRIBUTE;
 __QAIC_HEADER_EXPORT int __QAIC_HEADER(ggmlop_dsp_softmax)(remote_handle64 _h, const dsptensor* src0, const dsptensor* src1, dsptensor* dst) __QAIC_HEADER_ATTRIBUTE;

ggml/src/ggml-hexagon/kernels/stub.c

@@ -312,9 +312,9 @@ static __inline int _stub_method(remote_handle64 _handle, uint32_t _mid, uint32_
    }
    return _nErr;
 }
-__QAIC_STUB_EXPORT AEEResult __QAIC_STUB(ggmlop_dsp_setclocks)(remote_handle64 _handle, int32 power_level, int32 latency, int32 dcvs_enable, int32 threads) __QAIC_STUB_ATTRIBUTE {
+__QAIC_STUB_EXPORT AEEResult __QAIC_STUB(ggmlop_dsp_setclocks)(remote_handle64 _handle, int32 power_level, int32 latency, int32 mulmat_algotype, int32 threads) __QAIC_STUB_ATTRIBUTE {
    uint32_t _mid = 2;
-   return _stub_method(_handle, _mid, (uint32_t*)&power_level, (uint32_t*)&latency, (uint32_t*)&dcvs_enable, (uint32_t*)&threads);
+   return _stub_method(_handle, _mid, (uint32_t*)&power_level, (uint32_t*)&latency, (uint32_t*)&mulmat_algotype, (uint32_t*)&threads);
 }
 static __inline int _stub_unpack(_ATTRIBUTE_UNUSED remote_arg* _praROutPost, _ATTRIBUTE_UNUSED remote_arg* _ppraROutPost[1], _ATTRIBUTE_UNUSED void* _primROut, _ATTRIBUTE_UNUSED uint32_t _rout0[1], _ATTRIBUTE_UNUSED uint32_t _rout1[4], _ATTRIBUTE_UNUSED uint32_t _rout2[4], _ATTRIBUTE_UNUSED uint32_t _rout3[1], _ATTRIBUTE_UNUSED uint32_t _rout4[16], _ATTRIBUTE_UNUSED uint32_t _rout5[1], _ATTRIBUTE_UNUSED char* _rout6[1], _ATTRIBUTE_UNUSED uint32_t _rout6Len[1]) {
    int _nErr = 0;

scripts/build-run-android.sh

@@ -486,16 +486,15 @@ function run_llamacli()
 function run_llamabench()
 {
     prepare_run_on_phone llama-bench
-    llamabench_running_params=" -ngl 99 -t 8 -n 256 "
 
     echo "adb shell \"cd ${REMOTE_PATH} \
                && export LD_LIBRARY_PATH=${REMOTE_PATH} \
-               && ${REMOTE_PATH}/llama-bench ${llamabench_running_params} -mg $qnnbackend -m ${GGUF_MODEL_NAME}\""
-    echo "${REMOTE_PATH}/llama-bench ${llamabench_running_params} -mg $qnnbackend -m ${GGUF_MODEL_NAME}"
+               && ${REMOTE_PATH}/llama-bench ${running_params} -mg $qnnbackend -m ${GGUF_MODEL_NAME}\""
+    echo "${REMOTE_PATH}/llama-bench ${running_params} -mg $qnnbackend -m ${GGUF_MODEL_NAME}"
 
     adb shell "cd ${REMOTE_PATH} \
                && export LD_LIBRARY_PATH=${REMOTE_PATH} \
-               && ${REMOTE_PATH}/llama-bench ${llamabench_running_params} -mg $qnnbackend -m ${GGUF_MODEL_NAME}"
+               && ${REMOTE_PATH}/llama-bench ${running_params} -mg $qnnbackend -m ${GGUF_MODEL_NAME}"
 
 }
 

scripts/ggml-hexagon-for-binary-lib.cfg

@@ -1,6 +1,6 @@
 [general]
 #version of ggml-hexagon.cpp on ARM-AP side
-version = "1.11"
+version = "1.12"
 #version of ggml-dsp.c on cDSP side
 ggmldsp_version = "0.97"
 
@@ -90,3 +90,6 @@ enable_all_q_mulmat = 1
 # 1    disable multi-threading on cDSP side
 # 2-8  thread_counts on cDSP side
 thread_counts = 8
+
+#algorithm type of mulmat on cDSP side
+mulmat_algotype = 0

scripts/ggml-hexagon.cfg

@@ -1,6 +1,6 @@
 [general]
 #version of ggml-hexagon.cpp on ARM-AP side
-version = "1.11"
+version = "1.12"
 #version of ggml-dsp.c on cDSP side
 ggmldsp_version = "0.63"
 
@@ -89,3 +89,6 @@ enable_all_q_mulmat = 0
 # 1    disable multi-threading on cDSP side
 # 2-8  thread_counts on cDSP side
 thread_counts = 1
+
+#algorithm type of mulmat on cDSP side
+mulmat_algotype = 0

tests/ggmlhexagon-benchmark.cpp

@@ -320,10 +320,10 @@ int main(int argc, char * argv[]) {
 #ifdef GGML_USE_HEXAGON
     //avoid manually modify ggml-hexagon.cfg
     if (n_backend_type >= HEXAGON_BACKEND_CDSP) {
-        ggml_backend_set_hexagon_cfg(n_backend_type, HWACCEL_CDSP);
+        ggml_backend_hexagon_set_cfg(n_backend_type, HWACCEL_CDSP);
     }
     if (n_backend_type < HEXAGON_BACKEND_CDSP) {
-        ggml_backend_set_hexagon_cfg(n_backend_type, HWACCEL_QNN);
+        ggml_backend_hexagon_set_cfg(n_backend_type, HWACCEL_QNN);
     }
 #endif
 
@@ -457,7 +457,11 @@ int main(int argc, char * argv[]) {
     get_timestring(currenttime_string);
 
 #ifdef GGML_USE_HEXAGON
-    printf("[%s] duration of ut GGML_OP_%s with backend %s: %ld milliseconds\n", currenttime_string, ggml_op_name((enum ggml_op)n_ggml_op_type), ggml_backend_hexagon_get_devname(n_backend_type), n_duration);
+    if (n_backend_type == HEXAGON_BACKEND_CDSP) {
+        printf("[%s] duration of ut GGML_OP_%s with backend %s(algo type:%d): %ld milliseconds\n", currenttime_string, ggml_op_name((enum ggml_op)n_ggml_op_type), ggml_backend_hexagon_get_devname(n_backend_type), ggml_backend_hexagon_get_mulmat_algotype(), n_duration);
+    } else {
+        printf("[%s] duration of ut GGML_OP_%s with backend %s: %ld milliseconds\n", currenttime_string, ggml_op_name((enum ggml_op)n_ggml_op_type), ggml_backend_hexagon_get_devname(n_backend_type), n_duration);
+    }
 #else
     printf("[%s] duration of ut GGML_OP_%s with the default ggml backend: %ld milliseconds\n", currenttime_string, ggml_op_name((enum ggml_op)n_ggml_op_type), n_duration);
 #endif

tests/test-thread-safety.cpp

@@ -26,10 +26,10 @@ int main(int argc, char ** argv) {
     }
     printf("backend %d\n", backend);
     if (backend >= HEXAGON_BACKEND_CDSP) {
-        ggml_backend_set_hexagon_cfg(backend, HWACCEL_CDSP);
+        ggml_backend_hexagon_set_cfg(backend, HWACCEL_CDSP);
     }
     if (backend < HEXAGON_BACKEND_CDSP) {
-        ggml_backend_set_hexagon_cfg(backend, HWACCEL_QNN);
+        ggml_backend_hexagon_set_cfg(backend, HWACCEL_QNN);
     }
 #endif
     common_params params;

tools/llama-bench/llama-bench.cpp

@@ -1849,10 +1849,10 @@ int main(int argc, char ** argv) {
     }
     printf("backend %d\n", backend);
     if (backend >= HEXAGON_BACKEND_CDSP) {
-        ggml_backend_set_hexagon_cfg(backend, HWACCEL_CDSP);
+        ggml_backend_hexagon_set_cfg(backend, HWACCEL_CDSP);
     }
     if (backend < HEXAGON_BACKEND_CDSP) {
-        ggml_backend_set_hexagon_cfg(backend, HWACCEL_QNN);
+        ggml_backend_hexagon_set_cfg(backend, HWACCEL_QNN);
     }
 #endif
     // try to set locale for unicode characters in markdown