ggml-qnn: refine ggml_qnn_mul_mat and ggml_qnn_general_node according to Grok 3's style

Author: zhouwg

ggml/src/ggml-qnn/ggml-qnn-impl.h

@@ -99,7 +99,7 @@ void   ggmlqnn_log_internal(ggml_log_level level, const char * file, const char
 #else
 #define GGMLQNN_DEBUG                           1  // for troubleshooting QNN backend
 #define ENABLE_QNNBACKEND_PERF                  0  // enable/disable op's perf info
-#define GGMLQNN_PRINT_QNN_INTERNAL_LOG          1  // enable/disable QNN's internal log
+#define GGMLQNN_PRINT_QNN_INTERNAL_LOG          0  // enable/disable QNN's internal log
 #define GGMLQNN_PRINT_OP_ADD_LOG                0  // GGML_OP_ADD already verified with QNN-CPU / QNN-GPU / QNN-NPU
 #define GGMLQNN_PRINT_OP_MUL_MAT_LOG            1
 #endif

ggml/src/ggml-qnn/ggml-qnn-ops.cpp

@@ -200,71 +200,25 @@ void ggml_qnn_general_node(ggml_backend_qnn_context * ctx, ggml_tensor * op) {
 
         auto  graph_item = std::make_tuple(graph_handle, ggml_op_add_tensors);
         instance->_qnn_graph_map[graph_name] = graph_item;
-    } else {
-        Qnn_DataType_t src0_qnn_type    = QNN_DATATYPE_FLOAT_32;
-        Qnn_DataType_t src1_qnn_type    = QNN_DATATYPE_FLOAT_32;
-        Qnn_DataType_t dst_qnn_type     = QNN_DATATYPE_FLOAT_32;
-
-        src0_qnn_type                   = ggmlqnn_datatype_from_ggml_datatype(src0->type);
-        src1_qnn_type                   = ggmlqnn_datatype_from_ggml_datatype(src1->type);
-        dst_qnn_type                    = ggmlqnn_datatype_from_ggml_datatype(dst->type);
-
-        uint32_t dimensions_input_0[] = {(uint32_t) src0->ne[0], (uint32_t) src0->ne[1],
-                                         (uint32_t) src0->ne[2], (uint32_t) src0->ne[3]};
-        uint32_t dimensions_input_1[] = {(uint32_t) src1->ne[0], (uint32_t) src1->ne[1],
-                                         (uint32_t) src1->ne[2], (uint32_t) src1->ne[3]};
-        uint32_t dimensions_output[]  = {(uint32_t) dst->ne[0], (uint32_t) dst->ne[1],
-                                         (uint32_t) dst->ne[2], (uint32_t) dst->ne[3]};
-
-        QNN_VER_PTR(*p_tensor0)->dimensions  = dimensions_input_0;
-        QNN_VER_PTR(*p_tensor0)->rank        = ggml_n_dims(src0);
-        QNN_VER_PTR(*p_tensor0)->dataType    = src0_qnn_type;
-
-        QNN_VER_PTR(*p_tensor1)->dimensions  = dimensions_input_1;
-        QNN_VER_PTR(*p_tensor1)->rank        = ggml_n_dims(src1);
-        QNN_VER_PTR(*p_tensor1)->dataType    = src1_qnn_type;
-
-        QNN_VER_PTR(*p_tensor2)->dimensions  = dimensions_output;
-        QNN_VER_PTR(*p_tensor2)->rank        = ggml_n_dims(dst);
-        QNN_VER_PTR(*p_tensor2)->dataType    = dst_qnn_type;
-
-        if (enable_npu_rpc) {
-            //TODO: NPU RPC feature will failed with test-backend-ops
-            uint8_t * qnn_buffer_0 = static_cast<uint8_t *>(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor0)->memHandle));
-            GGMLQNN_LOG_INFO("qnn_rpcbuffer_0 = %p\n", qnn_buffer_0);
-            if (nullptr != qnn_buffer_0) {
-                memcpy(qnn_buffer_0, src0->data, ggml_nbytes(src0));
-            }
-
-            uint8_t * qnn_buffer_1 = static_cast<uint8_t *>(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor1)->memHandle));
-            GGMLQNN_LOG_INFO("qnn_rpcbuffer_1 = %p\n", qnn_buffer_1);
-            if (nullptr != qnn_buffer_1) {
-                memcpy(qnn_buffer_1, src1->data, ggml_nbytes(src1));
-            }
-        } else {
-            QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)};
-            QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)};
-            QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)};
-        }
-
-        Qnn_Tensor_t tensor_inputs[] = {
-                *p_tensor0,
-                *p_tensor1
-        };
-        Qnn_Tensor_t tensor_outputs[] = {
-                *p_tensor2
-        };
-        CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle,
-                                                            tensor_inputs, 2,
-                                                            tensor_outputs, 1,
-                                                            nullptr, nullptr));
+    }
 
-        if (enable_npu_rpc) {
-            //TODO:NPU RPC feature will failed with test-backend-ops
-            uint8_t * qnn_buffer_2 = static_cast<uint8_t *>(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle));
-            if (nullptr != qnn_buffer_2) {
-                memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst));
-            }
+    Qnn_Tensor_t tensor_inputs[] = {
+            *p_tensor0,
+            *p_tensor1
+    };
+    Qnn_Tensor_t tensor_outputs[] = {
+            *p_tensor2
+    };
+    CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle,
+                                                        tensor_inputs, 2,
+                                                        tensor_outputs, 1,
+                                                        nullptr, nullptr));
+
+    if (enable_npu_rpc) {
+        //TODO:NPU RPC feature will failed with test-backend-ops
+        uint8_t * qnn_buffer_2 = static_cast<uint8_t *>(instance->get_rpcmem_from_memhandle(QNN_VER_PTR(*p_tensor2)->memHandle));
+        if (nullptr != qnn_buffer_2) {
+            memcpy(dst->data, qnn_buffer_2, ggml_nbytes(dst));
         }
     }
 
@@ -461,12 +415,14 @@ static void ggml_qnn_mul_mat_4d(ggml_backend_qnn_context *ctx, ggml_tensor *op)
     CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle, input_tensors, 2,
                                                         output_tensors, 1, NULL, NULL));
 
+#if 0
     // Log dst for debugging
     float *dst_data = (float *)dst->data;
     GGMLQNN_LOG_DEBUG("dst shape: [%d, %d, %d, %d]\n", dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
     for (int i = 0; i < dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3]; i++) {
         GGMLQNN_LOG_DEBUG("dst[%d] = %f\n", i, dst_data[i]);
     }
+#endif
 
     op_perf.info();
 }
@@ -665,14 +621,8 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) {
 #endif
         CHECK_QNN_API(error, qnn_raw_interface.graphAddNode(graph_handle,out_trans1_0));
 
-        //step-6: finalize qnn graph and execute qnn graph
+        //step-6: finalize qnn graph
         CHECK_QNN_API(error, qnn_raw_interface.graphFinalize(graph_handle, nullptr, nullptr));
-        Qnn_Tensor_t input_tensors_0[]  = {*p_tensor0, *p_tensor1};
-        Qnn_Tensor_t output_tensors_0[] = {*p_tensor2};
-        CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle,
-                                                            input_tensors_0, 2,
-                                                            output_tensors_0, 1,
-                                                            nullptr, nullptr));
 
         qnn_tensors_t ggml_op_mulmat_tensors;
         ggml_op_mulmat_tensors.reserve(5);
@@ -683,30 +633,30 @@ void ggml_qnn_mul_mat(ggml_backend_qnn_context * ctx, ggml_tensor * op) {
         ggml_op_mulmat_tensors.push_back(p_tensor2_transpose);
         auto  graph_item = std::make_tuple(graph_handle, ggml_op_mulmat_tensors);
         instance->_qnn_graph_map[graph_name] = graph_item;
-    } else {
-        if (src0_type != GGML_TYPE_F32) {
-            QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast<uint32_t>(desired_size)};
-        } else {
-            QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)};
-        }
-        QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)};
-        QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)};
+    }
 
-        Qnn_Tensor_t tensor_inputs[] = {
-                *p_tensor0,
-                *p_tensor1
-        };
-        Qnn_Tensor_t tensor_outputs[] = {
-                *p_tensor2
-        };
-        // this is the second technical approach or another pipeline of "how to utilize the Hexagon
-        // NPU maximally" through QNN SDK, details could be found at
-        // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360
-        CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle,
-                                                            tensor_inputs, 2,
-                                                            tensor_outputs, 1,
-                                                            nullptr, nullptr));
+    if (src0_type != GGML_TYPE_F32) {
+        QNN_VER_PTR(*p_tensor0)->clientBuf = {wdata, static_cast<uint32_t>(desired_size)};
+    } else {
+        QNN_VER_PTR(*p_tensor0)->clientBuf = {src0->data, ggmlqnn_get_tensor_data_size(src0)};
     }
+    QNN_VER_PTR(*p_tensor1)->clientBuf = {src1->data, ggmlqnn_get_tensor_data_size(src1)};
+    QNN_VER_PTR(*p_tensor2)->clientBuf = {dst->data, ggmlqnn_get_tensor_data_size(dst)};
+
+    Qnn_Tensor_t tensor_inputs[] = {
+            *p_tensor0,
+            *p_tensor1
+    };
+    Qnn_Tensor_t tensor_outputs[] = {
+            *p_tensor2
+    };
+    // this is the second technical approach or another pipeline of "how to utilize the Hexagon
+    // NPU maximally" through QNN SDK, details could be found at
+    // https://github.com/ggml-org/llama.cpp/pull/12049#issuecomment-2678308360
+    CHECK_QNN_API(error, qnn_raw_interface.graphExecute(graph_handle,
+                                                        tensor_inputs, 2,
+                                                        tensor_outputs, 1,
+                                                        nullptr, nullptr));
 
     // restore the original dimensions of qnn tensors to avoid memory leak in func free_qnn_tensor
     QNN_VER_PTR(*p_tensor0)->dimensions = tensor_0_dimensions;

tests/ggml-qnn-ut.cpp

@@ -332,37 +332,36 @@ int main(int argc, char * argv[]) {
     std::vector<ggml_backend_ptr> backends;
     std::vector<std::pair<ggml_backend_t, ggml_backend_set_n_threads_t>> set_n_threads_fns;
     printf("Testing %zu devices\n\n", ggml_backend_dev_count());
-    //for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
-    for (size_t i = 0; i < 2; i++) {
-            ggml_backend_dev_t dev = ggml_backend_dev_get(i);
+    for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
+        ggml_backend_dev_t dev = ggml_backend_dev_get(i);
 
-            printf("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(),
-                   ggml_backend_dev_name(dev));
+        printf("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(),
+               ggml_backend_dev_name(dev));
 
-            if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
-                printf("  Skipping CPU backend\n");
-                continue;
-            }
+        if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
+            printf("  Skipping CPU backend\n");
+            continue;
+        }
 
-            backend = ggml_backend_dev_init(dev, reinterpret_cast<const char *>(i));
-            GGML_ASSERT(backend != NULL);
-            if (backend != nullptr) {
-                printf("%s: initialize %s backend\n", __func__, ggml_backend_dev_name(dev));
-            }
-            backends.emplace_back(backend);
+        backend = ggml_backend_dev_init(dev, reinterpret_cast<const char *>(i));
+        GGML_ASSERT(backend != NULL);
+        if (backend != nullptr) {
+            printf("%s: initialize %s backend\n", __func__, ggml_backend_dev_name(dev));
+        }
+        backends.emplace_back(backend);
 
-            ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
-            auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(
-                    reg, "ggml_backend_set_n_threads");
-            if (ggml_backend_set_n_threads_fn) {
-                ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency());
-            }
+        ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
+        auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(
+                reg, "ggml_backend_set_n_threads");
+        if (ggml_backend_set_n_threads_fn) {
+            ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency());
+        }
 
-            printf("  Device description: %s\n", ggml_backend_dev_description(dev));
-            size_t free, total;
-            ggml_backend_dev_memory(dev, &free, &total);
-            printf("  Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024);
-            printf("\n");
+        printf("  Device description: %s\n", ggml_backend_dev_description(dev));
+        size_t free, total;
+        ggml_backend_dev_memory(dev, &free, &total);
+        printf("  Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024);
+        printf("\n");
     }
 
     ggml_backend_t backend_cpu = nullptr;