ggml-qnn: Windows port --- step3

Author: zhouwg

CMakeLists.txt

@@ -6,6 +6,7 @@ include(CheckIncludeFileCXX)
 set(CMAKE_WARN_UNUSED_CLI YES)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_VERBOSE_MAKEFILE on)
 
 if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)

examples/export-lora/export-lora.cpp

@@ -148,7 +148,7 @@ struct lora_merge_ctx {
 
         ctx_out = gguf_init_empty();
         struct ggml_init_params params = {
-            /*.mem_size   =*/ gguf_get_n_tensors(base_model.ctx_gguf)*ggml_tensor_overhead(),
+            /*.mem_size   =*/ static_cast<size_t>(gguf_get_n_tensors(base_model.ctx_gguf)*ggml_tensor_overhead()),
             /*.mem_buffer =*/ NULL,
             /*.no_alloc   =*/ true,
         };

ggml/src/ggml-qnn/CMakeLists.txt

@@ -0,0 +1,35 @@
+message(STATUS "Using QNN backend")
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Android")
+    find_library(LOG_LIB log)
+    set(QNN_LINK_LIBRARIES ${LOG_LIB})
+    set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend")
+elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+    set(QNN_DEFAULT_LIB_SEARCH_PATH "C:\\" CACHE STRING "customized library search path for QNN backend")
+else()
+    message(FATAL_ERROR "QNN now only available on Android and Windows(Windows on ARM)")
+endif()
+
+if(NOT DEFINED GGML_QNN_SDK_PATH)
+# try read from environment variable
+    if(DEFINED ENV{QNN_SDK_PATH})
+        set(GGML_QNN_SDK_PATH $ENV{QNN_SDK_PATH})
+    else()
+        message(FATAL_ERROR "GGML_QNN_SDK_PATH not defined")
+    endif()
+endif()
+
+message("QNN_SDK_PATH: ${GGML_QNN_SDK_PATH}")
+
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
+
+file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp")
+    ggml_add_backend_library(ggml-qnn
+    ${QNN_SOURCES}
+)
+
+target_include_directories(ggml-qnn PRIVATE ${GGML_QNN_SDK_PATH}/include/QNN ${CMAKE_CURRENT_LIST_DIR})
+target_link_libraries(ggml-qnn PRIVATE ${QNN_LINK_LIBRARIES})
+
+string(REGEX REPLACE "/$" "" GGML_QNN_DEFAULT_LIB_SEARCH_PATH "${QNN_DEFAULT_LIB_SEARCH_PATH}")
+target_compile_definitions(ggml-qnn PRIVATE GGML_QNN_DEFAULT_LIB_SEARCH_PATH="${QNN_DEFAULT_LIB_SEARCH_PATH}/")

ggml/src/ggml-qnn/ggml-qnn-impl.h

@@ -255,7 +255,9 @@ class qnn_perf {
 #else
 class qnn_perf {
 public:
-    qnn_perf(const std::string & perf_name) {}
+    qnn_perf(const std::string & perf_name) {
+        GGML_UNUSED(perf_name);
+    }
     qnn_perf() = delete;
     qnn_perf(const qnn_perf & ) = delete;
     qnn_perf & operator= (const qnn_perf & ) = delete;
@@ -287,86 +289,86 @@ class qnn_interface {
     qnn_interface() = default;
 
     // QnnBackend
-    DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate);
+    DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree);
+    DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage);
+    DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig);
+    DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion);
+    DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion)
 
     // QnnDevice
-    DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate);
+    DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree);
+    DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure);
+    DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo);
+    DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo);
+    DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo)
 
     // QnnContext
-    DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate);
+    DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize);
+    DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary);
+    DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary);
+    DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree);
+    DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree)
 
     // QnnGraph
-    DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate);
+    DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode);
+    DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize);
+    DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute);
+    DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve);
+    DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve)
 
     // QnnLog
-    DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate);
+    DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree);
+    DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel);
+    DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel)
 
     // QnnProfile
-    DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate);
+    DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents);
+    DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents);
+    DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData);
+    DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree);
+    DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree)
 
     // QnnMem
-    DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister);
+    DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister);
+    DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister)
 
     // QnnProperty
-    DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability);
+    DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability)
 
     // QnnTensor
-    DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor);
+    DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor)
 
-    DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor);
+    DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor)
 
     // QnnSystem
-    DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate);
+    DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate)
 
-    DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo);
+    DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo)
 
-    DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree);
+    DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree)
 
     void set_qnn_interface(const QnnInterface_t * qnn_interface) {
         _qnn_interface = qnn_interface;
@@ -398,7 +400,7 @@ class qnn_instance {
                           const std::string & model_name) :
             _lib_path(std::move(lib_path)),
             _backend_name(std::move(backend_name)),
-            _model_name(std::move(model_name)) {};
+            _model_name(std::move(model_name)) {}
 
     ~qnn_instance() {
     }
@@ -428,19 +430,19 @@ class qnn_instance {
         return _qnn_raw_system_interface;
     }
 
-    const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; }
+    Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; }
 
-    const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; }
+    Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; }
 
-    const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; }
+    Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; }
 
-    const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; }
+    Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; }
 
-    const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; }
+    Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; }
 
-    const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; }
+    QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; }
 
-    const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; }
+    Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; }
 
     int init_qnn_graph(const char * graph_name,
                        bool debug,