Dev refactoring

CMakeLists.txt

@@ -107,6 +107,7 @@ llama_option_depr(WARNING     LLAMA_RPC                 GGML_RPC)
 llama_option_depr(WARNING     LLAMA_SYCL                GGML_SYCL)
 llama_option_depr(WARNING     LLAMA_SYCL_F16            GGML_SYCL_F16)
 llama_option_depr(WARNING     LLAMA_CANN                GGML_CANN)
+llama_option_depr(WARNING     LLAMA_QNN                 GGML_QNN)
 
 #
 # build the library

ggml/CMakeLists.txt

@@ -153,6 +153,7 @@ option(GGML_SYCL                            "ggml: use SYCL"
 option(GGML_SYCL_F16                        "ggml: use 16 bit floats for sycl calculations"   OFF)
 set   (GGML_SYCL_TARGET "INTEL" CACHE STRING
                                             "ggml: sycl target device")
+option(GGML_QNN                             "ggml: use QNN"                                   OFF)
 
 # extra artifacts
 option(GGML_BUILD_TESTS    "ggml: build tests"    ${GGML_STANDALONE})
@@ -165,7 +166,7 @@ option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})
 set(CMAKE_C_STANDARD 11)
 set(CMAKE_C_STANDARD_REQUIRED true)
 
-if (GGML_SYCL)
+if (GGML_SYCL OR GGML_QNN)
     set(CMAKE_CXX_STANDARD 17)
 else()
     set(CMAKE_CXX_STANDARD 11)

ggml/include/ggml-qnn.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "ggml.h"
+
+#include "ggml-backend.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define GGML_QNN_MAX_DEVICES 3
+
+enum QNNBackend {
+    QNN_BACKEND_CPU = 0,
+    QNN_BACKEND_GPU,
+    QNN_BACKEND_NPU,
+    QNN_BACKEND_GGML, //"fake" QNN backend, used for compare performance between
+                      // QNN and original GGML
+};
+
+/**
+ *
+ * @param device                      0: QNN_BACKEND_CPU 1: QNN_BACKEND_GPU 2:QNN_BACKEND_NPU
+ * @param extend_lib_search_path      extened lib search path for searching QNN backend dynamic libs
+ * @return
+ */
+GGML_API ggml_backend_t ggml_backend_qnn_init(size_t dev_num, const char *extend_lib_search_path);
+
+GGML_API bool ggml_backend_is_qnn(ggml_backend_t backend);
+
+GGML_API void ggml_backend_qnn_set_n_threads(ggml_backend_t backend, int thread_counts);
+
+GGML_API int ggml_backend_qnn_get_device_count(void);
+
+GGML_API void ggml_backend_qnn_get_device_description(size_t dev_num, char *description, size_t description_size);
+
+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_qnn_buffer_type(size_t dev_num);
+
+#ifdef __cplusplus
+}
+#endif

ggml/src/CMakeLists.txt

@@ -898,6 +898,36 @@ if (GGML_CANN)
     endif()
 endif()
 
+if (GGML_QNN)
+    if (CMAKE_SYSTEM_NAME STREQUAL "Android")
+        find_library(LOG_LIB log)
+        find_library(ANDROID_LIB android)
+        set(GGML_EXTRA_LIBS ${GGML_EXTRA_LIBS} ${LOG_LIB} ${ANDROID_LIB})
+        set(GGML_QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend")
+    else()
+        message(FATAL_ERROR "QNN now only available on Android")
+    endif()
+
+    string(REGEX REPLACE "/$" "" GGML_QNN_DEFAULT_LIB_SEARCH_PATH "${GGML_QNN_DEFAULT_LIB_SEARCH_PATH}")
+    add_compile_definitions(GGML_QNN_DEFAULT_LIB_SEARCH_PATH="${GGML_QNN_DEFAULT_LIB_SEARCH_PATH}/")
+    if (NOT DEFINED GGML_QNN_SDK_PATH)
+        # try read from environment variable
+        if (DEFINED ENV{QNN_SDK_PATH})
+            set(GGML_QNN_SDK_PATH $ENV{QNN_SDK_PATH})
+        else()
+            message(FATAL_ERROR "GGML_QNN_SDK_PATH not defined")
+        endif()
+    endif()
+
+    message("QNN_SDK_PATH: ${GGML_QNN_SDK_PATH}")
+    file(GLOB   GGML_SOURCES_QNN "ggml-qnn/*.cpp")
+    list(APPEND GGML_SOURCES_QNN "ggml-qnn.cpp")
+    set(GGML_HEADERS_QNN ../include/ggml-qnn.h)
+    set(QNN_INC_PATH ${GGML_QNN_SDK_PATH}/include/QNN)
+    set(GGML_EXTRA_INCLUDES ${GGML_EXTRA_INCLUDES} ${QNN_INC_PATH} "ggml-qnn")
+    list(APPEND GGML_CDEF_PUBLIC GGML_USE_QNN)
+endif()
+
 function(get_flags CCID CCVER)
     set(C_FLAGS "")
     set(CXX_FLAGS "")
@@ -1315,6 +1345,7 @@ add_library(ggml
             ${GGML_SOURCES_BLAS}      ${GGML_HEADERS_BLAS}
             ${GGML_SOURCES_LLAMAFILE} ${GGML_HEADERS_LLAMAFILE}
             ${GGML_SOURCES_CANN}      ${GGML_HEADERS_CANN}
+            ${GGML_SOURCES_QNN}       ${GGML_HEADERS_QNN}
             ggml-aarch64.c            ggml-aarch64.h
             )
 

ggml/src/ggml-backend.c

@@ -449,6 +449,11 @@ GGML_CALL static void ggml_backend_registry_init(void) {
     extern GGML_CALL int ggml_backend_cann_reg_devices(void);
     ggml_backend_cann_reg_devices();
 #endif
+
+#ifdef GGML_USE_QNN
+    extern GGML_CALL void ggml_backend_qnn_reg_devices(void);
+    ggml_backend_qnn_reg_devices();
+#endif
 }
 
 GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {