Skip to content

Commit beff5c4

Browse files
chraacchraac
authored
feat: op perf opt (#38)
* add op define xml * copy qnn libs in cmake * fix htp skel path * add windows copy file list * wip * add generated package * remove unused params * add cmake list * set qnn sdk and hexagon sdk path * wip * wip * fix tools version * fix compiling error * fix dims calc * wip * add mulmat 2d * wip * reduction * wip * wip * fix compiling error in x64 * wip * fix device description in emulator * wip * add flag * copy necessary libs * wip * load HtpPrepare first for emulator * enable custom op for 2d matrix * verify op config before add to node * Revert "verify op config before add to node" This reverts commit 206dec8. * wip * wip * wip * revert tool version change * use hexagon sdk version 5.5.0 https://docs.qualcomm.com/bundle/publicresource/topics/80-77512-2/release-notes-wrapper.html?product=1601111740010422#5.5.0 * wip * move to sub dir * add hexagon npu device and server lib * fix npu lib build * refactoring: rename QNNBackend enum * fix compiling error * wip * remove qnn/backend.hpp * add hexagon dsp host layer * extract rpc_mem from qnn submodule * fix dsp compiling error * wip * wip * open and lose npu device * split objects into separated files * fix linking error * add npu_tensor * add host graph * map rpc buffer before usage * fix some todos * add shared module * split rpc_interface from rpc_mem * get get_dsp_arch from device * wip * rename host classes * fix hexagon sdk arch getter * fix device open * fix linking error * fix crash * use tensor_data_type * fix npu lib crash * fix debug log print * skip empty graph * wip * add log * fix unmap fail * fix tensor set * remove some logs * flush back memory after finished * fix nb * wip * wip * add helper function * impl add op * fix some add in test-backend-ops * add elt wise sub and mul * fix crash on some inplace op * wip * fix elt wise op calc * wip * split mul_mat into file * add caps array * wip * wip * print support/unsupport op * copy lldb-server for newer android sdk * add tensor_spec * add assert * fix crash when loading model * rename cmake option * fix name * fix device memory and description * fix compiling error on qnn only build * fix some potential UBs * fix comments
1 parent 9e41f79 commit beff5c4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+4334
-484
lines changed

‎ggml/include/ggml-qnn.h‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,11 @@
11
#pragma once
22

33
#include "ggml-backend.h"
4-
#include "ggml.h"
54

65
#ifdef __cplusplus
76
extern "C" {
87
#endif
98

10-
#define GGML_QNN_NAME "qnn"
11-
#define GGML_QNN_MAX_DEVICES QNN_BACKEND_COUNT
12-
13-
enum QNNBackend {
14-
QNN_BACKEND_CPU = 0,
15-
QNN_BACKEND_GPU,
16-
QNN_BACKEND_NPU,
17-
QNN_BACKEND_COUNT,
18-
};
19-
20-
GGML_API bool ggml_backend_is_qnn(ggml_backend_t backend);
21-
229
GGML_API ggml_backend_reg_t ggml_backend_qnn_reg(void);
2310

2411
#ifdef __cplusplus

‎ggml/src/ggml-qnn/CMakeLists.txt‎

Lines changed: 109 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
message(STATUS "Using QNN backend")
22

3+
option(GGML_HEXAGON_NPU_ONLY "ggml-qnn: Only use Hexagon NPU" OFF)
4+
option(GGML_QNN_ENABLE_HEXAGON_BACKEND "ggml-qnn: Enable Hexagon custom package" ${GGML_HEXAGON_NPU_ONLY})
5+
36
if(CMAKE_SYSTEM_NAME STREQUAL "Android")
47
find_library(LOG_LIB log)
58
set(QNN_LINK_LIBRARIES ${LOG_LIB})
69
set(QNN_DEFAULT_LIB_SEARCH_PATH "/data/local/tmp/" CACHE STRING "customized library search path for QNN backend")
10+
add_compile_options(-g -O0)
711
elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows" OR CMAKE_SYSTEM_NAME STREQUAL "Linux")
812
set(QNN_DEFAULT_LIB_SEARCH_PATH "" CACHE STRING "customized library search path for QNN backend")
913
else()
@@ -21,15 +25,22 @@ if(NOT DEFINED GGML_QNN_SDK_PATH)
2125
endif()
2226

2327
message("CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
28+
message("CMAKE_CXX_FLAGS_DEBUG: ${CMAKE_CXX_FLAGS_DEBUG}")
2429
message("CMAKE_CXX_FLAGS_RELEASE: ${CMAKE_CXX_FLAGS_RELEASE}")
2530
message("QNN_SDK_PATH: ${GGML_QNN_SDK_PATH}")
2631

27-
file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp")
32+
file(GLOB QNN_SOURCES "${CMAKE_CURRENT_LIST_DIR}/qnn/*.cpp")
33+
file(GLOB COMMON_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.cpp")
2834
ggml_add_backend_library(ggml-qnn
2935
${QNN_SOURCES}
36+
${COMMON_SOURCES}
3037
)
3138

32-
target_include_directories(ggml-qnn PRIVATE ${GGML_QNN_SDK_PATH}/include/QNN ${CMAKE_CURRENT_LIST_DIR})
39+
target_include_directories(ggml-qnn PRIVATE
40+
${GGML_QNN_SDK_PATH}/include/QNN
41+
${CMAKE_CURRENT_LIST_DIR}/qnn
42+
${CMAKE_CURRENT_LIST_DIR}
43+
)
3344
target_link_libraries(ggml-qnn PRIVATE ${QNN_LINK_LIBRARIES})
3445

3546
if(NOT "${QNN_DEFAULT_LIB_SEARCH_PATH}" STREQUAL "")
@@ -52,3 +63,99 @@ if(GGML_QNN_ENABLE_PERFORMANCE_TRACKING)
5263
else()
5364
message("GGML_QNN_ENABLE_PERFORMANCE_TRACKING is disabled")
5465
endif()
66+
67+
add_subdirectory(shared)
68+
69+
if(GGML_HEXAGON_NPU_ONLY)
70+
message("GGML_HEXAGON_NPU_ONLY is enabled")
71+
add_compile_definitions(GGML_HEXAGON_NPU_ONLY)
72+
set(GGML_QNN_ENABLE_HEXAGON_BACKEND ON)
73+
else()
74+
message("GGML_HEXAGON_NPU_ONLY is disabled")
75+
endif()
76+
77+
if(GGML_QNN_ENABLE_HEXAGON_BACKEND)
78+
message("GGML_QNN_ENABLE_HEXAGON_BACKEND is enabled")
79+
add_subdirectory(npu)
80+
target_link_libraries(hexagon-npu-host runtime-common)
81+
target_link_libraries(ggml-qnn PRIVATE hexagon-npu-host)
82+
else()
83+
message("GGML_QNN_ENABLE_HEXAGON_BACKEND is disabled")
84+
target_link_libraries(ggml-qnn PRIVATE runtime-common)
85+
endif()
86+
87+
# Copy QNN dynamic libraries
88+
set(QNN_DYNAMIC_LIBS "")
89+
90+
if(CMAKE_SYSTEM_NAME STREQUAL "Android" OR CMAKE_SYSTEM_NAME STREQUAL "Linux")
91+
if(CMAKE_SYSTEM_NAME STREQUAL "Android")
92+
# Android
93+
set(QNN_SDK_LIB_PATH "${GGML_QNN_SDK_PATH}/lib/aarch64-android")
94+
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
95+
# Linux x86_64
96+
set(QNN_SDK_LIB_PATH "${GGML_QNN_SDK_PATH}/lib/x86_64-linux-clang")
97+
else()
98+
# Linux aarch64
99+
set(QNN_SDK_LIB_PATH "${GGML_QNN_SDK_PATH}/lib/aarch64-oe-linux-gcc11.2")
100+
endif()
101+
102+
list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/libQnnSystem.so")
103+
list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/libQnnCpu.so")
104+
list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/libQnnGpu.so")
105+
list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/libQnnHtp.so")
106+
file(GLOB HTP_STUB_LIBS "${QNN_SDK_LIB_PATH}/libQnnHtp*.so")
107+
list(APPEND QNN_DYNAMIC_LIBS ${HTP_STUB_LIBS})
108+
109+
if(CMAKE_SYSTEM_NAME STREQUAL "Android")
110+
file(GLOB HTP_SKEL_LIBS "${GGML_QNN_SDK_PATH}/lib/hexagon-*/unsigned/libQnnHtp*Skel.so")
111+
list(APPEND QNN_DYNAMIC_LIBS ${HTP_SKEL_LIBS})
112+
113+
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
114+
if(EXISTS "${CMAKE_ANDROID_NDK}/prebuilt/android-arm64/gdbserver/gdbserver")
115+
list(APPEND QNN_DYNAMIC_LIBS "${CMAKE_ANDROID_NDK}/prebuilt/android-arm64/gdbserver/gdbserver")
116+
message("old ndk, copy gdbserver")
117+
else()
118+
file(GLOB LLDB_SERVER "${CMAKE_ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/*/lib/linux/aarch64/lldb-server")
119+
list(APPEND QNN_DYNAMIC_LIBS ${LLDB_SERVER})
120+
message("new ndk, copy lldb-server")
121+
endif()
122+
123+
file(GLOB OMP_LIBS "${CMAKE_ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/*/lib/linux/aarch64/libomp.so")
124+
file(GLOB ASAN_LIBS "${CMAKE_ANDROID_NDK}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/*/lib/linux/libclang_rt.asan-aarch64-android.so")
125+
list(APPEND QNN_DYNAMIC_LIBS ${OMP_LIBS})
126+
list(APPEND QNN_DYNAMIC_LIBS ${ASAN_LIBS})
127+
endif()
128+
else()
129+
# Linux
130+
list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/libHtpPrepare.so")
131+
endif()
132+
elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
133+
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
134+
# x86_64
135+
set(QNN_SDK_LIB_PATH "${GGML_QNN_SDK_PATH}/lib/x86_64-windows-msvc")
136+
else()
137+
# aarch64
138+
set(QNN_SDK_LIB_PATH "${GGML_QNN_SDK_PATH}/lib/aarch64-windows-msvc")
139+
endif()
140+
141+
list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/QnnSystem.dll")
142+
list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/QnnCpu.dll")
143+
list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/QnnGpu.dll")
144+
list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/QnnHtp.dll")
145+
file(GLOB HTP_STUB_LIBS "${QNN_SDK_LIB_PATH}/QnnHtp*.dll")
146+
147+
if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
148+
list(APPEND QNN_DYNAMIC_LIBS "${QNN_SDK_LIB_PATH}/HtpPrepare.dll")
149+
endif()
150+
151+
list(APPEND QNN_DYNAMIC_LIBS ${HTP_STUB_LIBS})
152+
endif()
153+
154+
foreach(QNN_DYNAMIC_LIB ${QNN_DYNAMIC_LIBS})
155+
message("Copy: ${QNN_DYNAMIC_LIB} -> ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")
156+
add_custom_command(
157+
TARGET ggml-qnn POST_BUILD
158+
COMMAND ${CMAKE_COMMAND} -E copy
159+
${QNN_DYNAMIC_LIB}
160+
${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
161+
endforeach()

‎ggml/src/ggml-qnn/backend-ops.hpp‎

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎ggml/src/ggml-qnn/npu/CMakeLists.txt‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
enable_language(ASM)
2+
cmake_policy(SET CMP0115 OLD)
3+
4+
if(DEFINED ENV{HEXAGON_SDK_ROOT})
5+
set(HEXAGON_SDK_ROOT $ENV{HEXAGON_SDK_ROOT})
6+
message("HEXAGON_SDK_ROOT: ${HEXAGON_SDK_ROOT}")
7+
else()
8+
message(FATAL_ERROR "HEXAGON_SDK_ROOT not defined")
9+
endif()
10+
11+
if(HEXAGON_SDK_ROOT)
12+
include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake)
13+
else()
14+
include(${HEXAGON_CMAKE_ROOT}/hexagon_fun.cmake)
15+
endif()
16+
17+
# Base Include dirs for the Project
18+
set(common_incs
19+
${CMAKE_CURRENT_BINARY_DIR}/
20+
${HEXAGON_SDK_ROOT}/incs/
21+
${HEXAGON_SDK_ROOT}/incs/stddef/
22+
${HEXAGON_SDK_ROOT}/incs/HAP/
23+
${HEXAGON_SDK_ROOT}/rtos/qurt/
24+
${HEXAGON_SDK_ROOT}/utils/examples/
25+
)
26+
27+
include_directories(${common_incs})
28+
29+
if(${CMAKE_SYSTEM_NAME} MATCHES "Android|Linux|Windows")
30+
# host build
31+
file(GLOB common_srcs "${CMAKE_CURRENT_LIST_DIR}/common/*.cpp")
32+
file(GLOB host_srcs "${CMAKE_CURRENT_LIST_DIR}/host/*.cpp")
33+
set(stub_srcs "${CMAKE_CURRENT_BINARY_DIR}/npu_device_stub.c")
34+
add_library(hexagon-npu-host STATIC
35+
${common_srcs}
36+
${host_srcs}
37+
${stub_srcs}
38+
)
39+
40+
# disable warnings for the stub
41+
set_source_files_properties(
42+
${stub_srcs}
43+
PROPERTIES
44+
COMPILE_FLAGS "-w"
45+
)
46+
47+
build_idl(idl/hexagon_npu.idl hexagon-npu-host)
48+
49+
# Add compile definitions to the target
50+
target_compile_definitions(hexagon-npu-host PUBLIC
51+
VERIFY_PRINT_ERROR
52+
GGML_QNN_ENABLE_HEXAGON_BACKEND
53+
)
54+
55+
target_include_directories(hexagon-npu-host PRIVATE
56+
${HEXAGON_SDK_ROOT}/ipc/fastrpc/rpcmem/inc/
57+
${QNN_SDK_ROOT}/include/QNN/
58+
${CMAKE_CURRENT_LIST_DIR}/host/
59+
${CMAKE_CURRENT_LIST_DIR}/
60+
)
61+
62+
target_include_directories(hexagon-npu-host PUBLIC
63+
${HEXAGON_SDK_ROOT}/incs/ # TODO: this is for rpc-mem
64+
)
65+
66+
if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
67+
set_target_properties(hexagon-npu-host PROPERTIES OUTPUT_NAME "hexagon_npu")
68+
endif()
69+
70+
if(${CMAKE_SYSTEM_NAME} MATCHES "Android|Linux")
71+
target_link_options(hexagon-npu-host PUBLIC -pie)
72+
endif()
73+
74+
link_options(hexagon-npu-host)
75+
76+
if(${CMAKE_SYSTEM_NAME} MATCHES "Android")
77+
set(PREBUILT_LIB_DIR "android_aarch64")
78+
elseif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
79+
set(PREBUILT_LIB_DIR "UbuntuARM_aarch64")
80+
else()
81+
# Windows
82+
set(PREBUILT_LIB_DIR "windows_aarch64")
83+
endif()
84+
85+
choose_dsprpc("3" dsprpc) # cdsprpc
86+
link_custom_library(hexagon-npu-host ${dsprpc})
87+
else()
88+
# hexagon npu build
89+
cmake_minimum_required(VERSION 3.14.3)
90+
project(hexagon_npu C CXX ASM)
91+
92+
# check if QNN_SDK_ROOT is set
93+
if(NOT DEFINED ENV{QNN_SDK_ROOT})
94+
message(FATAL_ERROR "QNN_SDK_ROOT not defined")
95+
endif()
96+
97+
set(QNN_SDK_ROOT $ENV{QNN_SDK_ROOT})
98+
message("QNN_SDK_ROOT: ${QNN_SDK_ROOT}")
99+
include_directories(
100+
${QNN_SDK_ROOT}/include/QNN/
101+
)
102+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
103+
104+
file(GLOB common_srcs "${CMAKE_CURRENT_LIST_DIR}/common/*.cpp")
105+
file(GLOB device_srcs "${CMAKE_CURRENT_LIST_DIR}/device/*.cpp")
106+
set(skel_srcs "${CMAKE_CURRENT_BINARY_DIR}/npu_device_skel.c")
107+
add_library(hexagon_npu_skel_OBJS OBJECT
108+
${common_srcs}
109+
${device_srcs}
110+
${skel_srcs}
111+
)
112+
113+
if(CMAKE_BUILD_TYPE MATCHES "Debug|Dbg")
114+
message("Debug build, enable all logging")
115+
target_compile_definitions(hexagon_npu_skel_OBJS PUBLIC
116+
_DEBUG
117+
DEBUG_LOGGING
118+
)
119+
else()
120+
message("Release build, disable debug logging")
121+
target_compile_definitions(hexagon_npu_skel_OBJS PUBLIC
122+
NDEBUG
123+
RELEASE_LOGGING
124+
)
125+
endif()
126+
127+
build_idl(idl/hexagon_npu.idl hexagon_npu_skel_OBJS)
128+
129+
# disable warnings for the skel
130+
set_source_files_properties(
131+
${skel_srcs}
132+
PROPERTIES
133+
COMPILE_FLAGS "-w"
134+
)
135+
136+
add_library(hexagon_npu_skel SHARED $<TARGET_OBJECTS:hexagon_npu_skel_OBJS>)
137+
138+
target_link_libraries(hexagon_npu_skel
139+
${HEXAGON_LIB_DIR}/${HEXAGON_ARCH}/G0/pic/libc++abi.a
140+
${HEXAGON_LIB_DIR}/${HEXAGON_ARCH}/G0/pic/libc++.a
141+
)
142+
set_target_properties(hexagon_npu_skel PROPERTIES OUTPUT_NAME "hexagon_npu_skel_${HEXAGON_ARCH}")
143+
144+
copy_binaries(hexagon_npu_skel)
145+
endif()
146+
147+
# vim: set noet fenc=utf-8 ff=unix ft=cmake :

0 commit comments

Comments
 (0)