Skip to content

Commit 67db4a4

Browse files
haowhsu-quicchenweng-quic
haowhsu-quic
authored and
chenweng-quic
committed
Qualcomm AI Engine Direct - gpu support part1
- rename folders in backends/qualcomm/runtime/backends - add gpu infra
1 parent dc5092b commit 67db4a4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+887
-73
lines changed

‎backends/qualcomm/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ backends/qualcomm
5555
| | # Meanwhile, this is also the runtime responsbile for executing compiled
5656
| | # models on a device.
5757
| └── backends # Backends supported by QNN.
58-
| └── htpbackend
58+
| └── gpu / htp
5959
| ├── aarch64 # Configuration required to run on device. (Device Part).
6060
| └── x86_64 # Configuration required to compile graph on host. (AoT Part).
6161
├── scripts # Misc supporting scripts, not related to core functionality.

‎backends/qualcomm/runtime/backends/CMakeLists.txt‎

Lines changed: 44 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -43,58 +43,73 @@ target_sources(
4343
${CMAKE_CURRENT_LIST_DIR}/QnnProfiler.cpp
4444
)
4545

46-
# qnn_device
47-
set(HOST_ARCHITECTURE
48-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/${CMAKE_SYSTEM_PROCESSOR}
46+
set(HOST_ARCHITECTURE_GPU
47+
${CMAKE_CURRENT_LIST_DIR}/gpu/${CMAKE_SYSTEM_PROCESSOR}
48+
)
49+
set(HOST_ARCHITECTURE_HTP
50+
${CMAKE_CURRENT_LIST_DIR}/htp/${CMAKE_SYSTEM_PROCESSOR}
51+
)
52+
set(HOST_ARCHITECTURE_IR
53+
${CMAKE_CURRENT_LIST_DIR}/ir/${CMAKE_SYSTEM_PROCESSOR}
4954
)
5055

56+
# qnn_device
5157
target_sources(
5258
qnn_device
5359
PUBLIC ${CMAKE_CURRENT_LIST_DIR}/QnnDeviceCommon.h
54-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpDevice.h
60+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuDevice.h
61+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpDevice.h
5562
PRIVATE ${CMAKE_CURRENT_LIST_DIR}/QnnDeviceCommon.cpp
56-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpDevice.cpp
57-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpDevicePlatformInfoConfig.h
58-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpDeviceCustomConfig.h
63+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpDevice.cpp
64+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpDevicePlatformInfoConfig.h
65+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpDeviceCustomConfig.h
5966
# When offline prepare context cache in x86 host we have to provide
6067
# platform infomation and SocModel to Qnn
61-
${HOST_ARCHITECTURE}/HtpDevicePlatformInfoConfig.cpp
62-
${HOST_ARCHITECTURE}/HtpDeviceCustomConfig.cpp
68+
${HOST_ARCHITECTURE_HTP}/HtpDevicePlatformInfoConfig.cpp
69+
${HOST_ARCHITECTURE_HTP}/HtpDeviceCustomConfig.cpp
6370
)
6471

6572
# qnn_context
6673
target_sources(
6774
qnn_context
6875
PUBLIC ${CMAKE_CURRENT_LIST_DIR}/QnnContextCommon.h
69-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpContext.h
70-
${CMAKE_CURRENT_LIST_DIR}/irbackend/IrContext.h
76+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpContext.h
77+
${CMAKE_CURRENT_LIST_DIR}/ir/IrContext.h
78+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuContext.h
7179
PRIVATE
7280
${CMAKE_CURRENT_LIST_DIR}/QnnContextCommon.cpp
73-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpContext.cpp
74-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpContextCustomConfig.h
75-
${HOST_ARCHITECTURE}/HtpContextCustomConfig.cpp
76-
${CMAKE_CURRENT_LIST_DIR}/irbackend/${CMAKE_SYSTEM_PROCESSOR}/IrContext.cpp
81+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpContext.cpp
82+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpContextCustomConfig.h
83+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuContext.cpp
84+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuContextCustomConfig.h
85+
${HOST_ARCHITECTURE_GPU}/GpuContextCustomConfig.cpp
86+
${HOST_ARCHITECTURE_HTP}/HtpContextCustomConfig.cpp
87+
${HOST_ARCHITECTURE_IR}/IrContext.cpp
7788
)
7889

7990
# qnn_backend_cache
8091
target_sources(
8192
qnn_backend_cache
8293
PUBLIC ${CMAKE_CURRENT_LIST_DIR}/QnnBackendCache.h
83-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpBackendCache.h
94+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpBackendCache.h
8495
PRIVATE ${CMAKE_CURRENT_LIST_DIR}/QnnBackendCache.cpp
85-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpBackendCache.cpp
96+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpBackendCache.cpp
8697
)
8798

8899
# qnn_graph
89100
target_sources(
90101
qnn_graph
91102
PUBLIC ${CMAKE_CURRENT_LIST_DIR}/QnnGraphCommon.h
92-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpGraph.h
103+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuGraph.h
104+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpGraph.h
93105
PRIVATE ${CMAKE_CURRENT_LIST_DIR}/QnnGraphCommon.cpp
94-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpGraph.cpp
95-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpGraphCustomConfig.h
96-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpGraphCustomConfig.cpp
97-
${HOST_ARCHITECTURE}/HtpGraphCustomConfig.cpp
106+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuGraph.cpp
107+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuGraphCustomConfig.h
108+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuGraphCustomConfig.cpp
109+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpGraph.cpp
110+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpGraphCustomConfig.h
111+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpGraphCustomConfig.cpp
112+
${HOST_ARCHITECTURE_HTP}/HtpGraphCustomConfig.cpp
98113
)
99114

100115
# qnn_op_package_manager
@@ -108,9 +123,13 @@ target_sources(
108123
target_sources(
109124
qnn_backend
110125
PUBLIC ${CMAKE_CURRENT_LIST_DIR}/QnnBackendCommon.h
111-
${CMAKE_CURRENT_LIST_DIR}/htpbackend/HtpBackend.h
112-
${CMAKE_CURRENT_LIST_DIR}/irbackend/IrBackend.h
126+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuBackend.h
127+
${CMAKE_CURRENT_LIST_DIR}/htp/HtpBackend.h
128+
${CMAKE_CURRENT_LIST_DIR}/ir/IrBackend.h
113129
PRIVATE ${CMAKE_CURRENT_LIST_DIR}/QnnBackendCommon.cpp
130+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuBackend.cpp
131+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuBackendCustomConfig.h
132+
${CMAKE_CURRENT_LIST_DIR}/gpu/GpuBackendCustomConfig.cpp
114133
)
115134

116135
# qnn_mem_manager
@@ -139,5 +158,5 @@ target_sources(
139158
qnn_dlc_manager
140159
PUBLIC ${CMAKE_CURRENT_LIST_DIR}/QnnDlcManager.h
141160
PRIVATE
142-
${CMAKE_CURRENT_LIST_DIR}/irbackend/${CMAKE_SYSTEM_PROCESSOR}/QnnDlcManager.cpp
161+
${HOST_ARCHITECTURE_IR}/QnnDlcManager.cpp
143162
)

‎backends/qualcomm/runtime/backends/QnnBackendFactory.cpp‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,16 @@ std::unique_ptr<BackendConfigParameters> QnnBackendFactory::Create(
5353
EnumNameQnnExecuTorchHtpPdSession(htp_options->pd_session()));
5454
QNN_EXECUTORCH_LOG_INFO(
5555
"use_conv_hmx in htp_options: %d", htp_options->use_conv_hmx());
56+
QNN_EXECUTORCH_LOG_INFO(
57+
"use_dlbc in htp_options: %d", htp_options->use_dlbc());
5658
QNN_EXECUTORCH_LOG_INFO(
5759
"use_fold_relu in htp_options: %d", htp_options->use_fold_relu());
60+
QNN_EXECUTORCH_LOG_INFO(
61+
"use_multi_contexts in htp_options: %d",
62+
htp_options->use_multi_contexts());
63+
QNN_EXECUTORCH_LOG_INFO(
64+
"use_weight_sharing in htp_options: %d",
65+
htp_options->use_weight_sharing());
5866
}
5967
backend_params->qnn_backend_ptr_ =
6068
std::make_unique<HtpBackend>(implementation, logger);
@@ -86,13 +94,66 @@ std::unique_ptr<BackendConfigParameters> QnnBackendFactory::Create(
8694
get_option(options->log_level()));
8795
backend_params->backend_init_state_ = BackendInitializeState::INITIALIZED;
8896
} break;
89-
case QnnExecuTorchBackendType::kGpuBackend:
97+
case QnnExecuTorchBackendType::kGpuBackend: {
98+
auto gpu_options = options->backend_options()->gpu_options();
99+
if (options->log_level() >= QnnExecuTorchLogLevel::kLogLevelInfo) {
100+
QNN_EXECUTORCH_LOG_INFO(
101+
"performance_mode in gpu_options: %s",
102+
EnumNameQnnExecuTorchGpuPerformanceMode(
103+
gpu_options->performance_mode()));
104+
QNN_EXECUTORCH_LOG_INFO(
105+
"precision in gpu_options: %s",
106+
EnumNameQnnExecuTorchGpuPrecision(gpu_options->precision()));
107+
QNN_EXECUTORCH_LOG_INFO(
108+
"use_memory_optimizations in gpu_options: %d",
109+
gpu_options->use_memory_optimizations());
110+
QNN_EXECUTORCH_LOG_INFO(
111+
"use_node_optimizations in gpu_options: %d",
112+
gpu_options->use_node_optimizations());
113+
QNN_EXECUTORCH_LOG_INFO(
114+
"use_queue_recording in gpu_options: %d",
115+
gpu_options->use_queue_recording());
116+
QNN_EXECUTORCH_LOG_INFO(
117+
"use_weight_sharing in gpu_options: %d",
118+
gpu_options->use_weight_sharing());
119+
}
120+
backend_params->qnn_backend_ptr_ =
121+
std::make_unique<GpuBackend>(implementation, logger, gpu_options);
122+
123+
backend_params->qnn_device_ptr_ =
124+
std::make_unique<GpuDevice>(implementation, logger);
125+
126+
backend_params->qnn_backend_cache_ptr_ =
127+
std::make_unique<QnnBackendCache>(qnn_context_blob);
128+
129+
backend_params->qnn_context_ptr_ = std::make_unique<GpuContext>(
130+
implementation,
131+
backend_params->qnn_backend_ptr_.get(),
132+
backend_params->qnn_device_ptr_.get(),
133+
backend_params->qnn_backend_cache_ptr_.get(),
134+
qnn_dlc_manager,
135+
gpu_options);
136+
137+
backend_params->qnn_graph_ptr_ = std::make_unique<GpuGraph>(
138+
implementation,
139+
backend_params->qnn_backend_ptr_.get(),
140+
backend_params->qnn_context_ptr_.get(),
141+
options->profile_level(),
142+
gpu_options);
143+
} break;
90144
case QnnExecuTorchBackendType::kDspBackend:
91145
case QnnExecuTorchBackendType::kUndefinedBackend:
92146
default:
93147
return nullptr;
94148
}
95149

150+
backend_params->qnn_mem_manager_ptr_ = std::make_unique<QnnMemManager>(
151+
implementation,
152+
backend_params->qnn_context_ptr_.get(),
153+
options->log_level());
154+
155+
backend_params->backend_init_state_ = BackendInitializeState::INITIALIZED;
156+
96157
if (backend_params->qnn_backend_ptr_->VerifyQNNSDKVersion() == Error::Ok) {
97158
return backend_params;
98159
}

‎backends/qualcomm/runtime/backends/QnnBackendFactory.h‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,15 @@
1717
#include <executorch/backends/qualcomm/runtime/backends/QnnImplementation.h>
1818
#include <executorch/backends/qualcomm/runtime/backends/QnnLogger.h>
1919
#include <executorch/backends/qualcomm/runtime/backends/QnnMemManager.h>
20-
#include <executorch/backends/qualcomm/runtime/backends/htpbackend/HtpBackend.h>
21-
#include <executorch/backends/qualcomm/runtime/backends/htpbackend/HtpBackendCache.h>
22-
#include <executorch/backends/qualcomm/runtime/backends/htpbackend/HtpContext.h>
23-
#include <executorch/backends/qualcomm/runtime/backends/htpbackend/HtpDevice.h>
24-
#include <executorch/backends/qualcomm/runtime/backends/htpbackend/HtpGraph.h>
20+
#include <executorch/backends/qualcomm/runtime/backends/gpu/GpuBackend.h>
21+
#include <executorch/backends/qualcomm/runtime/backends/gpu/GpuContext.h>
22+
#include <executorch/backends/qualcomm/runtime/backends/gpu/GpuDevice.h>
23+
#include <executorch/backends/qualcomm/runtime/backends/gpu/GpuGraph.h>
24+
#include <executorch/backends/qualcomm/runtime/backends/htp/HtpBackend.h>
25+
#include <executorch/backends/qualcomm/runtime/backends/htp/HtpBackendCache.h>
26+
#include <executorch/backends/qualcomm/runtime/backends/htp/HtpContext.h>
27+
#include <executorch/backends/qualcomm/runtime/backends/htp/HtpDevice.h>
28+
#include <executorch/backends/qualcomm/runtime/backends/htp/HtpGraph.h>
2529

2630
#include <memory>
2731
namespace executorch {

‎backends/qualcomm/runtime/backends/QnnDeviceCommon.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class QnnDevice {
2929
return handle_;
3030
}
3131

32-
executorch::runtime::Error Configure();
32+
virtual executorch::runtime::Error Configure();
3333

3434
protected:
3535
virtual executorch::runtime::Error MakeConfig(

‎backends/qualcomm/runtime/backends/QnnDlcManager.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
#include <QnnTypes.h>
1212
#include <executorch/backends/qualcomm/runtime/backends/QnnBackendFactory.h>
13-
#include <executorch/backends/qualcomm/runtime/backends/irbackend/IrContext.h>
13+
#include <executorch/backends/qualcomm/runtime/backends/ir/IrContext.h>
1414

1515
#include "QnnWrapperUtils.hpp"
1616
namespace executorch {

‎backends/qualcomm/runtime/backends/gpu/GpuBackend.cpp‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright (c) Qualcomm Innovation Center, Inc.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#include <executorch/backends/qualcomm/runtime/backends/gpu/GpuBackend.h>
10+
11+
#include "GPU/QnnGpuCommon.h"
12+
13+
namespace executorch {
14+
namespace backends {
15+
namespace qnn {
16+
17+
using executorch::runtime::Error;
18+
19+
GpuBackend::GpuBackend(
20+
const QnnImplementation& implementation,
21+
QnnLogger* logger,
22+
const QnnExecuTorchGpuBackendOptions* gpu_options)
23+
: QnnBackend(implementation, logger) {
24+
gpu_backend_custom_config_ =
25+
std::make_unique<GpuBackendCustomConfig>(gpu_options);
26+
}
27+
28+
Qnn_Version_t GpuBackend::GetExpectedBackendVersion() const {
29+
Qnn_Version_t backend_version;
30+
backend_version.major = QNN_GPU_API_VERSION_MAJOR;
31+
backend_version.minor = QNN_GPU_API_VERSION_MINOR;
32+
backend_version.patch = QNN_GPU_API_VERSION_PATCH;
33+
return backend_version;
34+
}
35+
36+
bool GpuBackend::IsProfileEventTypeParentOfNodeTime(
37+
QnnProfile_EventType_t event_type) {
38+
return (event_type == QNN_PROFILE_EVENTTYPE_EXECUTE);
39+
}
40+
41+
Error GpuBackend::MakeConfig(std::vector<const QnnBackend_Config_t*>& config) {
42+
const std::vector<QnnBackend_CustomConfig_t>& backend_custom_config =
43+
gpu_backend_custom_config_->CreateBackendCustomConfig();
44+
45+
uint32_t num_custom_configs = backend_custom_config.size();
46+
backend_config_.resize(num_custom_configs);
47+
// +1 for null terminated
48+
config.reserve(num_custom_configs + 1);
49+
50+
for (std::size_t i = 0; i < num_custom_configs; ++i) {
51+
backend_config_[i].option = QNN_BACKEND_CONFIG_OPTION_CUSTOM;
52+
backend_config_[i].customConfig = backend_custom_config[i];
53+
config.push_back(&backend_config_[i]);
54+
}
55+
56+
config.push_back(nullptr);
57+
return Error::Ok;
58+
}
59+
60+
} // namespace qnn
61+
} // namespace backends
62+
} // namespace executorch

‎backends/qualcomm/runtime/backends/gpu/GpuBackend.h‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright (c) Qualcomm Innovation Center, Inc.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#pragma once
10+
11+
#include <executorch/backends/qualcomm/runtime/backends/gpu/GpuBackendCustomConfig.h>
12+
13+
namespace executorch {
14+
namespace backends {
15+
namespace qnn {
16+
17+
class GpuBackend : public QnnBackend {
18+
public:
19+
GpuBackend(
20+
const QnnImplementation& implementation,
21+
QnnLogger* logger,
22+
const QnnExecuTorchGpuBackendOptions* gpu_options);
23+
24+
Qnn_Version_t GetExpectedBackendVersion() const override;
25+
26+
bool IsProfileEventTypeParentOfNodeTime(
27+
QnnProfile_EventType_t event_type) override;
28+
29+
protected:
30+
executorch::runtime::Error MakeConfig(
31+
std::vector<const QnnBackend_Config_t*>& config) override;
32+
33+
private:
34+
std::vector<QnnBackend_Config_t> backend_config_;
35+
std::unique_ptr<GpuBackendCustomConfig> gpu_backend_custom_config_;
36+
};
37+
38+
} // namespace qnn
39+
} // namespace backends
40+
} // namespace executorch

0 commit comments

Comments
 (0)