Qualcomm AI Engine Direct - Support Qnn IR backend in online preparation

 - Support Qnn IR backend
 - Replace QCir with Dlc in online prepare flow
 - Fix SDK version checking
 - Add config for Saver backend
 - Block online preparation if the QNN version is below 2.30.
 - Change the QNN version from 2.28 to 2.31 in CI to align with QnnIR

Author: DannyYuyang-quic

.ci/scripts/build-qnn-sdk.sh

@@ -12,7 +12,7 @@ set -o xtrace
 build_qnn_backend() {
   echo "Start building qnn backend."
   export ANDROID_NDK_ROOT=/opt/ndk
-  export QNN_SDK_ROOT=/tmp/qnn/2.28.0.241029
+  export QNN_SDK_ROOT=/tmp/qnn/2.31.0.250130
   export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
 
   # Workaround to avoid issues around missing flatccrt library (depending on the

.ci/scripts/setup-qnn-deps.sh

@@ -16,9 +16,9 @@ install_qnn() {
   QNN_INSTALLATION_DIR=/tmp/qnn
   mkdir -p "${QNN_INSTALLATION_DIR}"
 
-  curl -Lo /tmp/v2.28.0.24.10.29.zip "https://softwarecenter.qualcomm.com/api/download/software/qualcomm_neural_processing_sdk/v2.28.0.241029.zip"
+  curl -Lo /tmp/v2.31.0.25.01.30.zip "https://softwarecenter.qualcomm.com/api/download/software/qualcomm_neural_processing_sdk/v2.31.0.250130.zip"
   echo "Finishing downloading qnn sdk."
-  unzip -qo /tmp/v2.28.0.24.10.29.zip -d /tmp
+  unzip -qo /tmp/v2.31.0.25.01.30.zip -d /tmp
   echo "Finishing unzip qnn sdk."
 
 

.ci/scripts/test_llama.sh

@@ -123,7 +123,7 @@ echo "COREML option ${COREML}"
 if [[ "${MODE}" =~ .*qnn.* ]]; then
   QNN=ON
   export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
-  export QNN_SDK_ROOT=/tmp/qnn/2.28.0.241029
+  export QNN_SDK_ROOT=/tmp/qnn/2.31.0.250130
   export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang"
   export PYTHONPATH=".."
   cp schema/program.fbs exir/_serialize/program.fbs

.ci/scripts/test_qnn_static_llama.sh

@@ -10,7 +10,7 @@ set -exu
 source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 
 export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
-export QNN_SDK_ROOT=/tmp/qnn/2.28.0.241029
+export QNN_SDK_ROOT=/tmp/qnn/2.31.0.250130
 export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang"
 export PYTHONPATH=".."
 cp schema/program.fbs exir/_serialize/program.fbs

.github/workflows/android-perf.yml

@@ -273,7 +273,7 @@ jobs:
                       --output_name="${OUT_ET_MODEL_NAME}.pte"
                     ls -lh "${OUT_ET_MODEL_NAME}.pte"
                 elif [[ ${{ matrix.config }} == "llama3_qnn_htp" ]]; then
-                    export QNN_SDK_ROOT=/tmp/qnn/2.28.0.241029
+                    export QNN_SDK_ROOT=/tmp/qnn/2.31.0.250130
                     export LD_LIBRARY_PATH=$QNN_SDK_ROOT/lib/x86_64-linux-clang/
                     export PYTHONPATH=$(pwd)/..
 

backends/qualcomm/CMakeLists.txt

@@ -73,6 +73,7 @@ endif()
 
 include_directories(
   BEFORE ${_common_include_directories} ${QNN_SDK_ROOT}/include/QNN
+  ${QNN_SDK_ROOT}/share/QNN/converter/jni
   ${EXECUTORCH_SOURCE_DIR}/third-party/flatbuffers/include
   ${EXECUTORCH_SOURCE_DIR}/runtime/core/portable_type/c10
 )
@@ -176,6 +177,7 @@ target_link_libraries(
   qnn_factory PRIVATE qnn_schema qnn_backend qnn_device qnn_context qnn_graph
           qnn_mem_manager qnn_custom_protocol
 )
+
 target_link_libraries(
   qnn_manager PRIVATE qnn_factory wrappers qnn_schema utils shared_buffer
 )

backends/qualcomm/aot/python/PyQnnManagerAdaptor.cpp

@@ -32,6 +32,7 @@ PYBIND11_MODULE(PyQnnManagerAdaptor, m) {
       .def(py::init<const py::bytes&, const py::bytes&>())
       .def(py::init<const py::bytes&, const py::list&>())
       .def("Init", &PyQnnManager::Init)
+      .def("GetQnnAPIVersion", &PyQnnManager::GetQnnAPIVersion)
       .def("IsNodeSupportedByBackend", &PyQnnManager::IsNodeSupportedByBackend)
       .def("Compile", py::overload_cast<>(&PyQnnManager::Compile))
       .def(

backends/qualcomm/aot/python/PyQnnManagerAdaptor.h

@@ -18,6 +18,7 @@
 #include <pybind11/stl.h>
 #include <memory>
 #include <string_view>
+#include "QnnTypes.h"
 
 namespace py = pybind11;
 namespace executorch {
@@ -34,7 +35,7 @@ class PyQnnManager {
     auto qnn_executorch_options = GetQnnExecuTorchOptions(
         qnn_executorch_option_ptr_.cast<std::string_view>().data());
     qnn_manager_ = std::make_shared<QnnManager>(
-        qnn_executorch_options, qnn_executorch_context_binary_);
+        qnn_executorch_options, qnn_executorch_context_binary_, false);
   }
 
   // used for loading context binary directly
@@ -47,7 +48,7 @@ class PyQnnManager {
     qnn_executorch_context_binary_.buffer = info.ptr;
     qnn_executorch_context_binary_.nbytes = info.size * info.itemsize;
     qnn_manager_ = std::make_shared<QnnManager>(
-        qnn_executorch_options, qnn_executorch_context_binary_);
+        qnn_executorch_options, qnn_executorch_context_binary_, false);
   }
 
   // used during stage 2 of multi-graph mode
@@ -160,7 +161,12 @@ class PyQnnManager {
     qnn_executorch_context_binary_ =
         MakeQcirCustomBinaryInfo(qcir_bin, tensor_data);
     qnn_manager_ = std::make_shared<QnnManager>(
-        qnn_executorch_options, qnn_executorch_context_binary_);
+        qnn_executorch_options, qnn_executorch_context_binary_, false);
+  }
+
+  std::vector<int> GetQnnAPIVersion() {
+    return {
+        QNN_API_VERSION_MAJOR, QNN_API_VERSION_MINOR, QNN_API_VERSION_PATCH};
   }
 
   executorch::runtime::Error Init() {
@@ -195,7 +201,7 @@ class PyQnnManager {
       std::vector<std::shared_ptr<OpWrapper>>& op_wrappers) {
     QnnExecuTorchContextBinary binary_info;
 
-    if (qnn_manager_->IsOnlinePrepare() || qnn_manager_->IsMultipleGraphs()) {
+    if (qnn_manager_->IsMultipleGraphs()) {
       builder_.Reset();
       std::vector<uint8_t> tensor_data;
       std::vector<uint64_t> offsets;
@@ -305,8 +311,11 @@ class PyQnnManager {
         QNN_EXECUTORCH_LOG_ERROR("Fail to compile QNN graph");
         return py::array_t<char>(0);
       }
-      if (qnn_manager_->GetContextBinary(binary_info) !=
-          executorch::runtime::Error::Ok) {
+      auto qnn_executorch_options = GetQnnExecuTorchOptions(
+          qnn_executorch_option_ptr_.cast<std::string_view>().data());
+      if (qnn_executorch_options->saver() ||
+          qnn_manager_->GetContextBinary(binary_info) !=
+              executorch::runtime::Error::Ok) {
         return py::array_t<char>(0);
       }
     }

backends/qualcomm/qnn_preprocess.py

@@ -20,6 +20,9 @@
 from executorch.backends.qualcomm.builders.node_visitor import get_node_visitors
 from executorch.backends.qualcomm.builders.qnn_constants import OpContextLoader
 from executorch.backends.qualcomm.partition.utils import generate_qnn_executorch_option
+from executorch.backends.qualcomm.serialization.qc_schema_serialize import (
+    flatbuffer_to_option,
+)
 from executorch.exir.backend.backend_details import (
     BackendDetails,
     CompileSpec,
@@ -43,6 +46,13 @@ def preprocess(
     ) -> PreprocessResult:
         option = generate_qnn_executorch_option(compile_specs)
         qnn_manager = PyQnnManager.QnnManager(option)
+        obj_options = flatbuffer_to_option(option)
+
+        core_api_version = qnn_manager.GetQnnAPIVersion()
+        assert not obj_options.online_prepare or (
+            core_api_version[0] >= 2 and core_api_version[1] >= 23
+        ), "Online prepare is disabled for Qnn API versions below 2.23.0."
+
         qnn_manager.Init()
 
         # QNN Delegate Specific Passes
@@ -107,6 +117,11 @@ def preprocess(
             qnn_manager.GetGraphNames()[0],
             [py_op_wrapper.GetOpWrapper() for py_op_wrapper in py_op_wrapper_list],
         )
+
+        if obj_options.saver:
+            exit(
+                f"Records all QNN API calls from saver backend at: {obj_options.saver_output_dir}"
+            )
         assert len(qnn_context_binary) != 0, "Failed to generate Qnn context binary."
         qnn_manager.Destroy()
         # For now, debug_handle_map is not used by QNN ExecuTorch

backends/qualcomm/runtime/QnnExecuTorchBackend.cpp

@@ -36,7 +36,6 @@ Result<DelegateHandle*> QnnExecuTorchBackend::init(
   // covert SizedBuffer to qnn ExecuTorch option
   QnnExecuTorchContextBinary qnn_context_blob;
   const qnn_delegate::QnnExecuTorchOptions* qnn_executorch_options = nullptr;
-
   auto [status, signature, ctx_size, ctx_bin] =
       QnnContextCustomProtocol().DeserializeContextCustomBuffer(
           const_cast<void*>(processed->data()));
@@ -49,6 +48,7 @@ Result<DelegateHandle*> QnnExecuTorchBackend::init(
     qnn_context_blob.buffer = ctx_bin;
   } else {
     // This buffer will be verified again in QnnBackendCache.
+
     QNN_EXECUTORCH_LOG_INFO(
         "Deserializing processed data using QnnQcirCustomProtocol");
     qnn_context_blob.buffer = const_cast<void*>(processed->data());
@@ -71,8 +71,7 @@ Result<DelegateHandle*> QnnExecuTorchBackend::init(
 
   // NOTE: Since we use placement new and since this type is not trivially
   // destructible, we must call the destructor manually in destroy().
-  new (qnn_manager) QnnManager(qnn_executorch_options, qnn_context_blob);
-
+  new (qnn_manager) QnnManager(qnn_executorch_options, qnn_context_blob, true);
   // TODO: this is a temporal solution for multi-graph support, will be
   //       removed once framework starts to accept runtime configuration
   // ---
@@ -94,9 +93,9 @@ Result<DelegateHandle*> QnnExecuTorchBackend::init(
 
   if (qnn_manager->IsOnlinePrepare()) {
     ET_CHECK_OR_RETURN_ERROR(
-        qnn_manager->CompileQcir() == Error::Ok,
+        qnn_manager->CompileGraphsFromDlc() == Error::Ok,
         Internal,
-        "Fail to compile binary in qcir format");
+        "Fail to compile binary in Dlc format");
   } else {
     for (const std::string& graph_name : qnn_manager->GetGraphNames()) {
       ET_CHECK_OR_RETURN_ERROR(