Qualcomm AI Engine Direct - Runtime Option

backends/qualcomm/CMakeLists.txt

@@ -116,6 +116,7 @@ add_library(qcir INTERFACE qcir_schema_output)
 add_library(qcir_utils STATIC)
 add_library(qnn_backend STATIC)
 add_library(qnn_backend_cache STATIC)
+add_library(qnn_backend_options STATIC)
 add_library(qnn_context STATIC)
 add_library(qnn_custom_protocol STATIC)
 add_library(qnn_dlc_manager STATIC)
@@ -159,6 +160,7 @@ target_link_libraries(
   qnn_backend PRIVATE qnn_implementation qnn_logger qnn_op_package_manager
 )
 target_link_libraries(qnn_custom_protocol PRIVATE qnn_logger)
+target_link_libraries(qnn_backend_options PRIVATE qnn_schema)
 target_link_libraries(
   qnn_device PRIVATE qnn_executorch_logging qnn_implementation qnn_logger
 )
@@ -197,7 +199,7 @@ target_link_libraries(
 )
 target_link_libraries(
   qnn_executorch_backend PRIVATE qnn_executorch_header qnn_schema qnn_manager
-                                 executorch_core extension_tensor
+                                 executorch_core extension_tensor qnn_backend_options
 )
 set_target_properties(
   qnn_executorch_backend PROPERTIES LINK_FLAGS "-Wl,-rpath='$ORIGIN'"
@@ -261,6 +263,7 @@ if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")
             qnn_executorch_header
             executorch
             extension_tensor
+            qnn_backend_options
   )
   target_link_libraries(
     PyQnnWrapperAdaptor PRIVATE pybind11::module pybind11::lto wrappers

backends/qualcomm/_passes/convert_conv1d_to_conv2d.py

@@ -105,7 +105,7 @@ def call(self, graph_module: torch.fx.GraphModule):
                         padding = [0] + node.args[4] if num_args > 4 else [0, 0]
                         if node.target == torch.ops.aten.conv1d.default:
                             dilation = [1] + node.args[5] if num_args > 5 else [1, 1]
-                            groups = node.args[6] if num_args > 5 else 1
+                            groups = node.args[6] if num_args > 6 else 1
                             conv_args = (
                                 qdq_node_after_unsqueeze,
                                 node.args[1],

backends/qualcomm/runtime/CMakeLists.txt

@@ -28,6 +28,13 @@ target_sources(
   PRIVATE ${CMAKE_CURRENT_LIST_DIR}/QnnManager.cpp
 )
 
+# qnn_backend_options
+target_sources(
+  qnn_backend_options
+  INTERFACE ${CMAKE_CURRENT_LIST_DIR}/QnnBackendOptions.h
+  PRIVATE ${CMAKE_CURRENT_LIST_DIR}/QnnBackendOptions.cpp
+)
+
 # logging
 target_sources(
   qnn_executorch_logging

backends/qualcomm/runtime/QnnBackendOptions.cpp

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) Qualcomm Innovation Center, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+#include <executorch/backends/qualcomm/qc_compiler_spec_generated.h>
+#include <executorch/backends/qualcomm/runtime/QnnBackendOptions.h>
+#include <executorch/backends/qualcomm/runtime/QnnExecuTorchBackend.h>
+
+namespace executorch {
+namespace backends {
+namespace qnn {
+
+using namespace qnn_delegate;
+
+template <typename T>
+T get_option(T aot_option) {
+  executorch::runtime::Error status;
+  executorch::runtime::BackendOption backend_option;
+
+  if constexpr (std::is_same_v<T, QnnExecuTorchLogLevel>) {
+    backend_option = {QNN_RUNTIME_LOG_LEVEL, -1};
+  } else if constexpr (std::is_same_v<T, QnnExecuTorchHtpPerformanceMode>) {
+    backend_option = {QNN_RUNTIME_HTP_PERFORMANCE_MODE, -1};
+  } else if constexpr (std::is_same_v<T, QnnExecuTorchProfileLevel>) {
+    backend_option = {QNN_RUNTIME_PROFILE_LEVEL, -1};
+  }
+  // This will call get_option under runtime backend interface
+  status = get_option(QNN_BACKEND, backend_option);
+
+  if (status != executorch::runtime::Error::Ok) {
+    return aot_option;
+  } else {
+    return static_cast<T>(std::get<int>(backend_option.value));
+  }
+}
+
+// Explicit instantiations
+template QnnExecuTorchLogLevel get_option<QnnExecuTorchLogLevel>(
+    QnnExecuTorchLogLevel);
+template QnnExecuTorchHtpPerformanceMode get_option<
+    QnnExecuTorchHtpPerformanceMode>(QnnExecuTorchHtpPerformanceMode);
+template QnnExecuTorchProfileLevel get_option<QnnExecuTorchProfileLevel>(
+    QnnExecuTorchProfileLevel);
+
+} // namespace qnn
+} // namespace backends
+} // namespace executorch

backends/qualcomm/runtime/QnnBackendOptions.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) Qualcomm Innovation Center, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+#pragma once
+
+#include <executorch/backends/qualcomm/runtime/QnnExecuTorch.h>
+#include <executorch/runtime/backend/interface.h>
+#include <executorch/runtime/backend/options.h>
+
+namespace executorch {
+namespace backends {
+namespace qnn {
+
+/**
+ * @brief Storing runtime option value.
+ * @param is_set True when user calls set_option api to set option, else False.
+ */
+struct RuntimeOption {
+  bool is_set;
+  executorch::runtime::OptionValue value;
+};
+
+/**
+ * @brief
+ * Get the backend option.
+ * This method checks both AOT option and runtime option.
+ * If runtime option is provided, it will have a higher priority.
+ *
+ * @param aot_option The flatbuffer option under qc_compiler_spec.fbs.
+ */
+
+template <typename T>
+T get_option(T aot_option);
+
+} // namespace qnn
+} // namespace backends
+} // namespace executorch

backends/qualcomm/runtime/QnnExecuTorch.h

@@ -16,6 +16,11 @@
 #include <stdint.h>
 #endif
 
+#define QNN_BACKEND "QnnBackend"
+#define QNN_RUNTIME_LOG_LEVEL "qnn_runtime_log_level"
+#define QNN_RUNTIME_HTP_PERFORMANCE_MODE "qnn_runtime_htp_performance_mode"
+#define QNN_RUNTIME_PROFILE_LEVEL "qnn_runtime_profile_level"
+
 #ifdef __cplusplus
 extern "C" {
 #endif // __cplusplus

backends/qualcomm/runtime/QnnExecuTorchBackend.cpp

@@ -8,10 +8,12 @@
 
 #include <executorch/backends/qualcomm/aot/wrappers/TensorWrapper.h>
 #include <executorch/backends/qualcomm/qc_compiler_spec_generated.h>
+#include <executorch/backends/qualcomm/runtime/QnnBackendOptions.h>
 #include <executorch/backends/qualcomm/runtime/QnnExecuTorchBackend.h>
 #include <executorch/backends/qualcomm/runtime/QnnManager.h>
 #include <executorch/backends/qualcomm/runtime/backends/QnnCustomProtocol.h>
-
+#include <executorch/runtime/backend/interface.h>
+#include <executorch/runtime/backend/options.h>
 namespace executorch {
 namespace backends {
 namespace qnn {
@@ -189,6 +191,77 @@ void QnnExecuTorchBackend::destroy(DelegateHandle* handle) const {
   }
 }
 
+executorch::runtime::Error QnnExecuTorchBackend::set_option(
+    executorch::runtime::BackendOptionContext& context,
+    const executorch::runtime::Span<executorch::runtime::BackendOption>&
+        backend_options) {
+  std::lock_guard<std::mutex> guard(runtime_option_mutex_);
+  size_t matches = backend_options.size();
+  for (const auto& option : backend_options) {
+    if (strcmp(option.key, QNN_RUNTIME_LOG_LEVEL) == 0) {
+      if (auto* val = std::get_if<int>(&option.value)) {
+        qnn_runtime_log_level_.value = *val;
+        qnn_runtime_log_level_.is_set = true;
+      }
+    } else if (strcmp(option.key, QNN_RUNTIME_HTP_PERFORMANCE_MODE) == 0) {
+      if (auto* val = std::get_if<int>(&option.value)) {
+        qnn_runtime_performance_mode_.value = *val;
+        qnn_runtime_performance_mode_.is_set = true;
+      }
+    } else if (strcmp(option.key, QNN_RUNTIME_PROFILE_LEVEL) == 0) {
+      if (auto* val = std::get_if<int>(&option.value)) {
+        qnn_runtime_profile_level_.value = *val;
+        qnn_runtime_profile_level_.is_set = true;
+      }
+    } else {
+      ET_LOG(
+          Error,
+          "Unable to set the following runtime option for QnnExecuTorchBackend: %s.",
+          option.key);
+      matches--;
+    }
+  }
+
+  ET_CHECK_OR_RETURN_ERROR(
+      matches == backend_options.size(),
+      Internal,
+      "Some set options are not supported by QnnExecuTorchBackend. %zu options provided but only %zu is supported.",
+      backend_options.size(),
+      matches);
+
+  return Error::Ok;
+}
+
+executorch::runtime::Error QnnExecuTorchBackend::get_option(
+    executorch::runtime::BackendOptionContext& context,
+    executorch::runtime::Span<executorch::runtime::BackendOption>&
+        backend_options) {
+  size_t matches = backend_options.size();
+  for (size_t i = 0; i < backend_options.size(); ++i) {
+    // Set the value to what was stored by set_option
+    if (strcmp(backend_options[i].key, QNN_RUNTIME_LOG_LEVEL) == 0 &&
+        qnn_runtime_log_level_.is_set) {
+      backend_options[i].value = qnn_runtime_log_level_.value;
+    } else if (
+        strcmp(backend_options[i].key, QNN_RUNTIME_HTP_PERFORMANCE_MODE) == 0 &&
+        qnn_runtime_performance_mode_.is_set) {
+      backend_options[i].value = qnn_runtime_performance_mode_.value;
+    } else if (
+        strcmp(backend_options[i].key, QNN_RUNTIME_PROFILE_LEVEL) == 0 &&
+        qnn_runtime_profile_level_.is_set) {
+      backend_options[i].value = qnn_runtime_profile_level_.value;
+    } else {
+      // either runtime never called set_option or key does not exist
+      matches--;
+    }
+  }
+
+  if (matches != backend_options.size()) {
+    return Error::Internal;
+  }
+  return Error::Ok;
+}
+
 bool QnnExecuTorchBackend::is_available() const {
   return true;
 }
@@ -214,7 +287,7 @@ void QnnExecuTorchBackend::erase_cached_delegate(
 
 namespace {
 auto cls = QnnExecuTorchBackend();
-executorch::runtime::Backend backend{"QnnBackend", &cls};
+executorch::runtime::Backend backend{QNN_BACKEND, &cls};
 static auto success_with_compiler = register_backend(backend);
 } // namespace
 } // namespace qnn

backends/qualcomm/runtime/QnnExecuTorchBackend.h

@@ -7,6 +7,7 @@
  */
 #pragma once
 
+#include <executorch/backends/qualcomm/runtime/QnnBackendOptions.h>
 #include <executorch/runtime/backend/interface.h>
 #include <executorch/runtime/core/error.h>
 #include <executorch/runtime/core/evalue.h>
@@ -34,6 +35,16 @@ class QnnExecuTorchBackend final
       executorch::runtime::DelegateHandle* handle,
       executorch::runtime::EValue** args) const override;
 
+  ET_NODISCARD executorch::runtime::Error set_option(
+      executorch::runtime::BackendOptionContext& context,
+      const executorch::runtime::Span<executorch::runtime::BackendOption>&
+          backend_options) override;
+
+  executorch::runtime::Error get_option(
+      executorch::runtime::BackendOptionContext& context,
+      executorch::runtime::Span<executorch::runtime::BackendOption>&
+          backend_options) override;
+
   void destroy(executorch::runtime::DelegateHandle* handle) const override;
 
   bool is_available() const override;
@@ -45,10 +56,15 @@ class QnnExecuTorchBackend final
   void erase_cached_delegate(executorch::runtime::DelegateHandle* handle) const;
 
   mutable std::mutex mutex_;
+  mutable std::mutex runtime_option_mutex_;
   mutable std::unordered_map<int64_t, executorch::runtime::DelegateHandle*>
       delegate_map_;
   mutable std::unordered_map<executorch::runtime::DelegateHandle*, std::int64_t>
       delegate_map_rev_;
+
+  RuntimeOption qnn_runtime_log_level_{false, 0};
+  RuntimeOption qnn_runtime_performance_mode_{false, 0};
+  RuntimeOption qnn_runtime_profile_level_{false, 0};
 };
 
 } // namespace qnn

backends/qualcomm/runtime/QnnManager.cpp

@@ -6,6 +6,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+#include <executorch/backends/qualcomm/runtime/QnnBackendOptions.h>
 #include <executorch/backends/qualcomm/runtime/QnnManager.h>
 #include <executorch/backends/qualcomm/runtime/SharedBuffer.h>
 #include <executorch/backends/qualcomm/runtime/Utils.h>
@@ -63,7 +64,8 @@ QnnManager::QnnManager(
       options->backend_options()->backend_type();
   std::string library_path = options->library_path()->str();
 
-  if (options->log_level() >= QnnExecuTorchLogLevel::kLogLevelInfo) {
+  if (get_option(options_->log_level()) >=
+      QnnExecuTorchLogLevel::kLogLevelInfo) {
     QNN_EXECUTORCH_LOG_INFO(
         "soc_model in soc_info: %s",
         EnumNameQcomChipset(options_->soc_info()->soc_model()));
@@ -75,10 +77,12 @@ QnnManager::QnnManager(
     QNN_EXECUTORCH_LOG_INFO("library_path: %s", library_path.c_str());
     QNN_EXECUTORCH_LOG_INFO("dump intermediate outputs: %s", IsTensorDump());
     QNN_EXECUTORCH_LOG_INFO(
-        "log_level: %s", EnumNameQnnExecuTorchLogLevel(options_->log_level()));
+        "log_level: %s",
+        EnumNameQnnExecuTorchLogLevel(get_option(options_->log_level())));
     QNN_EXECUTORCH_LOG_INFO(
         "profile_level: %s",
-        EnumNameQnnExecuTorchProfileLevel(options_->profile_level()));
+        EnumNameQnnExecuTorchProfileLevel(
+            get_option(options_->profile_level())));
     QNN_EXECUTORCH_LOG_INFO(
         "the size of qnn context binary: %d",
         qnn_executorch_context_binary.nbytes);
@@ -202,7 +206,8 @@ Error QnnManager::RegisterIonMem(
     return Error::Internal;
   } else if (backend_params_ptr_->qnn_mem_manager_ptr_->IsRegistered(
                  tensor_wrapper->GetMemHandle(), data_ptr)) {
-    if (options_->log_level() >= QnnExecuTorchLogLevel::kLogLevelInfo)
+    if (get_option(options_->log_level()) >=
+        QnnExecuTorchLogLevel::kLogLevelInfo)
       QNN_EXECUTORCH_LOG_INFO(
           "Tensor name %s has been registered shared memory.",
           tensor_wrapper->GetName().c_str());
@@ -231,7 +236,8 @@ Error QnnManager::RegisterCustomMem(
     const std::shared_ptr<TensorWrapper>& tensor_wrapper) {
   if (backend_params_ptr_->qnn_mem_manager_ptr_->IsRegistered(
           tensor_wrapper->GetMemHandle(), data_ptr)) {
-    if (options_->log_level() >= QnnExecuTorchLogLevel::kLogLevelInfo)
+    if (get_option(options_->log_level()) >=
+        QnnExecuTorchLogLevel::kLogLevelInfo)
       QNN_EXECUTORCH_LOG_INFO(
           "Tensor name %s has been registered shared memory.",
           tensor_wrapper->GetName().c_str());
@@ -251,7 +257,8 @@ Error QnnManager::RegisterCustomMem(
   Qnn_MemHandle_t pre_registered_handle =
       backend_params_ptr_->qnn_mem_manager_ptr_->GetPreRegisteredHandle(info);
   if (pre_registered_handle != nullptr) {
-    if (options_->log_level() >= QnnExecuTorchLogLevel::kLogLevelInfo) {
+    if (get_option(options_->log_level()) >=
+        QnnExecuTorchLogLevel::kLogLevelInfo) {
       QNN_EXECUTORCH_LOG_INFO(
           "Tensor name %s found a pre-registered memHandle.",
           tensor_wrapper->GetName().c_str());
@@ -295,7 +302,7 @@ Error QnnManager::Init() {
   ET_CHECK_OR_RETURN_ERROR(
       LoadQnnLibrary() == Error::Ok, Internal, "Fail to load Qnn library");
   logger_ = std::make_unique<QnnLogger>(
-      qnn_loaded_backend_, LoggingCallback, options_->log_level());
+      qnn_loaded_backend_, LoggingCallback, get_option(options_->log_level()));
   std::vector<std::string> graph_names;
   for (auto name : *options_->graph_name()) {
     graph_names.emplace_back(name->str());
@@ -492,7 +499,8 @@ Error QnnManager::ProfileExecuteData(
     const std::string& graph_name,
     executorch::runtime::EventTracer* event_tracer) {
   Qnn_ErrorHandle_t error = QNN_SUCCESS;
-  if (options_->profile_level() != QnnExecuTorchProfileLevel::kProfileOff) {
+  if (get_option(options_->profile_level()) !=
+      QnnExecuTorchProfileLevel::kProfileOff) {
     error = backend_params_ptr_->qnn_graph_ptr_->ProfileExecuteData(
         graph_name, event_tracer);
     if (error != QNN_SUCCESS) {