Qualcomm AI Engine Direct - LE support

Summary:
- enable linux embedded support for toolchain:
  aarch64-oe-linux-gcc9.3 (SXR1230P)
- update document a bit

Author: haowhsu-quic

backends/qualcomm/aot/ir/qcir_utils.cpp

@@ -322,7 +322,9 @@ Qnn_Tensor_t ToTensor(const tensor_type& tensor, const uint8_t* data_ptr) {
     return type < QNN_TENSOR_TYPE_STATIC;
   };
 
-  Qnn_Tensor_t t({.version = QNN_TENSOR_VERSION_2, .v2 = QNN_TENSOR_V2_INIT});
+  Qnn_Tensor_t t;
+  t.version = QNN_TENSOR_VERSION_2;
+  t.v2 = QNN_TENSOR_V2_INIT;
   QNN_TENSOR_VER_PTR(t)->name = tensor->name()->c_str();
   QNN_TENSOR_VER_PTR(t)->type = ToTensorType(tensor->type());
   QNN_TENSOR_VER_PTR(t)->dataType = ToDataType(tensor->dtype());

backends/qualcomm/scripts/build.sh

@@ -16,8 +16,10 @@ fi
 usage() {
   echo "Usage: Build the aarch64 version of executor runner or the python interface of Qnn Manager"
   echo "First, you need to set the environment variable for QNN_SDK_ROOT"
-  echo ", and if you want to build the aarch64 version of executor runner"
+  echo ", and if you want to build the android version of executor runner"
   echo ", you need to export ANDROID_NDK_ROOT=/path/to/android_ndkXX"
+  echo "(or export TOOLCHAIN_ROOT_HOST=/path/to/sysroots/xx_host, "
+  echo "TOOLCHAIN_ROOT_TARGET=/path/to/sysroots/xx_target for linux embedded with --le)"
   echo "e.g.: executorch$ ./backends/qualcomm/scripts/build.sh --skip_x86_64"
   exit 1
 }
@@ -27,8 +29,10 @@ usage() {
 
 BUILD_X86_64="true"
 CMAKE_X86_64="build-x86"
-BUILD_AARCH64="true"
-CMAKE_AARCH64="build-android"
+BUILD_ANDROID="true"
+CMAKE_ANDROID="build-android"
+BUILD_OE_LINUX="false"
+CMAKE_OE_LINUX="build-oe-linux"
 CLEAN="true"
 BUILD_TYPE="RelWithDebInfo"
 BUILD_JOB_NUMBER="16"
@@ -41,7 +45,7 @@ if [ -z BUCK2 ]; then
   BUCK2="buck2"
 fi
 
-long_options=skip_x86_64,skip_aarch64,no_clean,release,job_number:
+long_options=skip_x86_64,skip_la,skip_le,le,no_clean,release,job_number:
 
 parsed_args=$(getopt -a --options '' --longoptions $long_options --name "$0" -- "$@")
 eval set -- "$parsed_args"
@@ -50,7 +54,9 @@ eval set -- "$parsed_args"
 while true ; do
     case "$1" in
         --skip_x86_64) BUILD_X86_64="false"; shift;;
-        --skip_aarch64) BUILD_AARCH64="false"; shift;;
+        --skip_la) BUILD_ANDROID="false"; shift;;
+        --skip_le) BUILD_OE_LINUX="false"; shift;;
+        --le) BUILD_ANDROID="false"; BUILD_OE_LINUX="true"; shift;;
         --no_clean) CLEAN="false"; shift;;
         --release) BUILD_TYPE="Release"; shift;;
         --job_number) BUILD_JOB_NUMBER="$2"; shift 2;;
@@ -60,13 +66,13 @@ done
 
 PRJ_ROOT="$( cd "$(dirname "$0")/../../.." ; pwd -P)"
 
-if [ "$BUILD_AARCH64" = true ]; then
+if [ "$BUILD_ANDROID" = true ]; then
     if [[ -z ${ANDROID_NDK_ROOT} ]]; then
         echo "Please export ANDROID_NDK_ROOT=/path/to/android_ndkXX"
         exit -1
     fi
 
-    BUILD_ROOT=$PRJ_ROOT/$CMAKE_AARCH64
+    BUILD_ROOT=$PRJ_ROOT/$CMAKE_ANDROID
     if [ "$CLEAN" = true ]; then
         rm -rf $BUILD_ROOT && mkdir $BUILD_ROOT
     else
@@ -112,6 +118,70 @@ if [ "$BUILD_AARCH64" = true ]; then
     cmake --build $EXAMPLE_ROOT -j$BUILD_JOB_NUMBER
 fi
 
+if [ "$BUILD_OE_LINUX" = true ]; then
+    if [[ -z ${TOOLCHAIN_ROOT_HOST} ]]; then
+        echo "Please export e.g. TOOLCHAIN_ROOT_HOST=/path/to/sysroots/x86_64-qtisdk-linux"
+        exit -1
+    fi
+    if [[ -z ${TOOLCHAIN_ROOT_TARGET} ]]; then
+        echo "Please export e.g. TOOLCHAIN_ROOT_TARGET=/path/to/sysroots/armv8a-oe-linux"
+        exit -1
+    fi
+
+    BUILD_ROOT=$PRJ_ROOT/$CMAKE_OE_LINUX
+    if [ "$CLEAN" = true ]; then
+        rm -rf $BUILD_ROOT && mkdir $BUILD_ROOT
+    else
+        # Force rebuild flatccrt for the correct platform
+        cd $BUILD_ROOT/third-party/flatcc && make clean
+    fi
+
+    TOOLCHAN_PREFIX=$TOOLCHAIN_ROOT_HOST/usr/bin/aarch64-oe-linux/aarch64-oe-linux-
+    cd $BUILD_ROOT
+    cmake .. \
+        -DCMAKE_INSTALL_PREFIX=$BUILD_ROOT \
+        -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+        -DCMAKE_C_COMPILER=${TOOLCHAN_PREFIX}gcc \
+        -DCMAKE_CXX_COMPILER=${TOOLCHAN_PREFIX}g++ \
+        -DCMAKE_SYSROOT=$TOOLCHAIN_ROOT_TARGET \
+        -DCMAKE_SYSTEM_NAME=Linux \
+        -DCMAKE_SYSTEM_PROCESSOR=aarch64 \
+        -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+        -DEXECUTORCH_BUILD_CPUINFO=OFF \
+        -DEXECUTORCH_BUILD_QNN=ON \
+        -DEXECUTORCH_BUILD_DEVTOOLS=ON \
+        -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
+        -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+        -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
+        -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
+        -DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
+        -DQNN_SDK_ROOT=$QNN_SDK_ROOT \
+        -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
+        -DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \
+        -B$BUILD_ROOT
+
+    cmake --build $BUILD_ROOT -j$BUILD_JOB_NUMBER --target install
+
+    EXAMPLE_ROOT=examples/qualcomm
+    CMAKE_PREFIX_PATH="${BUILD_ROOT}/lib/cmake/ExecuTorch;${BUILD_ROOT}/third-party/gflags;"
+
+    cmake $PRJ_ROOT/$EXAMPLE_ROOT \
+        -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+        -DCMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH \
+        -DCMAKE_C_COMPILER=${TOOLCHAN_PREFIX}gcc \
+        -DCMAKE_CXX_COMPILER=${TOOLCHAN_PREFIX}g++ \
+        -DCMAKE_SYSROOT=$TOOLCHAIN_ROOT_TARGET \
+        -DCMAKE_SYSTEM_NAME=Linux \
+        -DCMAKE_SYSTEM_PROCESSOR=aarch64 \
+        -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
+        -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+        -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH \
+        -DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \
+        -B$EXAMPLE_ROOT
+
+    cmake --build $EXAMPLE_ROOT -j$BUILD_JOB_NUMBER
+fi
+
 if [ "$BUILD_X86_64" = true ]; then
     BUILD_ROOT=$PRJ_ROOT/$CMAKE_X86_64
     if [ "$CLEAN" = true ]; then

backends/qualcomm/tests/test_qnn_delegate.py

@@ -5988,7 +5988,13 @@ def setup_environment():
         default="",
         type=str,
     )
-
+    parser.add_argument(
+        "-t",
+        "--target",
+        help="Target platform for deployment",
+        default="aarch64-android",
+        type=str,
+    )
     parser.add_argument(
         "--pre_gen_pte",
         help="Run the pre-generated pte in the given directory.",
@@ -6023,6 +6029,7 @@ def setup_environment():
     TestQNN.pre_gen_pte = args.pre_gen_pte
     TestQNN.llama_artifacts = args.llama_artifacts
     TestQNN.op_package_dir = args.op_package_dir
+    TestQNN.target = args.target
     return sys.argv[:1] + ns_args
 
 

backends/qualcomm/tests/utils.py

@@ -188,10 +188,13 @@ class TestQNN(unittest.TestCase):
     pretrained_weight: str = ""
     enable_profile: bool = False
     op_package_dir: str = ""
+    target: str = ""
+    model_name: str = ""
     online_prepare: bool = False
     use_8a8w: str = "8a8w"
     use_16a16w: str = "16a16w"
     use_16a4w: str = "16a4w"
+    oss_repo: str = ""
     shared_buffer: bool = False
     enable_x86_64: bool = False
     compile_only: bool = False
@@ -432,6 +435,7 @@ def validate_intermediate_tensor():
                         if check_io_shape
                         else None
                     ),
+                    target=self.target,
                 )
                 adb.push(
                     inputs=[processed_inputs],
@@ -616,6 +620,7 @@ def get_adb_tool(self, pte_fname):
             host_id=self.host,
             soc_model=self.model,
             error_only=self.error_only,
+            target=self.target,
         )
         return adb
 

docs/source/backends-qualcomm.md

@@ -53,7 +53,7 @@ For more details and troubleshooting, refer to the official Microsoft WSL instal
 👉 [Install WSL | Microsoft Learn](https://learn.microsoft.com/en-us/windows/wsl/install)
 
 ### Hardware:
-You will need an Android smartphone with adb-connected running on one of below Qualcomm SoCs:
+You will need an Android / Linux device with adb-connected running on one of below Qualcomm SoCs:
  - SA8295
  - SM8450 (Snapdragon 8 Gen 1)
  - SM8475 (Snapdragon 8 Gen 1+)
@@ -62,7 +62,7 @@ You will need an Android smartphone with adb-connected running on one of below Q
  - SM8750 (Snapdragon 8 Elite)
  - SSG2115P
  - SSG2125P
- - SXR1230P
+ - SXR1230P (Linux Embedded)
  - SXR2230P
  - SXR2330P
 
@@ -73,6 +73,7 @@ This example is verified with SM8550 and SM8450.
  - Follow ExecuTorch recommended Python version.
  - A compiler to compile AOT parts, e.g., the GCC compiler comes with Ubuntu LTS.
  - [Android NDK](https://developer.android.com/ndk). This example is verified with NDK 26c.
+ - (Optional) Target toolchain for linux embedded platform.
  - [Qualcomm AI Engine Direct SDK](https://developer.qualcomm.com/software/qualcomm-ai-engine-direct-sdk)
    - Click the "Get Software" button to download a version of QNN SDK.
    - However, at the moment of updating this tutorial, the above website doesn't provide QNN SDK newer than 2.22.6.
@@ -131,8 +132,11 @@ The above script is actively used. It is updated more frquently than this tutori
 An example usage is
 ```bash
 cd $EXECUTORCH_ROOT
+# android target
 ./backends/qualcomm/scripts/build.sh
-# or
+# (optional) linux embedded target
+./backends/qualcomm/scripts/build.sh --le
+# for release build
 ./backends/qualcomm/scripts/build.sh --release
 ```
 
@@ -148,11 +152,14 @@ cd build-x86
 # Please refer to the above build.sh for latest workable commands.
 cmake .. \
   -DCMAKE_INSTALL_PREFIX=$PWD \
+  -DCMAKE_BUILD_TYPE=RelWithDebInfo \
   -DEXECUTORCH_BUILD_QNN=ON \
   -DQNN_SDK_ROOT=${QNN_SDK_ROOT} \
   -DEXECUTORCH_BUILD_DEVTOOLS=ON \
   -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
   -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
+  -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
+  -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
   -DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
   -DPYTHON_EXECUTABLE=python3
 
@@ -177,17 +184,23 @@ A example `qnn_executor_runner` executable would be used to run the compiled `pt
 Commands to build `qnn_executor_runner` for Android:
 
 ```bash
+export ANDROID_NDK_ROOT=/path/to/android_ndk_r26c/..
+
 cd $EXECUTORCH_ROOT
 mkdir build-android
 cd build-android
 # build executorch & qnn_executorch_backend
 cmake .. \
     -DCMAKE_INSTALL_PREFIX=$PWD \
+    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
     -DEXECUTORCH_BUILD_QNN=ON \
     -DQNN_SDK_ROOT=$QNN_SDK_ROOT \
     -DEXECUTORCH_BUILD_DEVTOOLS=ON \
     -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
     -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
+    -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
     -DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
     -DPYTHON_EXECUTABLE=python3 \
     -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_ROOT/build/cmake/android.toolchain.cmake \
@@ -199,6 +212,7 @@ cmake .. \
 cmake --build $PWD --target install -j$(nproc)
 
 cmake ../examples/qualcomm \
+    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
     -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_ROOT/build/cmake/android.toolchain.cmake \
     -DANDROID_ABI='arm64-v8a' \
     -DANDROID_PLATFORM=android-30 \
@@ -214,6 +228,61 @@ cmake --build examples/qualcomm -j$(nproc)
 ls examples/qualcomm
 ```
 
+(Optional) Commands to build `qnn_executor_runner` for Linux Embedded:
+
+```bash
+export TOOLCHAIN_ROOT_HOST=/path/to/sysroots/HOST # e.g. x86_64-qtisdk-linux
+export TOOLCHAIN_ROOT_TARGET=/path/to/sysroots/TARGET # e.g. armv8a-oe-linux
+
+cd $EXECUTORCH_ROOT
+mkdir build-oe-linux
+cd build-oe-linux
+# build executorch & qnn_executorch_backend
+cmake .. \
+    -DCMAKE_INSTALL_PREFIX=$PWD \
+    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+    -DCMAKE_C_COMPILER=${TOOLCHAIN_ROOT_HOST}/usr/bin/aarch64-oe-linux/aarch64-oe-linux-gcc \
+    -DCMAKE_CXX_COMPILER=${TOOLCHAIN_ROOT_HOST}/usr/bin/aarch64-oe-linux/aarch64-oe-linux-g++ \
+    -DCMAKE_SYSROOT=$TOOLCHAIN_ROOT_TARGET \
+    -DCMAKE_SYSTEM_NAME=Linux \
+    -DCMAKE_SYSTEM_PROCESSOR=aarch64 \
+    -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+    -DEXECUTORCH_BUILD_CPUINFO=OFF \
+    -DEXECUTORCH_BUILD_QNN=ON \
+    -DQNN_SDK_ROOT=$QNN_SDK_ROOT \
+    -DEXECUTORCH_BUILD_DEVTOOLS=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
+    -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
+    -DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
+    -DPYTHON_EXECUTABLE=python3 \
+
+# nproc is used to detect the number of available CPU.
+# If it is not applicable, please feel free to use the number you want.
+cmake --build $PWD --target install -j$(nproc)
+
+cmake ../examples/qualcomm \
+    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+    -DCMAKE_PREFIX_PATH="$PWD/lib/cmake/ExecuTorch;$PWD/third-party/gflags;" \
+    -DCMAKE_C_COMPILER=${TOOLCHAIN_ROOT_HOST}/usr/bin/aarch64-oe-linux/aarch64-oe-linux-gcc \
+    -DCMAKE_CXX_COMPILER=${TOOLCHAIN_ROOT_HOST}/usr/bin/aarch64-oe-linux/aarch64-oe-linux-g++ \
+    -DCMAKE_SYSROOT=$TOOLCHAIN_ROOT_TARGET \
+    -DCMAKE_SYSTEM_NAME=Linux \
+    -DCMAKE_SYSTEM_PROCESSOR=aarch64 \
+    -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
+    -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH \
+    -DPYTHON_EXECUTABLE=python3 \
+    -Bexamples/qualcomm
+
+cmake --build examples/qualcomm -j$(nproc)
+
+# qnn_executor_runner can be found under examples/qualcomm
+# The full path is $EXECUTORCH_ROOT/build-oe-linux/examples/qualcomm/qnn_executor_runner
+ls examples/qualcomm
+```
+
 **Note:** If you want to build for release, add `-DCMAKE_BUILD_TYPE=Release` to the `cmake` command options.
 
 
@@ -351,7 +420,10 @@ I 00:00:00.364875 executorch:qnn_executor_runner.cpp:425] Write etdump to etdump
 The model is merely executed. If we want to feed real inputs and get model outputs, we can use
 ```bash
 cd $EXECUTORCH_ROOT
+# android
 python -m examples.qualcomm.scripts.deeplab_v3 -b build-android -m SM8550 --download -s <device_serial>
+# (optional) linux embedded
+python -m examples.qualcomm.scripts.deeplab_v3 -b build-oe-linux -m SXR1230P --download -s <device_serial> -t aarch64-oe-linux-gcc-9.3
 ```
 The `<device_serial>` can be found by `adb devices` command.
 

examples/qualcomm/utils.py

@@ -81,6 +81,7 @@ def __init__(
         shared_buffer=False,
         dump_intermediate_outputs=False,
         runner="examples/qualcomm/executor_runner/qnn_executor_runner",
+        target="aarch64-android",
         expected_input_shape=None,
         expected_output_shape=None,
     ):
@@ -100,6 +101,7 @@ def __init__(
         self.error_only = error_only
         self.shared_buffer = shared_buffer
         self.runner = runner
+        self.target = target
         self.expected_input_shape = expected_input_shape
         self.expected_output_shape = expected_output_shape
         self.extra_cmds = ""
@@ -122,20 +124,17 @@ def push(self, inputs=None, input_list=None, files=None):
         # necessary artifacts
         artifacts = [
             *self.pte_path,
-            f"{self.qnn_sdk}/lib/aarch64-android/libQnnHtp.so",
+            f"{self.qnn_sdk}/lib/{self.target}/libQnnHtp.so",
             (
                 f"{self.qnn_sdk}/lib/hexagon-v{self.htp_arch}/"
                 f"unsigned/libQnnHtpV{self.htp_arch}Skel.so"
             ),
-            (
-                f"{self.qnn_sdk}/lib/aarch64-android/"
-                f"libQnnHtpV{self.htp_arch}Stub.so"
-            ),
-            f"{self.qnn_sdk}/lib/aarch64-android/libQnnHtpPrepare.so",
-            f"{self.qnn_sdk}/lib/aarch64-android/libQnnSystem.so",
+            (f"{self.qnn_sdk}/lib/{self.target}/" f"libQnnHtpV{self.htp_arch}Stub.so"),
+            f"{self.qnn_sdk}/lib/{self.target}/libQnnHtpPrepare.so",
+            f"{self.qnn_sdk}/lib/{self.target}/libQnnSystem.so",
             f"{self.build_path}/{self.runner}",
             f"{self.build_path}/backends/qualcomm/libqnn_executorch_backend.so",
-            f"{self.qnn_sdk}/lib/aarch64-android/libQnnModelDlc.so",
+            f"{self.qnn_sdk}/lib/{self.target}/libQnnModelDlc.so",
         ]
         input_list_file, input_files = generate_inputs(
             self.working_dir, self.input_list_filename, inputs, input_list