Qualcomm AI Engine Direct - CI for Non-LLM GA model Part1

Author: winskuo-quic

.ci/scripts/test_model.sh

@@ -192,16 +192,37 @@ test_model_with_qnn() {
     EXPORT_SCRIPT=edsr
     # Additional deps for edsr
     pip install piq
+  elif [[ "${MODEL_NAME}" == "deit" ]]; then
+    EXPORT_SCRIPT=deit
+  elif [[ "${MODEL_NAME}" == "pvt" ]]; then
+    EXPORT_SCRIPT=pvt
+  elif [[ "${MODEL_NAME}" == "swin" ]]; then
+    EXPORT_SCRIPT=swin_transformer
   else
     echo "Unsupported model $MODEL_NAME"
     exit 1
   fi
 
+  SCRIPT_FOLDER=""
+  case "${MODEL_NAME}" in
+    "dl3"|"mv3"|"mv2"|"ic4"|"ic3"|"vit"|"mb"|"w2l")
+        SCRIPT_FOLDER=scripts
+        ;;
+    "deit"|"pvt"|"swin")
+        SCRIPT_FOLDER=oss_scripts
+        ;;
+    *)
+        echo "Unsupported model $MODEL_NAME"
+        exit 1
+        ;;
+  esac
+  
+
   # Use SM8450 for S22, SM8550 for S23, and SM8560 for S24
   # TODO(guangyang): Make QNN chipset matches the target device
   QNN_CHIPSET=SM8450
 
-  "${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} --ci --compile_only $EXTRA_FLAGS
+  "${PYTHON_EXECUTABLE}" -m examples.qualcomm.${SCRIPT_FOLDER}.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} --ci --compile_only $EXTRA_FLAGS
   EXPORTED_MODEL=$(find "./${EXPORT_SCRIPT}" -type f -name "${MODEL_NAME}*.pte" -print -quit)
 }
 

.github/workflows/trunk.yml

@@ -479,6 +479,32 @@ jobs:
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh
         PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh
         PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh ${{ matrix.model }} "cmake" "qnn"
+  
+  test-qnn-optimum-model:
+    name: test-qnn-optimum-model
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      matrix:
+        dtype: [fp32]
+        model: [deit, pvt, swin]
+      fail-fast: false
+    with:
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-qnn-sdk
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 900
+      script: |
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh
+        PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh
+        PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh ${{ matrix.model }} "cmake" "qnn"
 
   test-apple-model:
     name: test-apple-model

backends/qualcomm/builders/op_slice_copy.py

@@ -55,7 +55,7 @@ def define_node(
         if start < 0:
             start = start % input_tensor.shape[dim]
 
-        if len(node.args) > 3:
+        if len(node.args) > 3 and node.args[3] is not None:
             end = min(cast(int, node.args[3]), input_tensor.shape[dim])
             if end < 0:
                 end = end % input_tensor.shape[dim]

examples/qualcomm/oss_scripts/deit.py

@@ -6,10 +6,12 @@
 
 import getpass
 import json
+import logging
 import os
 from multiprocessing.connection import Client
 
 import numpy as np
+import torch
 from executorch.backends.qualcomm._passes.qnn_pass_manager import (
     get_capture_program_passes,
 )
@@ -46,12 +48,19 @@ def main(args):
     data_num = 100
     height = config.image_size
     width = config.image_size
-    inputs, targets, input_list = get_imagenet_dataset(
-        dataset_path=f"{args.dataset}",
-        data_size=data_num,
-        image_shape=(height, width),
-        crop_size=(height, width),
-    )
+
+    if args.ci:
+        inputs = [(torch.rand(1, 3, height, width),)]
+        logging.warning(
+            "This option is for CI to verify the export flow. It uses random input and will result in poor accuracy."
+        )
+    else:
+        inputs, targets, input_list = get_imagenet_dataset(
+            dataset_path=f"{args.dataset}",
+            data_size=data_num,
+            image_shape=(height, width),
+            crop_size=(height, width),
+        )
 
     # Get the Deit model.
     model = get_instance()
@@ -134,7 +143,7 @@ def main(args):
             "for https://www.kaggle.com/datasets/ifigotin/imagenetmini-1000)"
         ),
         type=str,
-        required=True,
+        required=False,
     )
 
     args = parser.parse_args()

examples/qualcomm/oss_scripts/swin_transformer.py

@@ -89,7 +89,7 @@ def main(args):
 
     data_num = 100
     if args.ci:
-        inputs = [torch.rand(1, 3, 224, 224)]
+        inputs = [(torch.rand(1, 3, 224, 224),)]
         logging.warning(
             "This option is for CI to verify the export flow. It uses random input and will result in poor accuracy."
         )