Update

[ghstack-poisoned]

Author: swolchok

.ci/scripts/setup-qnn-deps.sh

@@ -7,47 +7,7 @@
 
 set -ex
 
-verify_pkg_installed() {
-  echo $(dpkg-query -W --showformat='${Status}\n' $1|grep "install ok installed")
-}
+source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh"
 
-install_qnn() {
-  echo "Start installing qnn."
-  QNN_INSTALLATION_DIR=/tmp/qnn
-  mkdir -p "${QNN_INSTALLATION_DIR}"
-
-  curl -Lo /tmp/v2.28.0.24.10.29.zip "https://softwarecenter.qualcomm.com/api/download/software/qualcomm_neural_processing_sdk/v2.28.0.241029.zip"
-  echo "Finishing downloading qnn sdk."
-  unzip -qo /tmp/v2.28.0.24.10.29.zip -d /tmp
-  echo "Finishing unzip qnn sdk."
-
-
-  # Print the content for manual verification
-  ls -lah "/tmp/qairt"
-  mv "/tmp/qairt"/* "${QNN_INSTALLATION_DIR}"
-  echo "Finishing installing qnn '${QNN_INSTALLATION_DIR}' ."
-
-  ls -lah "${QNN_INSTALLATION_DIR}"
-}
-
-setup_libc++() {
-  clang_version=$1
-  sudo apt-get update
-  pkgs_to_check=("libc++-${clang_version}-dev")
-  j=0
-  while [ $j -lt ${#pkgs_to_check[*]} ]; do
-    install_status=$(verify_pkg_installed ${pkgs_to_check[$j]})
-    if [ "$install_status" == "" ]; then
-      sudo apt-get install -y ${pkgs_to_check[$j]}
-      if [[ $? -ne 0 ]]; then
-        echo "ERROR: Failed to install required packages for libc++"
-        exit 1
-      fi
-    fi
-    j=$(( $j +1));
-  done
-}
-
-# This needs to match with the clang version from the Docker image
-setup_libc++ 12
+setup_libcpp 12
 install_qnn

backends/qualcomm/scripts/install_qnn_sdk.sh

@@ -0,0 +1,153 @@
+set -ex
+
+# Get the absolute path of this script
+SCRIPT_DIR="$( cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 ; pwd -P )"
+
+# Source QNN configuration from the same directory
+source "${SCRIPT_DIR}/qnn_config.sh"
+
+# Function to install Android NDK (only if not already set)
+setup_android_ndk() {
+    # Check if ANDROID_NDK_ROOT is already set and valid
+    if [ -n "${ANDROID_NDK_ROOT}" ] && [ -d "${ANDROID_NDK_ROOT}" ]; then
+        echo "Android NDK already set to ${ANDROID_NDK_ROOT} - skipping installation"
+        return
+    fi
+
+    NDK_VERSION="r26c"
+    NDK_INSTALL_DIR="/tmp/android-ndk"
+
+    if [ -d "${NDK_INSTALL_DIR}/ndk" ]; then
+        echo "Android NDK already installed at ${NDK_INSTALL_DIR}/ndk"
+        export ANDROID_NDK_ROOT="${NDK_INSTALL_DIR}/ndk"
+        return
+    fi
+
+    echo "Installing Android NDK ${NDK_VERSION}"
+    mkdir -p "${NDK_INSTALL_DIR}"
+    NDK_ZIP="android-ndk-${NDK_VERSION}-linux.zip"
+
+    curl -Lo "/tmp/${NDK_ZIP}" "https://dl.google.com/android/repository/${NDK_ZIP}"
+    unzip -q "/tmp/${NDK_ZIP}" -d "${NDK_INSTALL_DIR}"
+    mv "${NDK_INSTALL_DIR}/android-ndk-${NDK_VERSION}" "${NDK_INSTALL_DIR}/ndk"
+
+    export ANDROID_NDK_ROOT="${NDK_INSTALL_DIR}/ndk"
+    echo "Android NDK installed to ${ANDROID_NDK_ROOT}"
+}
+
+verify_pkg_installed() {
+  dpkg-query -W --showformat='${Status}\n' "$1" | grep -q "install ok installed"
+}
+
+install_qnn() {
+  # Check if QNN_SDK_ROOT is already set and valid
+  if [ -n "${QNN_SDK_ROOT}" ] && [ -d "${QNN_SDK_ROOT}" ]; then
+    echo "QNN SDK already set to ${QNN_SDK_ROOT} - skipping installation"
+    return
+  fi
+
+  echo "Start installing qnn v${QNN_VERSION}"
+  QNN_INSTALLATION_DIR="/tmp/qnn"
+
+  # Clean up any previous installation
+  if [ -d "${QNN_INSTALLATION_DIR}" ]; then
+    echo "Removing previous QNN installation at ${QNN_INSTALLATION_DIR}"
+    rm -rf "${QNN_INSTALLATION_DIR}"
+  fi
+
+  mkdir -p "${QNN_INSTALLATION_DIR}"
+
+  QNN_ZIP_FILE="v${QNN_VERSION}.zip"
+  curl -Lo "/tmp/${QNN_ZIP_FILE}" "${QNN_ZIP_URL}"
+  echo "Finishing downloading qnn sdk."
+  unzip -qo "/tmp/${QNN_ZIP_FILE}" -d /tmp
+  echo "Finishing unzip qnn sdk."
+
+  # Print the content for manual verification
+  echo "Contents of /tmp/qairt:"
+  ls -lah "/tmp/qairt"
+
+  # Move the specific version directory
+  if [ -d "/tmp/qairt/${QNN_VERSION}" ]; then
+    mv "/tmp/qairt/${QNN_VERSION}" "${QNN_INSTALLATION_DIR}"
+  else
+    mv "/tmp/qairt"/* "${QNN_INSTALLATION_DIR}"
+  fi
+
+  echo "Finishing installing qnn '${QNN_INSTALLATION_DIR}' ."
+  echo "Final QNN installation contents:"
+  ls -lah "${QNN_INSTALLATION_DIR}"
+
+  # Set QNN_SDK_ROOT environment variable
+  export QNN_SDK_ROOT="${QNN_INSTALLATION_DIR}"
+  echo "Set QNN_SDK_ROOT=${QNN_SDK_ROOT}"
+}
+
+setup_libcpp() {
+  clang_version=$1
+  LLVM_VERSION="14.0.0"
+  INSTALL_DIR="/tmp/libcxx-${LLVM_VERSION}"
+
+  # Check if we already have a local installation
+  if [ -d "${INSTALL_DIR}/include" ] && [ -d "${INSTALL_DIR}/lib" ]; then
+    echo "Local libc++ already installed at ${INSTALL_DIR} - skipping"
+    # Set environment variables
+    export CPLUS_INCLUDE_PATH="${INSTALL_DIR}/include:$CPLUS_INCLUDE_PATH"
+    export LD_LIBRARY_PATH="${INSTALL_DIR}/lib:$LD_LIBRARY_PATH"
+    export LIBRARY_PATH="${INSTALL_DIR}/lib:$LIBRARY_PATH"
+    return
+  fi
+
+  echo "Installing libc++ manually to ${INSTALL_DIR}"
+
+  # Create temporary directory
+  TEMP_DIR=$(mktemp -d)
+  # Ensure cleanup on exit or return
+  trap 'rm -rf "$TEMP_DIR"' RETURN
+
+  pushd "${TEMP_DIR}" >/dev/null
+
+  BASE_NAME="clang+llvm-${LLVM_VERSION}-x86_64-linux-gnu-ubuntu-18.04"
+  LLVM_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/${BASE_NAME}.tar.xz"
+
+  echo "Downloading LLVM from ${LLVM_URL}"
+  curl -fLO "${LLVM_URL}" || {
+      echo "Error: Failed to download LLVM"
+      exit 1
+  }
+
+  echo "Extracting ${BASE_NAME}.tar.xz"
+  tar -xf "${BASE_NAME}.tar.xz" || {
+      echo "Error: Failed to extract LLVM archive"
+      exit 1
+  }
+
+  # Create installation directory
+  mkdir -p "${INSTALL_DIR}/include"
+  mkdir -p "${INSTALL_DIR}/lib"
+
+  # Copy libc++ headers and libraries
+  cp -r "${BASE_NAME}/include/c++/v1/"* "${INSTALL_DIR}/include/"
+  cp -r "${BASE_NAME}/lib/"*.so* "${INSTALL_DIR}/lib/"
+
+  popd >/dev/null
+
+  # Create necessary symlinks locally
+  pushd "${INSTALL_DIR}/lib" >/dev/null
+  ln -sf libc++.so.1.0 libc++.so.1
+  ln -sf libc++.so.1 libc++.so
+  ln -sf libc++abi.so.1.0 libc++abi.so.1
+  ln -sf libc++abi.so.1 libc++abi.so
+  popd >/dev/null
+
+  # Set environment variables
+  export CPLUS_INCLUDE_PATH="${INSTALL_DIR}/include:$CPLUS_INCLUDE_PATH"
+  export LD_LIBRARY_PATH="${INSTALL_DIR}/lib:$LD_LIBRARY_PATH"
+  export LIBRARY_PATH="${INSTALL_DIR}/lib:$LIBRARY_PATH"
+
+  echo "libc++ installed to ${INSTALL_DIR}"
+}
+
+setup_libcpp 12
+setup_android_ndk
+install_qnn

backends/qualcomm/scripts/qnn_config.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# QNN SDK Configuration
+QNN_VERSION="2.28.0.241029"
+QNN_ZIP_URL="https://softwarecenter.qualcomm.com/api/download/software/qualcomm_neural_processing_sdk/v${QNN_VERSION}.zip"

backends/test/suite/discovery.py

@@ -68,6 +68,10 @@ def _filter_tests(
 
 def _is_test_enabled(test_case: unittest.TestCase, test_filter: TestFilter) -> bool:
     test_method = getattr(test_case, test_case._testMethodName)
+
+    if not hasattr(test_method, "_flow"):
+        print(f"Test missing flow: {test_method}")
+
     flow: TestFlow = test_method._flow
 
     if test_filter.backends is not None and flow.backend not in test_filter.backends:

backends/test/suite/models/__init__.py

@@ -0,0 +1,136 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# pyre-unsafe
+
+import itertools
+import os
+import unittest
+from typing import Any, Callable
+
+import torch
+from executorch.backends.test.harness import Tester
+from executorch.backends.test.suite import get_test_flows
+from executorch.backends.test.suite.context import get_active_test_context, TestContext
+from executorch.backends.test.suite.flow import TestFlow
+from executorch.backends.test.suite.reporting import log_test_summary
+from executorch.backends.test.suite.runner import run_test
+
+
+DTYPES: list[torch.dtype] = [
+    torch.float16,
+    torch.float32,
+    torch.float64,
+]
+
+
+def load_tests(loader, suite, pattern):
+    package_dir = os.path.dirname(__file__)
+    discovered_suite = loader.discover(
+        start_dir=package_dir, pattern=pattern or "test_*.py"
+    )
+    suite.addTests(discovered_suite)
+    return suite
+
+
+def _create_test(
+    cls,
+    test_func: Callable,
+    flow: TestFlow,
+    dtype: torch.dtype,
+    use_dynamic_shapes: bool,
+):
+    def wrapped_test(self):
+        params = {
+            "dtype": dtype,
+            "use_dynamic_shapes": use_dynamic_shapes,
+        }
+        with TestContext(test_name, flow.name, params):
+            test_func(self, dtype, use_dynamic_shapes, flow.tester_factory)
+
+    dtype_name = str(dtype)[6:]  # strip "torch."
+    test_name = f"{test_func.__name__}_{flow.name}_{dtype_name}"
+    if use_dynamic_shapes:
+        test_name += "_dynamic_shape"
+
+    wrapped_test._name = test_func.__name__  # type: ignore
+    wrapped_test._flow = flow  # type: ignore
+
+    setattr(cls, test_name, wrapped_test)
+
+
+# Expand a test into variants for each registered flow.
+def _expand_test(cls, test_name: str) -> None:
+    test_func = getattr(cls, test_name)
+    supports_dynamic_shapes = getattr(test_func, "supports_dynamic_shapes", True)
+    dynamic_shape_values = [True, False] if supports_dynamic_shapes else [False]
+    dtypes = getattr(test_func, "dtypes", DTYPES)
+
+    for flow, dtype, use_dynamic_shapes in itertools.product(
+        get_test_flows().values(), dtypes, dynamic_shape_values
+    ):
+        _create_test(cls, test_func, flow, dtype, use_dynamic_shapes)
+    delattr(cls, test_name)
+
+
+def model_test_cls(cls) -> Callable | None:
+    """Decorator for model tests. Handles generating test variants for each test flow and configuration."""
+    for key in dir(cls):
+        if key.startswith("test_"):
+            _expand_test(cls, key)
+    return cls
+
+
+def model_test_params(
+    supports_dynamic_shapes: bool = True,
+    dtypes: list[torch.dtype] | None = None,
+) -> Callable:
+    """Optional parameter decorator for model tests. Specifies test pararameters. Only valid with a class decorated by model_test_cls."""
+
+    def inner_decorator(func: Callable) -> Callable:
+        func.supports_dynamic_shapes = supports_dynamic_shapes  # type: ignore
+
+        if dtypes is not None:
+            func.dtypes = dtypes  # type: ignore
+
+        return func
+
+    return inner_decorator
+
+
+def run_model_test(
+    model: torch.nn.Module,
+    inputs: tuple[Any],
+    dtype: torch.dtype,
+    dynamic_shapes: Any | None,
+    tester_factory: Callable[[], Tester],
+):
+    model = model.to(dtype)
+    context = get_active_test_context()
+
+    # This should be set in the wrapped test. See _create_test above.
+    assert context is not None, "Missing test context."
+
+    run_summary = run_test(
+        model,
+        inputs,
+        tester_factory,
+        context.test_name,
+        context.flow_name,
+        context.params,
+        dynamic_shapes=dynamic_shapes,
+    )
+
+    log_test_summary(run_summary)
+
+    if not run_summary.result.is_success():
+        if run_summary.result.is_backend_failure():
+            raise RuntimeError("Test failure.") from run_summary.error
+        else:
+            # Non-backend failure indicates a bad test. Mark as skipped.
+            raise unittest.SkipTest(
+                f"Test failed for reasons other than backend failure. Error: {run_summary.error}"
+            )