[Backend Tester] Add nightly CI job for XNNPACK (#13390)

Set up a nightly CI job to run the backend model and operator tests for
xnnpack (unquantized and static int8 per channel). This is the first
step in setting up continuous coverage using the new suite. I've chosen
nightly as the initial run cadence as some of the larger test suites are
very long. In the medium-term, I'd like to run a subset of these tests
on pull or trunk, but we can start with this.

Author: GregoryComer

.ci/scripts/setup-linux.sh

@@ -11,6 +11,7 @@ set -exu
 source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 
 read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@")
+echo "Build tool: $BUILD_TOOL, Mode: $BUILD_MODE"
 
 # As Linux job is running inside a Docker container, all of its dependencies
 # have already been installed, so we use PyTorch build from source here instead

.ci/scripts/test_backend_linux.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+SUITE=$1
+FLOW=$2
+ARTIFACT_DIR=$3
+
+echo "Running backend test job for suite $SUITE, flow $FLOW."
+echo "Saving job artifacts to $ARTIFACT_DIR."
+
+# The generic Linux job chooses to use base env, not the one setup by the image
+eval "$(conda shell.bash hook)"
+CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+conda activate "${CONDA_ENV}"
+
+# Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
+#source .ci/scripts/setup-vulkan-linux-deps.sh
+
+# We need the runner to test the built library.
+PYTHON_EXECUTABLE=python .ci/scripts/setup-linux.sh --build-tool cmake --build-mode Release
+
+python -m executorch.backends.test.suite.runner $SUITE --flow $FLOW --report "$ARTIFACT_DIR/test_results.csv"

.github/workflows/nightly.yml

@@ -36,3 +36,25 @@ jobs:
     uses: ./.github/workflows/_link_check.yml
     with:
       ref: ${{ github.sha }}
+
+  backend-test-linux:
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    strategy:
+      fail-fast: false
+      matrix:
+        flow: [xnnpack, xnnpack_static_int8_per_channel]
+        suite: [models, operators]
+    with:
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      runner: linux.4xlarge.memory
+      docker-image: ci-image:executorch-ubuntu-22.04-clang12
+      submodules: recursive
+      timeout: 120
+      upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }}
+      script: |
+        set -eux
+        # Intentionally suppressing exit code for now.
+        # TODO (gjcomer) Remove this when jobs are stable.
+        EXIT_CODE=0
+        .ci/scripts/test_backend_linux.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}" || EXIT_CODE=$?
+        echo "Test run complete with exit code $EXIT_CODE."