[Backend Tester] Run Core ML tests in nightly CI

ghstack-source-id: deb6486
ghstack-comment-id: 3190288611
Pull-Request: #13446

Author: GregoryComer

.ci/scripts/backend-test-macos.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+set -eux
+
+SUITE=$1
+FLOW=$2
+ARTIFACT_DIR=$3
+
+echo "Running backend test job for suite $SUITE, flow $FLOW."
+echo "Saving job artifacts to $ARTIFACT_DIR."
+
+eval "$(conda shell.bash hook)"
+CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+conda activate "${CONDA_ENV}"
+
+PYTHON_EXECUTABLE=python .ci/scripts/setup-macos.sh --build-tool cmake --build-mode Release
+
+python -m executorch.backends.test.suite.runner $SUITE --flow $FLOW --report "$ARTIFACT_DIR/test_results.csv"

.github/workflows/nightly.yml

@@ -50,7 +50,30 @@ jobs:
       docker-image: ci-image:executorch-ubuntu-22.04-clang12
       submodules: recursive
       timeout: 90
-      upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }
+      upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }}
       script: |
         set -eux
         .ci/scripts/backend-test-linux.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}"
+
+  backend-test-macos:
+    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
+    strategy:
+      fail-fast: false
+      matrix:
+        flow: [coreml, coreml_static_int8]
+        suite: [models, operators]
+    with:
+      job-name: build
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      runner: macos-latest-xlarge
+      python-version: 3.12
+      submodules: recursive
+      timeout: 90
+      upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }}
+      script: |
+        set -eux
+
+        # This is needed to get the prebuilt PyTorch wheel from S3
+        ${CONDA_RUN} --no-capture-output pip install awscli==1.37.21
+
+        .ci/scripts/backend-test-macos.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}"