Update

[ghstack-poisoned]

Author: GregoryComer

.ci/docker/common/install_zephyr.sh

@@ -8,6 +8,9 @@
 
 set -ex
 
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
 # Double check if the NDK version is set
 [ -n "${ZEPHYR_SDK}" ]
 
@@ -77,16 +80,8 @@ install_prerequiresites() {
         chmod +x kitware-archive.sh && \
         ./kitware-archive.sh && \
         rm -f kitware-archive.sh
-    useradd -d /home/zephyruser -m -s /bin/bash zephyruser
-}
-
-install_sdk() {
-    wget https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.0/zephyr-sdk-0.16.0_linux-x86_64.tar.xz && \
-         tar -xf zephyr-sdk-0.16.0_linux-x86_64.tar.xz && \
-         rm -f zephyr-sdk-0.16.0_linux-x86_64.tar.xz && \
-         cd zephyr-sdk-0.16.0/ && \
-         ./setup.sh -c -t arm-zephyr-eabi
+    pip_install --no-cache-dir west
+    pip_install pyelftools
 }
 
 install_prerequiresites
-install_sdk

.ci/docker/ubuntu/Dockerfile

@@ -86,8 +86,9 @@ ARG ARM_SDK
 
 ARG ZEPHYR_SDK
 COPY ./common/install_zephyr.sh install_zephyr.sh
+COPY ./common/utils.sh utils.sh
 RUN if [ -n "${ZEPHYR_SDK}" ]; then bash ./install_zephyr.sh; fi
-RUN rm install_zephyr.sh
+RUN rm install_zephyr.sh utils.sh
 
 ARG QNN_SDK
 

.ci/scripts/zephyr-utils.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+download_arm_zephyr_sdk () {
+    wget https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.0/zephyr-sdk-0.16.0_linux-x86_64.tar.xz
+    tar -xf zephyr-sdk-0.16.0_linux-x86_64.tar.xz
+    rm -f zephyr-sdk-0.16.0_linux-x86_64.tar.xz
+}
+
+setup_zephyr_et_module () {
+    git clone --branch executorch-module-integration https://github.com/BujSet/zephyr.git
+    west init -l zephyr
+    west config manifest.project-filter -- +executorch
+    west -v update
+}

.github/workflows/trunk.yml

@@ -55,6 +55,49 @@ jobs:
         # Build and test executorch
         PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}"
 
+  test-models-arm-zephyr:
+    name: test-models-arm-zephyr
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    strategy:
+      matrix:
+        model: [add]
+      fail-fast: false
+    with:
+      runner: linux.2xlarge
+      docker-image: ci-image:executorch-ubuntu-22.04-zephyr-sdk
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 120
+      script: |
+        MODEL_NAME=${{ matrix.model }}
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        source .ci/scripts/utils.sh
+        source .ci/scripts/zephyr-utils.sh
+        mkdir -p zephyr_scratch/
+        cd zephyr_scratch
+        export ZEPHYR_PROJ_ROOT=$(realpath $(pwd))
+
+        download_arm_zephyr_sdk
+        ./zephyr-sdk-0.16.0/setup.sh -c -t arm-zephyr-eabi
+
+        cd $ZEPHYR_PROJ_ROOT
+        setup_zephyr_et_module
+
+        cd $ZEPHYR_PROJ_ROOT/modules/lib/executorch
+        install_executorch "--use-pt-pinned-commit"
+        .ci/scripts/setup-arm-baremetal-tools.sh --target-toolchain zephyr
+        source examples/arm/ethos-u-scratch/setup_path.sh
+        source $ZEPHYR_PROJ_ROOT/zephyr/zephyr-env.sh
+        cd $ZEPHYR_PROJ_ROOT/zephyr/samples/modules/executorch/arm/hello_world
+        west build -p always -b mps3/corstone300/fvp
+        FVP_Corstone_SSE-300_Ethos-U55 -a build/zephyr/zephyr.elf -C mps3_board.visualisation.disable-visualisation=1 -C mps3_board.telnetterminal0.start_telnet=0 -C mps3_board.uart0.out_file='sim.out'  -C cpu0.CFGITCMSZ=15 -C cpu0.CFGDTCMSZ=15 --simlimit 120
+
+        grep -qF "Output[0][0]: (float) 2.000000" sim.out
+        exit_status=$? #store 0 if found (success), 1 if not (failure)
+        exit $exit_status
+
   test-models-linux-aarch64:
     name: test-models-linux-aarch64
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main

backends/test/harness/tester.py

@@ -1,4 +1,3 @@
-import math
 import random
 from collections import Counter, OrderedDict
 from typing import Any, Callable, Dict, List, Optional, Tuple
@@ -18,7 +17,6 @@
     ToExecutorch,
 )
 from executorch.exir.dim_order_utils import get_memory_format
-from torch.ao.ns.fx.utils import compute_sqnr
 
 from torch.export import ExportedProgram
 from torch.testing import FileCheck
@@ -304,13 +302,13 @@ def run_method_and_compare_outputs(
         atol=1e-03,
         rtol=1e-03,
         qtol=0,
-        snr: float | None = None,
     ):
         number_of_runs = 1 if inputs is not None else num_runs
         reference_stage = self.stages[StageType.EXPORT]
 
         stage = stage or self.cur
 
+        print(f"Comparing Stage {stage} with Stage {reference_stage}")
         for run_iteration in range(number_of_runs):
             inputs_to_run = inputs if inputs else next(self.generate_random_inputs())
             input_shapes = [
@@ -330,21 +328,13 @@ def run_method_and_compare_outputs(
             # Output from running artifact at stage
             stage_output = self.stages[stage].run_artifact(inputs_to_run)
             self._compare_outputs(
-                reference_output,
-                stage_output,
-                quantization_scale,
-                atol,
-                rtol,
-                qtol,
-                snr,
+                reference_output, stage_output, quantization_scale, atol, rtol, qtol
             )
 
         return self
 
     @staticmethod
-    def _assert_outputs_equal(
-        model_output, ref_output, atol=1e-03, rtol=1e-03, snr: float | None = None
-    ):
+    def _assert_outputs_equal(model_output, ref_output, atol=1e-03, rtol=1e-03):
         """
         Helper testing function that asserts that the model output and the reference output
         are equal with some tolerance. Due to numerical differences between eager mode and
@@ -369,22 +359,15 @@ def _assert_outputs_equal(
                     f"\tMismatched count: {(model != ref).sum().item()} / {model.numel()}\n"
                 )
             else:
-                computed_snr = compute_sqnr(model.to(torch.float), ref.to(torch.float))
-                snr = snr or float("-inf")
-
-                assert (
-                    torch.allclose(
-                        model,
-                        ref,
-                        atol=atol,
-                        rtol=rtol,
-                        equal_nan=True,
-                    )
-                    and computed_snr >= snr
-                    or math.isnan(computed_snr)
+                assert torch.allclose(
+                    model,
+                    ref,
+                    atol=atol,
+                    rtol=rtol,
+                    equal_nan=True,
                 ), (
                     f"Output {i} does not match reference output.\n"
-                    f"\tGiven atol: {atol}, rtol: {rtol}, snr: {snr}.\n"
+                    f"\tGiven atol: {atol}, rtol: {rtol}.\n"
                     f"\tOutput tensor shape: {model.shape}, dtype: {model.dtype}\n"
                     f"\tDifference: max: {torch.max(model-ref)}, abs: {torch.max(torch.abs(model-ref))}, mean abs error: {torch.mean(torch.abs(model-ref).to(torch.double))}.\n"
                     f"\t-- Model vs. Reference --\n"
@@ -393,7 +376,6 @@ def _assert_outputs_equal(
                     f"\t  Mean: {model.to(torch.double).mean()}, {ref.to(torch.double).mean()}\n"
                     f"\t   Max: {model.max()}, {ref.max()}\n"
                     f"\t   Min: {model.min()}, {ref.min()}\n"
-                    f"\t   SNR: {computed_snr}\n"
                 )
 
     @staticmethod
@@ -404,7 +386,6 @@ def _compare_outputs(
         atol=1e-03,
         rtol=1e-03,
         qtol=0,
-        snr: float | None = None,
     ):
         """
         Compares the original of the original nn module with the output of the generated artifact.
@@ -427,7 +408,6 @@ def _compare_outputs(
             reference_output,
             atol=atol,
             rtol=rtol,
-            snr=snr,
         )
 
     @staticmethod

backends/test/suite/operators/test_abs.py

@@ -49,9 +49,9 @@ def test_abs_edge_cases(self, flow: TestFlow) -> None:
         # Test edge cases
 
         # Tensor with infinity
-        x = (torch.tensor([float("inf"), float("-inf"), 1.0, -1.0]),)
+        x = torch.tensor([float("inf"), float("-inf"), 1.0, -1.0])
         self._test_op(AbsModel(), (x,), flow, generate_random_test_inputs=False)
 
         # Tensor with NaN
-        x = (torch.tensor([float("nan"), 1.0, -1.0]),)
+        x = torch.tensor([float("nan"), 1.0, -1.0])
         self._test_op(AbsModel(), (x,), flow, generate_random_test_inputs=False)

backends/test/suite/runner.py

@@ -104,10 +104,7 @@ def build_result(
         # AssertionErrors to catch output mismatches, but this might catch more than that.
         try:
             tester.run_method_and_compare_outputs(
-                inputs=None if generate_random_test_inputs else inputs,
-                atol=5e-2,
-                rtol=5e-2,
-                snr=40,
+                inputs=None if generate_random_test_inputs else inputs
             )
         except AssertionError as e:
             return build_result(TestResult.OUTPUT_MISMATCH_FAIL, e)

extension/android/build.gradle

@@ -17,7 +17,7 @@ allprojects {
 
         dependencies {
             classpath 'com.android.tools.build:gradle:8.9.0'
-            classpath 'com.vanniktech:gradle-maven-publish-plugin:0.31.0'
+            classpath 'com.vanniktech:gradle-maven-publish-plugin:0.34.0'
         }
 
     }

extension/android/executorch_android/build.gradle

@@ -58,13 +58,11 @@ dependencies {
     androidTestImplementation 'org.jetbrains.kotlin:kotlin-test:1.9.23'
 }
 
-import com.vanniktech.maven.publish.SonatypeHost
-
 mavenPublishing {
-  publishToMavenCentral(SonatypeHost.DEFAULT)
+  publishToMavenCentral()
   signAllPublications()
 
-  coordinates("org.pytorch", "executorch-android", "0.7.0")
+  coordinates("org.pytorch", "executorch-android", "0.7.0-SNAPSHOT")
 
   pom {
     name = "ExecuTorch Android"
@@ -96,6 +94,6 @@ mavenPublishing {
 
 repositories {
     maven {
-        url "https://oss.sonatype.org/content/repositories/snapshots"
+        url "https://central.sonatype.com/repository/maven-snapshots/"
     }
 }