Update on "Support export program in intermediate numeric discrepancy detector"

This diff enables intermediate numeric discrepancy detector to leverage export program as label.

More specific, if user creates etrecord with exported program, and the exported program is one of the exported programs in the export flow, then our numeric discrepancy detector will use it as label. Otherwise, we will continue use edge dialect graph as label.

Differential Revision: [D78298935](https://our.internmc.facebook.com/intern/diff/D78298935/)

[ghstack-poisoned]

Author: Gasoonjia

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-ab43fe4bdf5ccd82897f0e982c451a0127bd175e
+90f1e7bed15ca5e48c61c5b6dc5ad4810524f82f

.github/workflows/trunk.yml

@@ -302,6 +302,37 @@ jobs:
           exit 1
         fi
 
+  test-arm-ootb-linux:
+    name: test-arm-ootb-linux
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-arm-sdk
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
+      script: |
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        # Follow the steps required before running the notebooks
+        # Try to mirror these as closely as possible
+        source .ci/scripts/utils.sh
+        install_executorch "--use-pt-pinned-commit"
+
+        .ci/scripts/setup-arm-baremetal-tools.sh
+        source examples/arm/ethos-u-scratch/setup_path.sh
+
+        # Install requirements for converting notebooks
+        pip install notebook
+
+        # Run OOTB tests
+        backends/arm/test/test_arm_ootb.sh
+
   test-coreml-delegate:
     name: test-coreml-delegate
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main

CMakeLists.txt

@@ -48,8 +48,6 @@
 cmake_minimum_required(VERSION 3.24)
 project(executorch)
 
-# MARK: - Start EXECUTORCH_H12025_BUILD_MIGRATION
-
 include(${PROJECT_SOURCE_DIR}/tools/cmake/common/preset.cmake)
 include(${PROJECT_SOURCE_DIR}/tools/cmake/Utils.cmake)
 include(CMakeDependentOption)
@@ -82,6 +80,7 @@ announce_configured_options(BUCK2)
 
 announce_configured_options(CMAKE_CXX_COMPILER_ID)
 announce_configured_options(CMAKE_TOOLCHAIN_FILE)
+announce_configured_options(BUILD_TESTING)
 
 load_build_preset()
 include(${PROJECT_SOURCE_DIR}/tools/cmake/preset/default.cmake)
@@ -97,11 +96,6 @@ else()
 endif()
 announce_configured_options(CCACHE_PROGRAM)
 
-# Print all the configs that were called with announce_configured_options.
-print_configured_options()
-
-# MARK: - End EXECUTORCH_H12025_BUILD_MIGRATION
-
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # Setup RPATH. See
@@ -750,3 +744,6 @@ if(EXECUTORCH_BUILD_ANDROID_JNI)
 endif()
 
 include(Test.cmake)
+
+# Print all the configs that were called with announce_configured_options.
+print_configured_options()

CMakePresets.json

@@ -8,7 +8,7 @@
     },
     {
       "name": "macos",
-      "displayName": "Build everything buildable on macOS",
+      "displayName": "Build ExecuTorch for macOS",
       "inherits": ["common"],
       "generator": "Xcode",
       "cacheVariables": {
@@ -25,7 +25,7 @@
     },
     {
       "name": "ios",
-      "displayName": "Build everything buildable on iOS",
+      "displayName": "Build ExecuTorch for iOS",
       "inherits": ["common"],
       "generator": "Xcode",
       "cacheVariables": {
@@ -42,7 +42,7 @@
     },
     {
       "name": "ios-simulator",
-      "displayName": "Build everything buildable on iOS simulator",
+      "displayName": "Build ExecuTorch for iOS Simulator",
       "inherits": ["common"],
       "generator": "Xcode",
       "cacheVariables": {
@@ -59,7 +59,7 @@
     },
     {
       "name": "linux",
-      "displayName": "Build everything buildable on Linux",
+      "displayName": "Build ExecuTorch for Linux",
       "inherits": ["common"],
       "cacheVariables": {
         "CMAKE_SYSTEM_NAME": "Linux",
@@ -88,29 +88,21 @@
     {
         "name": "llm",
         "displayName": "Build LLM libraries",
-        "inherits": [
-            "common"
-        ],
+        "inherits": ["common"],
         "cacheVariables": {
             "EXECUTORCH_BUILD_PRESET_FILE": "${sourceDir}/tools/cmake/preset/llm.cmake",
             "CMAKE_OSX_DEPLOYMENT_TARGET": "12.0"
         },
         "condition": {
             "type": "inList",
             "string": "${hostSystemName}",
-            "list": [
-                "Darwin",
-                "Linux",
-                "Windows"
-            ]
+            "list": ["Darwin", "Linux", "Windows"]
         }
     },
     {
         "name": "zephyr",
-        "displayName": "Build everything buildable on Zephyr RTOS",
-        "inherits": [
-            "common"
-        ],
+        "displayName": "Build ExecuTorch for Zephyr RTOS",
+        "inherits": ["common"],
         "cacheVariables": {
             "EXECUTORCH_BUILD_PRESET_FILE": "${sourceDir}/tools/cmake/preset/zephyr.cmake",
             "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/examples/zephyr/x86_64-linux-arm-zephyr-eabi-gcc.cmake"

backends/arm/_passes/decompose_asin_pass.py

@@ -85,12 +85,11 @@ def _build_polynomial(
         return result
 
     def call_operator(self, op, args, kwargs, meta):
+        if op not in edge_asin_op:
+            return super().call_operator(op, args, kwargs, meta)
         logging.info(
             f"Approximating asin. This may introduce small numerical errors. For details, see {__file__}."
         )
-        if op not in edge_asin_op:
-            return super().call_operator(op, args, kwargs, meta)
-
         x = args[0]
         half = 0.5
         one = 1.0

backends/arm/test/ops/test_add.py

@@ -205,6 +205,5 @@ def test_add_tensor_vgf_int(test_data: input_t1):
         aten_op,
         exir_op,
         tosa_version="TOSA-1.0+INT",
-        symmetric_io_quantization=True,
     )
     pipeline.run()

backends/arm/test/test_arm_ootb.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+run_ootb_tests_ethos_u() {
+    echo "$FUNCNAME: Running out-of-the-box tests for Arm Ethos-U"
+    jupyter nbconvert \
+        --to notebook \
+        --execute examples/arm/ethos_u_minimal_example.ipynb
+    echo "${FUNCNAME}: PASS"
+}
+
+run_ootb_tests_ethos_u

backends/arm/test/tester/arm_tester.py

@@ -43,6 +43,7 @@
     EthosUQuantizer,
     get_symmetric_quantization_config,
     TOSAQuantizer,
+    VgfQuantizer,
 )
 from executorch.backends.arm.test.runner_utils import (
     dbg_tosa_fb_to_json,
@@ -332,6 +333,8 @@ def quantize(
                 quantizer = TOSAQuantizer(tosa_spec)
             elif is_ethosu(self.compile_spec):
                 quantizer = EthosUQuantizer(self.compile_spec)
+            elif is_vgf(self.compile_spec):
+                quantizer = VgfQuantizer(self.compile_spec)
             quantize_stage = tester.Quantize(
                 quantizer,
                 get_symmetric_quantization_config(),

backends/arm/test/tester/test_pipeline.py

@@ -861,18 +861,15 @@ def __init__(
         rtol: float = 1e-03,
         qtol: int = 1,
         dynamic_shapes: Optional[Tuple[Any]] = None,
+        transform_passes: Optional[
+            Union[Sequence[PassType], Dict[str, Sequence[PassType]]]
+        ] = None,
     ):
 
         if (
             symmetric_io_quantization or per_channel_quantization
         ) and tosa_version == "TOSA-1.0+FP":
             raise ValueError("Dont configure quantization with FP TOSA profile.")
-        if (
-            symmetric_io_quantization is False
-            and per_channel_quantization is False
-            and tosa_version == "TOSA-1.0+INT"
-        ):
-            raise ValueError("Missing quantization options for INT TOSA profile.")
 
         tosa_profile = TosaSpecification.create_from_string(tosa_version)
         compile_spec = common.get_vgf_compile_spec(
@@ -887,6 +884,7 @@ def __init__(
             exir_op,
             use_to_edge_transform_and_lower,
             dynamic_shapes,
+            transform_passes=transform_passes,
         )
 
         if symmetric_io_quantization or per_channel_quantization:
@@ -900,7 +898,7 @@ def __init__(
         else:
             quant_stage = None
 
-        if quant_stage:
+        if "INT" in tosa_version:
             self.add_stage(self.tester.quantize, quant_stage, pos=0)
 
             self.add_stage_after(

backends/cadence/aot/tests/test_fusion_ops_passes.py

@@ -12,7 +12,6 @@
 
 import executorch.backends.cadence.aot.ops_registrations  # noqa
 import torch
-from executorch.backends.cadence.aot import compiler
 from executorch.backends.cadence.aot.fuse_ops import (
     FuseCascadedTransposeOrPermuteOps,
     FuseCascadedViewOps,
@@ -30,7 +29,6 @@
 from executorch.exir.dialects._ops import ops as exir_ops
 from executorch.exir.dialects.edge._ops import EdgeOpOverload
 from executorch.exir.pass_base import PassResult, ProxyValue
-from torch import nn
 
 
 class TestFusionPassesBase(unittest.TestCase):
@@ -178,43 +176,6 @@ def test_keep_mm_add_with_multiple_users(self) -> None:
         self.assertEqual(count_node(converted_graph, exir_ops.edge.aten.mm.default), 1)
         self.assertEqual(count_node(converted_graph, exir_ops.edge.aten.add.Tensor), 3)
 
-    # TODO(matthiascremon) -> None: enable that pass with new flow
-    @torch.no_grad()
-    @unittest.expectedFailure
-    def test_legacy_conv_bn_fusion(self) -> None:
-        class ModelConvBN(torch.nn.Module):
-            def __init__(
-                self, in_features: int, out_features: int, kernel_size: int
-            ) -> None:
-                super().__init__()
-                self.conv1d = nn.Conv1d(in_features, out_features, kernel_size)
-                self.bn = nn.BatchNorm1d(out_features)
-
-            def forward(self, x: torch.Tensor) -> torch.Tensor:
-                y = self.conv1d(x)
-                return self.bn(y)
-
-        model = ModelConvBN(64, 1, 2)
-        x = torch.randn(1, 64, 4)
-
-        graph_module = (
-            compiler.export_to_executorch_gen_etrecord(model.eval(), (x,))
-            .exported_program()
-            .graph_module
-        )
-        # Assert that after running the fusion passes, batchnorm was fused with conv1d
-        self.assertEqual(
-            count_node(graph_module, torch.ops.aten.linear.out)
-            + count_node(graph_module, torch.ops.cadence.convolution.out),
-            1,
-        )
-        self.assertEqual(
-            count_node(
-                graph_module, torch.ops.aten._native_batch_norm_legit_no_training.out
-            ),
-            0,
-        )
-
     def test_permute_transpose_fusion(self) -> None:
         builder = GraphBuilder()
         x = builder.placeholder("x", torch.randn(3, 1, 3, 1, 4, dtype=torch.float32))