Remove --quantize_io pytest option

In most test cases we need the quantized_ops_aot
lib which this flag toggles loading.
Instead, always load the quantized_ops_aot lib,
unless we are only doing collection.

Signed-off-by: Erik Lundell <[email protected]>
Change-Id: I820b5decdf12a5825f4e52a0c0412bbfaf959761

Author: Erik-Lundell

backends/arm/README.md

@@ -119,7 +119,7 @@ backends/arm/test/setup_testing.sh
 The you can run the tests with
 
 ```
-pytest -c /dev/null -v -n auto backends/arm/test --arm_quantize_io --arm_run_corstoneFVP
+pytest -c /dev/null -v -n auto backends/arm/test --arm_run_corstoneFVP
 ```
 
 ### Code coverage

backends/arm/test/common.py

@@ -12,8 +12,6 @@
 from pathlib import Path
 
 from executorch.backends.arm.arm_backend import ArmCompileSpecBuilder
-
-from executorch.backends.arm.test.conftest import is_option_enabled
 from executorch.exir.backend.compile_spec_schema import CompileSpec
 
 
@@ -84,7 +82,7 @@ def get_tosa_compile_spec_unbuilt(
 
 
 def get_u55_compile_spec(
-    quantize_io=False,
+    quantize_io=True,
     custom_path=None,
     reorder_inputs=None,
 ) -> list[CompileSpec]:
@@ -99,7 +97,7 @@ def get_u55_compile_spec(
 
 
 def get_u85_compile_spec(
-    quantize_io=False,
+    quantize_io=True,
     custom_path=None,
     reorder_inputs=None,
 ) -> list[CompileSpec]:
@@ -114,7 +112,7 @@ def get_u85_compile_spec(
 
 
 def get_u55_compile_spec_unbuilt(
-    quantize_io=False,
+    quantize_io=True,
     custom_path=None,
     reorder_inputs=None,
 ) -> ArmCompileSpecBuilder:
@@ -132,15 +130,15 @@ def get_u55_compile_spec_unbuilt(
             memory_mode="Shared_Sram",
             extra_flags="--debug-force-regor --output-format=raw",
         )
-        .set_quantize_io(is_option_enabled("quantize_io") or quantize_io)
+        .set_quantize_io(quantize_io)
         .dump_intermediate_artifacts_to(artifact_path)
         .set_input_order(reorder_inputs)
     )
     return compile_spec
 
 
 def get_u85_compile_spec_unbuilt(
-    quantize_io=False,
+    quantize_io=True,
     custom_path=None,
     reorder_inputs=None,
 ) -> list[CompileSpec]:
@@ -156,7 +154,7 @@ def get_u85_compile_spec_unbuilt(
             memory_mode="Shared_Sram",
             extra_flags="--output-format=raw",
         )
-        .set_quantize_io(is_option_enabled("quantize_io") or quantize_io)
+        .set_quantize_io(quantize_io)
         .dump_intermediate_artifacts_to(artifact_path)
         .set_input_order(reorder_inputs)
     )

backends/arm/test/conftest.py

@@ -1,4 +1,4 @@
-# Copyright 2024 Arm Limited and/or its affiliates.
+# Copyright 2024-2025 Arm Limited and/or its affiliates.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
@@ -7,7 +7,6 @@
 import os
 import platform
 import random
-import re
 import shutil
 import subprocess
 import sys
@@ -25,46 +24,34 @@
 
 
 def pytest_configure(config):
-    pytest._test_options = {}
 
-    if config.option.arm_quantize_io:
-        _load_libquantized_ops_aot_lib()
-        pytest._test_options["quantize_io"] = True
+    pytest._test_options = {}  # type: ignore[attr-defined]
+
     if config.option.arm_run_corstoneFVP:
         corstone300_exists = shutil.which("FVP_Corstone_SSE-300_Ethos-U55")
         corstone320_exists = shutil.which("FVP_Corstone_SSE-320")
         if not (corstone300_exists and corstone320_exists):
             raise RuntimeError(
                 "Tests are run with --arm_run_corstoneFVP but corstone FVP is not installed."
             )
-        pytest._test_options["corstone_fvp"] = True
-    pytest._test_options["fast_fvp"] = config.option.fast_fvp
+        pytest._test_options["corstone_fvp"] = True  # type: ignore[attr-defined]
+    pytest._test_options["fast_fvp"] = config.option.fast_fvp  # type: ignore[attr-defined]
     logging.basicConfig(level=logging.INFO, stream=sys.stdout)
 
 
 def pytest_collection_modifyitems(config, items):
-    """
-    Skip all tests that require run on Ethos-U if the option arm_quantize_io is
-    not set.
-    """
-    if not config.option.arm_quantize_io:
-        skip_if_aot_lib_not_loaded = pytest.mark.skip(
-            "Ethos-U tests can only run on FVP with quantize_io=True."
-        )
-
-        for item in items:
-            if re.search(r"u55|u65|u85", item.name, re.IGNORECASE):
-                item.add_marker(skip_if_aot_lib_not_loaded)
+    pass
 
 
 def pytest_addoption(parser):
-    parser.addoption("--arm_quantize_io", action="store_true")
+    parser.addoption("--arm_quantize_io", action="store_true", help="Deprecated.")
     parser.addoption("--arm_run_corstoneFVP", action="store_true")
     parser.addoption("--fast_fvp", action="store_true")
 
 
 def pytest_sessionstart(session):
-    pass
+    if not session.config.option.collectonly:
+        _load_libquantized_ops_aot_lib()
 
 
 def pytest_sessionfinish(session, exitstatus):
@@ -128,15 +115,12 @@ def is_option_enabled(option: str, fail_if_not_enabled: bool = False) -> bool:
     """
     Returns whether an option is successfully enabled, i.e. if the flag was
     given to pytest and the necessary requirements are available.
-    Implemented options are:
-        - corstone_fvp.
-        - quantize_io.
 
     The optional parameter 'fail_if_not_enabled' makes the function raise
       a RuntimeError instead of returning False.
     """
 
-    if option in pytest._test_options and pytest._test_options[option]:
+    if option in pytest._test_options and pytest._test_options[option]:  # type: ignore[attr-defined]
         return True
     else:
         if fail_if_not_enabled:
@@ -152,15 +136,14 @@ def get_option(option: str) -> Any | None:
     Args:
         option (str): The option to check for.
     """
-    if option in pytest._test_options:
-        return pytest._test_options[option]
+    if option in pytest._test_options:  # type: ignore[attr-defined]
+        return pytest._test_options[option]  # type: ignore[attr-defined]
     return None
 
 
 def _load_libquantized_ops_aot_lib():
     """
-    Load the libquantized_ops_aot_lib shared library. It's required when
-    arm_quantize_io is set.
+    Find and load the libquantized_ops_aot_lib shared library.
     """
     so_ext = {
         "Darwin": "dylib",
@@ -181,5 +164,5 @@ def _load_libquantized_ops_aot_lib():
         torch.ops.load_library(library_path)
     else:
         raise RuntimeError(
-            f"Failed to load libquantized_ops_aot_lib.{so_ext}. Did you build it?"
+            f"Did not find libquantized_ops_aot_lib.{so_ext} in cmake-out-aot-lib. Did you build it?"
         )

backends/arm/test/ops/test_depthwise_conv.py

@@ -260,7 +260,7 @@ def test_dw_conv_tosa_BI(self, test_name: str, model: torch.nn.Module):
     @parameterized.expand(testsuite_conv2d[:4], skip_on_empty=True)
     @pytest.mark.corstone_fvp
     def test_dw_conv2d_u55_BI(
-        self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
+        self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = True
     ):
         self._test_dw_conv_ethos_BI_pipeline(
             model,
@@ -270,7 +270,7 @@ def test_dw_conv2d_u55_BI(
 
     @parameterized.expand(testsuite_conv2d[4:], skip_on_empty=True)
     @pytest.mark.corstone_fvp
-    @unittest.expectedFailure  # TODO: MLETORCH-516
+    @conftest.expectedFailureOnFVP  # TODO: MLETORCH-516
     def test_dw_conv2d_u55_BI_xfails(
         self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
     ):
@@ -283,7 +283,7 @@ def test_dw_conv2d_u55_BI_xfails(
     @parameterized.expand(testsuite_conv1d, skip_on_empty=True)
     @pytest.mark.corstone_fvp
     def test_dw_conv1d_u55_BI(
-        self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
+        self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = True
     ):
         self._test_dw_conv_ethos_BI_pipeline(
             model,
@@ -294,7 +294,7 @@ def test_dw_conv1d_u55_BI(
     @parameterized.expand(testsuite_conv1d + testsuite_conv2d_u85)
     @pytest.mark.corstone_fvp
     def test_dw_conv_u85_BI(
-        self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
+        self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = True
     ):
         self._test_dw_conv_ethos_BI_pipeline(
             model,
@@ -307,7 +307,7 @@ def test_dw_conv_u85_BI(
     @pytest.mark.corstone_fvp
     @conftest.expectedFailureOnFVP
     def test_dw_conv_u85_BI_xfails(
-        self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
+        self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = True
     ):
         self._test_dw_conv_ethos_BI_pipeline(
             model,

backends/arm/test/ops/test_layer_norm.py

@@ -174,7 +174,7 @@ def test_layer_norm_u55_BI(
     # Skip tests that require transposes.
     @parameterized.expand(test_data_suite[:4])
     @pytest.mark.corstone_fvp
-    @unittest.expectedFailure
+    @conftest.expectedFailureOnFVP
     def test_layer_norm_u55_BI_xfails(
         self,
         test_name: str,
@@ -188,7 +188,7 @@ def test_layer_norm_u55_BI_xfails(
     # Numerical issues on FVP likely due to mul op, MLETORCH-521
     @parameterized.expand(test_data_suite[:-2])
     @pytest.mark.corstone_fvp
-    @unittest.expectedFailure
+    @conftest.expectedFailureOnFVP
     def test_layer_norm_u85_BI_xfails(
         self,
         test_name: str,

backends/arm/test/ops/test_maximum.py

@@ -1,5 +1,5 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
-# Copyright 2024 Arm Limited and/or its affiliates.
+# Copyright 2024-2025 Arm Limited and/or its affiliates.
 # All rights reserved.
 #
 # This source code is licensed under the BSD-style license found in the
@@ -10,7 +10,7 @@
 from typing import Tuple
 
 import torch
-from executorch.backends.arm.test import common
+from executorch.backends.arm.test import common, conftest
 from executorch.backends.arm.test.tester.arm_tester import ArmTester
 from executorch.exir import EdgeCompileConfig
 from executorch.exir.backend.compile_spec_schema import CompileSpec
@@ -120,7 +120,7 @@ def test_maximum_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         tester = self._test_maximum_ethos_BI_pipeline(
             self.Maximum(), common.get_u55_compile_spec(), test_data
         )
-        if common.is_option_enabled("corstone_fvp"):
+        if conftest.is_option_enabled("corstone_fvp"):
             tester.run_method_and_compare_outputs(
                 qtol=1, inputs=test_data, target_board="corstone-300"
             )
@@ -131,7 +131,7 @@ def test_maximum_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         tester = self._test_maximum_ethos_BI_pipeline(
             self.Maximum(), common.get_u85_compile_spec(), test_data
         )
-        if common.is_option_enabled("corstone_fvp"):
+        if conftest.is_option_enabled("corstone_fvp"):
             tester.run_method_and_compare_outputs(
                 qtol=1, inputs=test_data, target_board="corstone-320"
             )

backends/arm/test/ops/test_minimum.py

@@ -1,5 +1,5 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
-# Copyright 2024 Arm Limited and/or its affiliates.
+# Copyright 2024-2025 Arm Limited and/or its affiliates.
 # All rights reserved.
 #
 # This source code is licensed under the BSD-style license found in the
@@ -10,7 +10,7 @@
 from typing import Tuple
 
 import torch
-from executorch.backends.arm.test import common
+from executorch.backends.arm.test import common, conftest
 from executorch.backends.arm.test.tester.arm_tester import ArmTester
 from executorch.exir import EdgeCompileConfig
 from executorch.exir.backend.compile_spec_schema import CompileSpec
@@ -120,7 +120,7 @@ def test_minimum_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         tester = self._test_minimum_ethos_BI_pipeline(
             self.Minimum(), common.get_u55_compile_spec(), test_data
         )
-        if common.is_option_enabled("corstone_fvp"):
+        if conftest.is_option_enabled("corstone_fvp"):
             tester.run_method_and_compare_outputs(
                 qtol=1, inputs=test_data, target_board="corstone-300"
             )
@@ -131,7 +131,7 @@ def test_minimum_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         tester = self._test_minimum_ethos_BI_pipeline(
             self.Minimum(), common.get_u85_compile_spec(), test_data
         )
-        if common.is_option_enabled("corstone_fvp"):
+        if conftest.is_option_enabled("corstone_fvp"):
             tester.run_method_and_compare_outputs(
                 qtol=1, inputs=test_data, target_board="corstone-320"
             )

backends/arm/test/test_arm_baremetal.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2024 Arm Limited and/or its affiliates.
+# Copyright 2024-2025 Arm Limited and/or its affiliates.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
@@ -63,6 +63,7 @@ test_pytest() { # Test ops and other things
     echo "${TEST_SUITE_NAME}: Run pytest"
     cd "${et_root_dir}"
     source examples/arm/ethos-u-scratch/setup_path.sh
+    backends/arm/scripts/build_quantized_ops_aot_lib.sh
 
     # Run arm baremetal pytest tests without FVP
     pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/
@@ -78,7 +79,7 @@ test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using
     backends/arm/test/setup_testing.sh
 
     # Run arm baremetal pytest tests with FVP
-    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/ --arm_quantize_io --arm_run_corstoneFVP
+    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/ --arm_run_corstoneFVP
 }
 
 test_run_ethosu_fvp() { # End to End model tests