Arm backend: use tosa_ref_model only if installed

Not making these tests XFail because there is still value in running
these tests w/o validating output since we do go through a lot of other
checks in the AoT flow when generating a PTE.

Tosa ref model being not installed is not the common case anyway.

Added explicit warnings (which should show through pytest) as a reminder
for the comparison is being skipped.

Author: digantdesai

backends/arm/test/models/test_conformer.py

@@ -5,6 +5,7 @@
 
 from typing import Tuple
 
+import conftest
 import pytest
 
 import torch
@@ -56,6 +57,7 @@ def test_conformer_tosa_FP():
         TestConformer.model_example_inputs,
         aten_op=TestConformer.aten_ops,
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.run()
@@ -67,17 +69,24 @@ def test_conformer_tosa_INT():
         TestConformer.model_example_inputs,
         aten_op=TestConformer.aten_ops,
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.pop_stage("check_count.exir")
-    pipeline.change_args(
-        "run_method_and_compare_outputs",
-        get_test_inputs(
-            TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
-        ),
-        rtol=1.0,
-        atol=3.0,
-    )
+
+    try:
+        if pipeline.find_pos("run_method_and_compare_outputs") >=0:
+            pipeline.change_args(
+                "run_method_and_compare_outputs",
+                get_test_inputs(
+                    TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
+                ),
+                rtol=1.0,
+                atol=3.0,
+            )
+    except Exception as e:
+        # tosa_ref_model must not be available
+        assert not conftest.is_option_enabled("tosa_ref_model"), f"TOSA reference model should be disabled, but error occurred: {e}" 
     pipeline.run()
 
 

backends/arm/test/models/test_lstm_arm.py

@@ -11,6 +11,7 @@
 from executorch.backends.arm.test.tester.test_pipeline import (
     EthosU55PipelineINT,
     EthosU85PipelineINT,
+    is_tosa_ref_model_available,
     TosaPipelineFP,
     TosaPipelineINT,
     VgfPipeline,
@@ -51,7 +52,16 @@ def test_lstm_tosa_FP():
         exir_op=[],
         use_to_edge_transform_and_lower=True,
     )
-    pipeline.change_args("run_method_and_compare_outputs", get_test_inputs(), atol=3e-1)
+    try:
+        if pipeline.find_pos("run_method_and_compare_outputs") >= 0:
+            pipeline.change_args(
+                "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1
+            )
+    except Exception as e:
+        # tosa_ref_model must not be available
+        assert (
+            is_tosa_ref_model_available() == False
+        ), "Expected TOSA reference model to be disabled, but error occurred: {e}"
     pipeline.run()
 
 
@@ -63,9 +73,16 @@ def test_lstm_tosa_INT():
         exir_op=[],
         use_to_edge_transform_and_lower=True,
     )
-    pipeline.change_args(
-        "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0
-    )
+    try:
+        if pipeline.find_pos("run_method_and_compare_outputs") >= 0:
+            pipeline.change_args(
+                "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0
+            )
+    except Exception as e:
+        # tosa_ref_model must not be available
+        assert (
+            is_tosa_ref_model_available() == False
+        ), "Expected TOSA reference model to be disabled, but error occurred: {e}"
     pipeline.run()
 
 

backends/arm/test/tester/test_pipeline.py

@@ -4,6 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 
 import logging
+import warnings as _warnings
 
 from typing import (
     Any,
@@ -41,6 +42,20 @@
 """ Generic type used for test data in the pipeline. Depends on which type the operator expects."""
 
 
+def is_tosa_ref_model_available():
+    """Checks if the TOSA reference model is available."""
+    # Not all deployments of ET have the TOSA reference model available.
+    # Make sure we don't try to use it if it's not available.
+    try:
+        import tosa_tools.tosa_ref_model as tosa_reference_model
+
+        if not dir(tosa_reference_model):
+            return False
+    except ImportError:
+        return False
+    return True
+
+
 class BasePipelineMaker(Generic[T]):
     """
     The BasePiplineMaker defines a list of stages to be applied to a torch.nn.module for lowering it
@@ -283,8 +298,6 @@ class TosaPipelineINT(BasePipelineMaker, Generic[T]):
        exir_ops: Exir dialect ops expected to be found in the graph after to_edge.
        if not using use_edge_to_transform_and_lower.
 
-       run_on_tosa_ref_model: Set to true to test the tosa file on the TOSA reference model.
-
        tosa_version: A string for identifying the TOSA version, see common.get_tosa_compile_spec for
                      options.
        use_edge_to_transform_and_lower: Selects betweeen two possible ways of lowering the module.
@@ -297,7 +310,6 @@ def __init__(
         test_data: T,
         aten_op: str | List[str],
         exir_op: Optional[str | List[str]] = None,
-        run_on_tosa_ref_model: bool = True,
         symmetric_io_quantization: bool = False,
         per_channel_quantization: bool = True,
         use_to_edge_transform_and_lower: bool = True,
@@ -360,14 +372,18 @@ def __init__(
             suffix="quant_nodes",
         )
 
-        if run_on_tosa_ref_model:
+        if is_tosa_ref_model_available():
             self.add_stage(
                 self.tester.run_method_and_compare_outputs,
                 atol=atol,
                 rtol=rtol,
                 qtol=qtol,
                 inputs=self.test_data,
             )
+        else:
+            _warnings.warn(
+                "Warning: Skipping run_method_and_compare_outputs stage. tosa reference model is not available."
+            )
 
 
 class TosaPipelineFP(BasePipelineMaker, Generic[T]):
@@ -382,8 +398,6 @@ class TosaPipelineFP(BasePipelineMaker, Generic[T]):
        exir_ops: Exir dialect ops expected to be found in the graph after to_edge.
        if not using use_edge_to_transform_and_lower.
 
-       run_on_tosa_ref_model: Set to true to test the tosa file on the TOSA reference model.
-
        tosa_version: A string for identifying the TOSA version, see common.get_tosa_compile_spec for
                      options.
        use_edge_to_transform_and_lower: Selects betweeen two possible ways of lowering the module.
@@ -435,14 +449,18 @@ def __init__(
             suffix="quant_nodes",
         )
 
-        if run_on_tosa_ref_model:
+        if is_tosa_ref_model_available():
             self.add_stage(
                 self.tester.run_method_and_compare_outputs,
                 atol=atol,
                 rtol=rtol,
                 qtol=qtol,
                 inputs=self.test_data,
             )
+        else:
+            _warnings.warn(
+                "Warning: Skipping run_method_and_compare_outputs stage. tosa reference model is not available"
+            )
 
 
 class EthosU55PipelineINT(BasePipelineMaker, Generic[T]):
@@ -701,7 +719,12 @@ def __init__(
             self.add_stage(self.tester.check_count, ops_after_pass, suffix="after")
         if ops_not_after_pass:
             self.add_stage(self.tester.check_not, ops_not_after_pass, suffix="after")
-        self.add_stage(self.tester.run_method_and_compare_outputs)
+        if is_tosa_ref_model_available():
+            self.add_stage(self.tester.run_method_and_compare_outputs)
+        else:
+            _warnings.warn(
+                "Warning: Skipping run_method_and_compare_outputs stage. Tosa reference model is not available."
+            )
 
 
 class TransformAnnotationPassPipeline(BasePipelineMaker, Generic[T]):
@@ -748,11 +771,16 @@ def __init__(
         self.pop_stage("to_executorch")
         self.pop_stage("to_edge_transform_and_lower")
         self.pop_stage("check.aten")
-        self.add_stage(
-            self.tester.run_method_and_compare_outputs,
-            inputs=test_data,
-            run_eager_mode=True,
-        )
+        if is_tosa_ref_model_available():
+            self.add_stage(
+                self.tester.run_method_and_compare_outputs,
+                inputs=test_data,
+                run_eager_mode=True,
+            )
+        else:
+            _warnings.warn(
+                "Warning: Skipping run_method_and_compare_outputs stage. Tosa reference model is not available."
+            )
 
 
 class OpNotSupportedPipeline(BasePipelineMaker, Generic[T]):