Arm backend: run on tosa_ref_model only when requested (pytorch#13165)

Summary:
Pull Request resolved: pytorch#13165

This is needed to support running the tests on setups where we may not have tosa reference model installed.

The downside is, it may pass without doing e2e validaiton, which shouldn't happen on the CI where this is installed.

Differential Revision: D79741689

Author: digantdesai

backends/arm/test/misc/test_bn_relu_folding_qat.py

@@ -47,7 +47,14 @@ def forward(self, x: torch.Tensor):
 
 @common.parametrize("model", models)
 def test_qat_tosa_INT(model: torch.nn.Module):
-    pipeline = TosaPipelineINT[input_t1](model, model.test_data, [], [], qtol=1)
+    pipeline = TosaPipelineINT[input_t1](
+        model, 
+        model.test_data, 
+        [], 
+        [], 
+        qtol=1,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     tosa_version = conftest.get_option("tosa_version")
     tosa_profiles = {
         "1.0": common.TosaSpecification.create_from_string("TOSA-1.0+INT"),

backends/arm/test/misc/test_custom_partition.py

@@ -3,6 +3,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import conftest
 import logging
 from typing import Tuple
 
@@ -50,7 +51,13 @@ def test_single_reject(caplog, test_data: input_t1):
     caplog.set_level(logging.INFO)
 
     module = CustomPartitioning()
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     check = DontPartition(exir_ops.edge.aten.sigmoid.default)
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
@@ -68,7 +75,13 @@ def test_single_reject(caplog, test_data: input_t1):
 @common.parametrize("test_data", CustomPartitioning.inputs)
 def test_multiple_reject(test_data: input_t1):
     module = CustomPartitioning()
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     check = DontPartition(
         exir_ops.edge.aten.sigmoid.default, exir_ops.edge.aten.mul.Tensor
     )
@@ -90,7 +103,13 @@ def test_torch_op_reject(caplog, test_data: input_t1):
 
     module = CustomPartitioning()
     check = DontPartition(torch.ops.aten.sigmoid.default)
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir", {"torch.ops.higher_order.executorch_call_delegate": 2}
@@ -108,7 +127,13 @@ def test_torch_op_reject(caplog, test_data: input_t1):
 def test_string_op_reject(test_data: input_t1):
     module = CustomPartitioning()
     check = DontPartition("aten.sigmoid.default")
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir", {"torch.ops.higher_order.executorch_call_delegate": 2}
@@ -127,7 +152,13 @@ def test_name_reject(caplog, test_data: input_t1):
 
     module = CustomPartitioning()
     check = DontPartitionName("mul", "sigmoid", exact=False)
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir",
@@ -142,7 +173,13 @@ def test_name_reject(caplog, test_data: input_t1):
 def test_module_reject(test_data: input_t1):
     module = NestedModule()
     check = DontPartitionModule(module_name="CustomPartitioning")
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir",
@@ -158,7 +195,13 @@ def test_inexact_module_reject(caplog, test_data: input_t1):
 
     module = NestedModule()
     check = DontPartitionModule(module_name="Custom", exact=False)
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir",
@@ -173,7 +216,13 @@ def test_inexact_module_reject(caplog, test_data: input_t1):
 def test_module_instance_reject(test_data: input_t1):
     module = NestedModule()
     check = DontPartitionModule(instance_name="nested")
-    pipeline = TosaPipelineFP[input_t1](module, test_data, [], exir_op=[])
+    pipeline = TosaPipelineFP[input_t1](
+        module, 
+        test_data, 
+        [], 
+        exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.change_args("to_edge_transform_and_lower", additional_checks=[check])
     pipeline.change_args(
         "check_count.exir",

backends/arm/test/models/test_conformer.py

@@ -5,6 +5,7 @@
 
 from typing import Tuple
 
+import conftest
 import pytest
 
 import torch
@@ -56,6 +57,7 @@ def test_conformer_tosa_FP():
         TestConformer.model_example_inputs,
         aten_op=TestConformer.aten_ops,
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.run()
@@ -67,6 +69,7 @@ def test_conformer_tosa_INT():
         TestConformer.model_example_inputs,
         aten_op=TestConformer.aten_ops,
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.pop_stage("check_count.exir")

backends/arm/test/models/test_deit_tiny_arm.py

@@ -7,6 +7,7 @@
 
 from typing import Tuple
 
+import conftest
 import timm
 
 import torch
@@ -43,6 +44,7 @@ def test_deit_tiny_tosa_FP():
         model_inputs,
         aten_op=[],
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.run()
@@ -54,6 +56,7 @@ def test_deit_tiny_tosa_INT():
         model_inputs,
         aten_op=[],
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
         atol=1.5,
         qtol=1,

backends/arm/test/models/test_llama.py

@@ -111,6 +111,7 @@ def test_llama_tosa_FP():
             llama_inputs,
             aten_op=[],
             exir_op=[],
+            run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
             use_to_edge_transform_and_lower=True,
             transform_passes=[InsertCastForOpsWithInt64InputPass()],
         )
@@ -129,6 +130,7 @@ def test_llama_tosa_INT():
             llama_inputs,
             aten_op=[],
             exir_op=[],
+            run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
             use_to_edge_transform_and_lower=True,
         )
         pipeline.run()

backends/arm/test/models/test_lstm_arm.py

@@ -5,6 +5,7 @@
 
 from typing import Tuple
 
+import conftest
 import torch
 
 from executorch.backends.arm.test import common
@@ -49,6 +50,7 @@ def test_lstm_tosa_FP():
         TestLSTM.model_example_inputs,
         aten_op=[],
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.change_args("run_method_and_compare_outputs", get_test_inputs(), atol=3e-1)
@@ -61,6 +63,7 @@ def test_lstm_tosa_INT():
         TestLSTM.model_example_inputs,
         aten_op=[],
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.change_args(

backends/arm/test/models/test_w2l_arm.py

@@ -8,6 +8,7 @@
 import unittest
 from typing import Tuple
 
+import conftest
 import pytest
 
 import torch
@@ -53,6 +54,7 @@ def test_w2l_tosa_FP():
         TestW2L.model_example_inputs,
         aten_op=[],
         exir_op=TestW2L.all_operators,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.run()
@@ -66,6 +68,7 @@ def test_w2l_tosa_INT():
         TestW2L.model_example_inputs,
         aten_op=[],
         exir_op=TestW2L.all_operators,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
         use_to_edge_transform_and_lower=True,
     )
     pipeline.run()

backends/arm/test/ops/test_abs.py

@@ -8,6 +8,7 @@
 
 from typing import Tuple
 
+import conftest
 import torch
 from executorch.backends.arm.test import common
 from executorch.backends.arm.test.tester.test_pipeline import (
@@ -41,13 +42,25 @@ def forward(self, x):
 
 @common.parametrize("test_data", Abs.test_parameters)
 def test_abs_tosa_FP(test_data: torch.Tensor):
-    pipeline = TosaPipelineFP[input_t1](Abs(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineFP[input_t1](
+        Abs(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
 @common.parametrize("test_data", Abs.test_parameters)
 def test_abs_tosa_INT(test_data: torch.Tensor):
-    pipeline = TosaPipelineINT[input_t1](Abs(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineINT[input_t1](
+        Abs(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 

backends/arm/test/ops/test_acosh.py

@@ -4,6 +4,7 @@
 # LICENSE file in the root directory of this source tree.
 from typing import Tuple
 
+import conftest
 import pytest
 
 import torch
@@ -55,6 +56,7 @@ def test_acosh_tosa_FP(test_data: Tuple):
         (test_data(),),
         aten_op,
         exir_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
     )
     pipeline.run()
 
@@ -65,6 +67,7 @@ def test_acosh_tosa_INT(test_data: Tuple):
         Acosh(),
         (test_data(),),
         aten_op=[],
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
     )
     pipeline.run()
 

backends/arm/test/ops/test_add.py

@@ -81,13 +81,25 @@ def forward(self, x: torch.Tensor, y: torch.Tensor):
 
 @common.parametrize("test_data", Add.test_data)
 def test_add_tensor_tosa_FP(test_data: input_t1):
-    pipeline = TosaPipelineFP[input_t1](Add(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineFP[input_t1](
+        Add(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
 @common.parametrize("test_data", Add.test_data)
 def test_add_tensor_tosa_INT(test_data: input_t1):
-    pipeline = TosaPipelineINT[input_t1](Add(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineINT[input_t1](
+        Add(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
@@ -146,25 +158,49 @@ def test_add_tensor_u85_INT(test_data: input_t1):
 
 @common.parametrize("test_data", Add2.test_data)
 def test_add_tensor_tosa_FP_2(test_data: input_t2):
-    pipeline = TosaPipelineFP[input_t2](Add2(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineFP[input_t2](
+        Add2(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
 @common.parametrize("test_data", Add3.test_data)
 def test_add_tensor_tosa_FP_3(test_data: input_t2):
-    pipeline = TosaPipelineFP[input_t2](Add3(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineFP[input_t2](
+        Add3(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
 @common.parametrize("test_data", Add3.test_data)
 def test_add_tensor_tosa_INT_3(test_data: input_t2):
-    pipeline = TosaPipelineINT[input_t2](Add3(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineINT[input_t2](
+        Add3(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()
 
 
 @common.parametrize("test_data", Add2.test_data)
 def test_add_tensor_tosa_INT_2(test_data: input_t2):
-    pipeline = TosaPipelineINT[input_t2](Add2(), test_data(), aten_op, exir_op)
+    pipeline = TosaPipelineINT[input_t2](
+        Add2(), 
+        test_data(), 
+        aten_op, 
+        exir_op,
+        run_on_tosa_ref_model=conftest.is_option_enabled("tosa_ref_model"),
+    )
     pipeline.run()