Arm backend: Add marks for test

Signed-off-by: Zingo Andersen <[email protected]>
Change-Id: I045b04a89b95791be29f34370cff67d99aad0c0e

Author: zingo

backends/arm/test/models/test_mobilenet_v2_arm.py

@@ -8,6 +8,8 @@
 import logging
 import unittest
 
+import pytest
+
 import torch
 from executorch.backends.arm.test import common, conftest
 
@@ -83,6 +85,8 @@ def test_mv2_tosa_BI(self):
             .run_method_and_compare_outputs(atol=1.0, qtol=1, inputs=self.model_inputs)
         )
 
+    @pytest.mark.slow
+    @pytest.mark.corstone_fvp
     def test_mv2_u55_BI(self):
         tester = (
             ArmTester(
@@ -101,6 +105,8 @@ def test_mv2_u55_BI(self):
                 atol=1.0, qtol=1, inputs=self.model_inputs, target_board="corstone-300"
             )
 
+    @pytest.mark.slow
+    @pytest.mark.corstone_fvp
     def test_mv2_u85_BI(self):
         tester = (
             ArmTester(

backends/arm/test/ops/test_add.py

@@ -9,6 +9,7 @@
 
 from typing import Tuple
 
+import pytest
 import torch
 from executorch.backends.arm.test import common, conftest
 from executorch.backends.arm.test.tester.arm_tester import ArmTester
@@ -131,6 +132,7 @@ def test_add_tosa_BI(self, test_data: torch.Tensor):
         self._test_add_tosa_BI_pipeline(self.Add(), test_data)
 
     @parameterized.expand(Add.test_parameters)
+    @pytest.mark.corstone_fvp
     def test_add_u55_BI(self, test_data: torch.Tensor):
         test_data = (test_data,)
         self._test_add_ethos_BI_pipeline(
@@ -140,6 +142,7 @@ def test_add_u55_BI(self, test_data: torch.Tensor):
         )
 
     @parameterized.expand(Add.test_parameters)
+    @pytest.mark.corstone_fvp
     def test_add_u85_BI(self, test_data: torch.Tensor):
         test_data = (test_data,)
         self._test_add_ethos_BI_pipeline(
@@ -159,13 +162,15 @@ def test_add2_tosa_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         self._test_add_tosa_BI_pipeline(self.Add2(), test_data)
 
     @parameterized.expand(Add2.test_parameters)
+    @pytest.mark.corstone_fvp
     def test_add2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         test_data = (operand1, operand2)
         self._test_add_ethos_BI_pipeline(
             self.Add2(), common.get_u55_compile_spec(), test_data
         )
 
     @parameterized.expand(Add2.test_parameters)
+    @pytest.mark.corstone_fvp
     def test_add2_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         test_data = (operand1, operand2)
         self._test_add_ethos_BI_pipeline(

backends/arm/test/ops/test_avg_pool.py

@@ -9,6 +9,8 @@
 
 from typing import Tuple
 
+import pytest
+
 import torch
 from executorch.backends.arm.quantizer.arm_quantizer import (
     ArmQuantizer,
@@ -144,6 +146,7 @@ def test_avgpool2d_tosa_BI(
         )
 
     @parameterized.expand(test_data_suite)
+    @pytest.mark.corstone_fvp
     def test_avgpool2d_tosa_u55_BI(
         self,
         test_name: str,
@@ -157,6 +160,7 @@ def test_avgpool2d_tosa_u55_BI(
         )
 
     @parameterized.expand(test_data_suite)
+    @pytest.mark.corstone_fvp
     def test_avgpool2d_tosa_u85_BI(
         self,
         test_name: str,

backends/arm/test/ops/test_bmm.py

@@ -8,6 +8,8 @@
 
 from typing import Tuple
 
+import pytest
+
 import torch
 from executorch.backends.arm.test import common, conftest
 from executorch.backends.arm.test.tester.arm_tester import ArmTester
@@ -146,6 +148,7 @@ def test_bmm_single_input_tosa_BI(self, operand1: torch.Tensor):
         self._test_bmm_tosa_BI_pipeline(self.BMMSingleInput(), test_data)
 
     @parameterized.expand(BMM.test_parameters)
+    @pytest.mark.corstone_fvp
     @unittest.expectedFailure
     def test_bmm_u55_BI_xfails(self, operand1: torch.Tensor, operand2: torch.Tensor):
         test_data = (operand1, operand2)
@@ -154,13 +157,15 @@ def test_bmm_u55_BI_xfails(self, operand1: torch.Tensor, operand2: torch.Tensor)
         )
 
     @parameterized.expand(BMM.test_parameters[:1])
+    @pytest.mark.corstone_fvp
     def test_bmm_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         test_data = (operand1, operand2)
         self._test_bmm_ethosu_BI_pipeline(
             self.BMM(), common.get_u85_compile_spec(), test_data
         )
 
     @parameterized.expand(BMM.test_parameters[1:])
+    @pytest.mark.corstone_fvp
     @conftest.expectedFailureOnFVP
     def test_bmm_u85_BI_xfails(self, operand1: torch.Tensor, operand2: torch.Tensor):
         test_data = (operand1, operand2)
@@ -170,6 +175,7 @@ def test_bmm_u85_BI_xfails(self, operand1: torch.Tensor, operand2: torch.Tensor)
 
     # Expected to fail with error: Warning, unsupported fusing of TOSA Rescale previous operator is of type: Memcpy
     @parameterized.expand(BMMSingleInput.test_parameters)
+    @pytest.mark.corstone_fvp
     @unittest.expectedFailure
     def test_bmm_single_input_u55_BI_xfails(self, operand1: torch.Tensor):
         test_data = (operand1,)
@@ -178,6 +184,7 @@ def test_bmm_single_input_u55_BI_xfails(self, operand1: torch.Tensor):
         )
 
     @parameterized.expand(BMMSingleInput.test_parameters)
+    @pytest.mark.corstone_fvp
     def test_bmm_single_input_u85_BI(self, operand1: torch.Tensor):
         test_data = (operand1,)
         self._test_bmm_ethosu_BI_pipeline(

backends/arm/test/ops/test_cat.py

@@ -9,6 +9,8 @@
 
 from typing import Tuple
 
+import pytest
+
 import torch
 from executorch.backends.arm.test import common, conftest
 from executorch.backends.arm.test.tester.arm_tester import ArmTester
@@ -134,6 +136,7 @@ def test_cat_tosa_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
 
     # Mismatch in provided number of inputs and model signature, MLETORCH 519
     @parameterized.expand(Cat.test_parameters)
+    @pytest.mark.corstone_fvp
     @conftest.expectedFailureOnFVP
     def test_cat_u55_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
         test_data = (operands, dim)
@@ -143,6 +146,7 @@ def test_cat_u55_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
 
     # Mismatch in provided number of inputs and model signature, MLETORCH 519
     @parameterized.expand(Cat.test_parameters)
+    @pytest.mark.corstone_fvp
     @conftest.expectedFailureOnFVP
     def test_cat_u85_BI(self, operands: tuple[torch.Tensor, ...], dim: int):
         test_data = (operands, dim)

backends/arm/test/ops/test_clone.py

@@ -11,6 +11,8 @@
 import unittest
 from typing import Tuple
 
+import pytest
+
 import torch
 
 from executorch.backends.arm.quantizer.arm_quantizer import (
@@ -122,9 +124,11 @@ def test_clone_tosa_BI(self, test_tensor: torch.Tensor):
         self._test_clone_tosa_BI_pipeline(self.Clone(), (test_tensor,))
 
     @parameterized.expand(Clone.test_parameters)
+    @pytest.mark.corstone_fvp
     def test_clone_u55_BI(self, test_tensor: torch.Tensor):
         self._test_clone_tosa_u55_pipeline(self.Clone(), (test_tensor,))
 
     @parameterized.expand(Clone.test_parameters)
+    @pytest.mark.corstone_fvp
     def test_clone_u85_BI(self, test_tensor: torch.Tensor):
         self._test_clone_tosa_u85_pipeline(self.Clone(), (test_tensor,))

backends/arm/test/ops/test_conv1d.py

@@ -8,6 +8,8 @@
 
 from typing import List, Optional, Tuple, Union
 
+import pytest
+
 import torch
 from executorch.backends.arm.test import common, conftest
 from executorch.backends.arm.test.tester.arm_tester import ArmTester
@@ -291,6 +293,7 @@ def test_conv1d_tosa_BI(self, test_name, model):
 
     # Expeted to fail as Conv1D requires transpoes which isn't supported on u55
     @parameterized.expand(testsuite)
+    @pytest.mark.corstone_fvp
     @unittest.expectedFailure
     def test_conv1d_u55_BI(self, test_name, model):
         self._test_conv1d_ethosu_BI_pipeline(
@@ -301,6 +304,7 @@ def test_conv1d_u55_BI(self, test_name, model):
     testsuite.remove(("5_3x2x128_st1", conv1d_5_3x2x128_st1))
 
     @parameterized.expand(testsuite)
+    @pytest.mark.corstone_fvp
     def test_conv1d_u85_BI(self, test_name, model):
         self._test_conv1d_ethosu_BI_pipeline(
             model, common.get_u85_compile_spec(), model.get_inputs()

backends/arm/test/ops/test_conv2d.py

@@ -8,6 +8,8 @@
 
 from typing import List, Optional, Tuple, Union
 
+import pytest
+
 import torch
 from executorch.backends.arm.test import common, conftest
 from executorch.backends.arm.test.tester.arm_tester import ArmTester
@@ -325,6 +327,7 @@ def test_conv2d_tosa_BI(self, test_name, model):
     testsuite.remove(("5x5_3x2x128x128_st1", conv2d_5x5_3x2x128x128_st1))
 
     @parameterized.expand(testsuite)
+    @pytest.mark.corstone_fvp
     def test_conv2d_u55_BI(self, test_name, model):
         self._test_conv2d_ethosu_BI_pipeline(
             common.get_u55_compile_spec(permute_memory_to_nhwc=True),
@@ -333,6 +336,7 @@ def test_conv2d_u55_BI(self, test_name, model):
         )
 
     @parameterized.expand(testsuite)
+    @pytest.mark.corstone_fvp
     def test_conv2d_u85_BI(self, test_name, model):
         self._test_conv2d_ethosu_BI_pipeline(
             common.get_u85_compile_spec(permute_memory_to_nhwc=True),

backends/arm/test/ops/test_conv_combos.py

@@ -267,6 +267,7 @@ def test_conv_meandim_tosa_BI(self):
         model = ComboConv2dMeandim()
         self._test_conv_combo_tosa_BI_pipeline(model, model.get_inputs())
 
+    @pytest.mark.corstone_fvp
     def test_conv_meandim_u55_BI(self):
         model = ComboConv2dMeandim()
         self._test_conv_combo_ethos_BI_pipeline(
@@ -275,6 +276,7 @@ def test_conv_meandim_u55_BI(self):
             model.get_inputs(),
         )
 
+    @pytest.mark.corstone_fvp
     def test_conv_meandim_u85_BI(self):
         model = ComboConv2dMeandim()
         self._test_conv_combo_ethos_BI_pipeline(
@@ -294,12 +296,14 @@ def test_conv_batchnorm_relu6_tosa_BI(self):
         model = ComboConvBatchnormRelu6()
         self._test_conv_combo_tosa_BI_pipeline(model, model.get_inputs())
 
+    @pytest.mark.corstone_fvp
     def test_conv_batchnorm_relu6_u55_BI(self):
         model = ComboConvBatchnormRelu6()
         self._test_conv_combo_ethos_BI_pipeline(
             model, common.get_u55_compile_spec(), model.get_inputs()
         )
 
+    @pytest.mark.corstone_fvp
     def test_conv_batchnorm_relu_u85_BI(self):
         model = ComboConvBatchnormRelu6()
         self._test_conv_combo_ethos_BI_pipeline(
@@ -324,6 +328,7 @@ def test_conv_relu6_tosa_BI(self, test_data: torch.Tensor):
         self._test_conv_combo_tosa_BI_pipeline(model, test_data)
 
     @parameterized.expand(ComboConvRelu6.test_data)
+    @pytest.mark.corstone_fvp
     def test_conv_relu6_u55_BI(self, test_data: torch.Tensor):
         model = ComboConvRelu6()
         test_data = (test_data,)
@@ -332,6 +337,7 @@ def test_conv_relu6_u55_BI(self, test_data: torch.Tensor):
         )
 
     @parameterized.expand(ComboConvRelu6.test_data)
+    @pytest.mark.corstone_fvp
     def test_conv_relu6_u85_BI(self, test_data: torch.Tensor):
         model = ComboConvRelu6()
         test_data = (test_data,)
@@ -347,11 +353,12 @@ def test_block_bottleneck_residual_tosa_MI(self):
         self._test_conv_combo_tosa_MI_pipeline(model, model.get_inputs())
 
     # TODO: Investigate flakyness (MLTORCH-307)
-    @pytest.mark.flaky(reruns=3)
+    @unittest.skip(reason="Skiped due to flakyness (MLTORCH-307)")
     def test_block_bottleneck_residual_tosa_BI(self):
         model = ComboBlockBottleneckResidual()
         self._test_conv_combo_tosa_BI_pipeline(model, model.get_inputs())
 
+    @pytest.mark.corstone_fvp
     def test_block_bottleneck_residual_u55_BI(self):
         model = ComboBlockBottleneckResidual()
         self._test_conv_combo_ethos_BI_pipeline(
@@ -360,6 +367,7 @@ def test_block_bottleneck_residual_u55_BI(self):
             model.get_inputs(),
         )
 
+    @pytest.mark.corstone_fvp
     def test_block_bottleneck_residual_u85_BI(self):
         model = ComboBlockBottleneckResidual()
         self._test_conv_combo_ethos_BI_pipeline(
@@ -384,6 +392,7 @@ def test_conv_avgpool2d_tosa_BI(self, test_data: torch.Tensor):
         self._test_conv_combo_tosa_BI_pipeline(model, test_data)
 
     @parameterized.expand(ComboConvAvgPool2d.test_data)
+    @pytest.mark.corstone_fvp
     def test_conv_avgpool2d_u55_BI(self, test_data: torch.Tensor):
         model = ComboConvAvgPool2d()
         test_data = (test_data,)
@@ -394,6 +403,7 @@ def test_conv_avgpool2d_u55_BI(self, test_data: torch.Tensor):
         )
 
     @parameterized.expand(ComboConvAvgPool2d.test_data)
+    @pytest.mark.corstone_fvp
     def test_conv_avgpool2d_u85_BI(self, test_data: torch.Tensor):
         model = ComboConvAvgPool2d()
         test_data = (test_data,)

backends/arm/test/ops/test_depthwise_conv.py

@@ -8,6 +8,8 @@
 
 from typing import Tuple
 
+import pytest
+
 import torch
 from executorch.backends.arm.test import common, conftest
 from executorch.backends.arm.test.ops.test_conv1d import Conv1d
@@ -260,6 +262,7 @@ def test_dw_conv_tosa_BI(self, test_name: str, model: torch.nn.Module):
     )  # Works
 
     @parameterized.expand(testsuite_conv2d, skip_on_empty=True)
+    @pytest.mark.corstone_fvp
     @unittest.expectedFailure
     def test_dw_conv2d_u55_BI(
         self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
@@ -275,6 +278,7 @@ def test_dw_conv2d_u55_BI(
     # Expected to fail as conv1d needs transpose which is not supported
     # on u55.
     @parameterized.expand(testsuite_conv1d, skip_on_empty=True)
+    @pytest.mark.corstone_fvp
     @unittest.expectedFailure
     def test_dw_conv1d_u55_BI(
         self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
@@ -288,6 +292,7 @@ def test_dw_conv1d_u55_BI(
         )
 
     @parameterized.expand(testsuite_conv1d + testsuite_conv2d_u85)
+    @pytest.mark.corstone_fvp
     def test_dw_conv_u85_BI(
         self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False
     ):
@@ -301,6 +306,7 @@ def test_dw_conv_u85_BI(
 
     # All test cases except 3x3_1x3x256x256_gp3_st1 have numerical issues on FVP. MLETORCH-520
     @parameterized.expand(testsuite_conv2d_u85_xfails)
+    @pytest.mark.corstone_fvp
     @conftest.expectedFailureOnFVP
     def test_dw_conv_u85_BI_xfails(
         self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False