diff --git a/backends/arm/test/models/test_mobilenet_v2_arm.py b/backends/arm/test/models/test_mobilenet_v2_arm.py index 6fbea63fb00..fca743a6fa2 100644 --- a/backends/arm/test/models/test_mobilenet_v2_arm.py +++ b/backends/arm/test/models/test_mobilenet_v2_arm.py @@ -8,6 +8,8 @@ import logging import unittest +import pytest + import torch from executorch.backends.arm.test import common, conftest @@ -83,6 +85,8 @@ def test_mv2_tosa_BI(self): .run_method_and_compare_outputs(atol=1.0, qtol=1, inputs=self.model_inputs) ) + @pytest.mark.slow + @pytest.mark.corstone_fvp def test_mv2_u55_BI(self): tester = ( ArmTester( @@ -101,6 +105,8 @@ def test_mv2_u55_BI(self): atol=1.0, qtol=1, inputs=self.model_inputs, target_board="corstone-300" ) + @pytest.mark.slow + @pytest.mark.corstone_fvp def test_mv2_u85_BI(self): tester = ( ArmTester( diff --git a/backends/arm/test/ops/test_add.py b/backends/arm/test/ops/test_add.py index 8d18c7ab539..24faace0070 100644 --- a/backends/arm/test/ops/test_add.py +++ b/backends/arm/test/ops/test_add.py @@ -9,6 +9,7 @@ from typing import Tuple +import pytest import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -131,6 +132,7 @@ def test_add_tosa_BI(self, test_data: torch.Tensor): self._test_add_tosa_BI_pipeline(self.Add(), test_data) @parameterized.expand(Add.test_parameters) + @pytest.mark.corstone_fvp def test_add_u55_BI(self, test_data: torch.Tensor): test_data = (test_data,) self._test_add_ethos_BI_pipeline( @@ -140,6 +142,7 @@ def test_add_u55_BI(self, test_data: torch.Tensor): ) @parameterized.expand(Add.test_parameters) + @pytest.mark.corstone_fvp def test_add_u85_BI(self, test_data: torch.Tensor): test_data = (test_data,) self._test_add_ethos_BI_pipeline( @@ -159,6 +162,7 @@ def test_add2_tosa_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): self._test_add_tosa_BI_pipeline(self.Add2(), test_data) @parameterized.expand(Add2.test_parameters) + @pytest.mark.corstone_fvp def test_add2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) self._test_add_ethos_BI_pipeline( @@ -166,6 +170,7 @@ def test_add2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): ) @parameterized.expand(Add2.test_parameters) + @pytest.mark.corstone_fvp def test_add2_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) self._test_add_ethos_BI_pipeline( diff --git a/backends/arm/test/ops/test_avg_pool.py b/backends/arm/test/ops/test_avg_pool.py index c517b5ebe4c..27629701c32 100644 --- a/backends/arm/test/ops/test_avg_pool.py +++ b/backends/arm/test/ops/test_avg_pool.py @@ -9,6 +9,8 @@ from typing import Tuple +import pytest + import torch from executorch.backends.arm.quantizer.arm_quantizer import ( ArmQuantizer, @@ -144,6 +146,7 @@ def test_avgpool2d_tosa_BI( ) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_avgpool2d_tosa_u55_BI( self, test_name: str, @@ -157,6 +160,7 @@ def test_avgpool2d_tosa_u55_BI( ) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_avgpool2d_tosa_u85_BI( self, test_name: str, diff --git a/backends/arm/test/ops/test_bmm.py b/backends/arm/test/ops/test_bmm.py index e7a9a2a810e..3a3c2772bd5 100644 --- a/backends/arm/test/ops/test_bmm.py +++ b/backends/arm/test/ops/test_bmm.py @@ -8,6 +8,8 @@ from typing import Tuple +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -146,6 +148,7 @@ def test_bmm_single_input_tosa_BI(self, operand1: torch.Tensor): self._test_bmm_tosa_BI_pipeline(self.BMMSingleInput(), test_data) @parameterized.expand(BMM.test_parameters) + @pytest.mark.corstone_fvp @unittest.expectedFailure def test_bmm_u55_BI_xfails(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) @@ -154,6 +157,7 @@ def test_bmm_u55_BI_xfails(self, operand1: torch.Tensor, operand2: torch.Tensor) ) @parameterized.expand(BMM.test_parameters[:1]) + @pytest.mark.corstone_fvp def test_bmm_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) self._test_bmm_ethosu_BI_pipeline( @@ -161,6 +165,7 @@ def test_bmm_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): ) @parameterized.expand(BMM.test_parameters[1:]) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_bmm_u85_BI_xfails(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) @@ -170,6 +175,7 @@ def test_bmm_u85_BI_xfails(self, operand1: torch.Tensor, operand2: torch.Tensor) # Expected to fail with error: Warning, unsupported fusing of TOSA Rescale previous operator is of type: Memcpy @parameterized.expand(BMMSingleInput.test_parameters) + @pytest.mark.corstone_fvp @unittest.expectedFailure def test_bmm_single_input_u55_BI_xfails(self, operand1: torch.Tensor): test_data = (operand1,) @@ -178,6 +184,7 @@ def test_bmm_single_input_u55_BI_xfails(self, operand1: torch.Tensor): ) @parameterized.expand(BMMSingleInput.test_parameters) + @pytest.mark.corstone_fvp def test_bmm_single_input_u85_BI(self, operand1: torch.Tensor): test_data = (operand1,) self._test_bmm_ethosu_BI_pipeline( diff --git a/backends/arm/test/ops/test_cat.py b/backends/arm/test/ops/test_cat.py index 674363e991a..115b4402f5b 100644 --- a/backends/arm/test/ops/test_cat.py +++ b/backends/arm/test/ops/test_cat.py @@ -9,6 +9,8 @@ from typing import Tuple +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -134,6 +136,7 @@ def test_cat_tosa_BI(self, operands: tuple[torch.Tensor, ...], dim: int): # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(Cat.test_parameters) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_cat_u55_BI(self, operands: tuple[torch.Tensor, ...], dim: int): test_data = (operands, dim) @@ -143,6 +146,7 @@ def test_cat_u55_BI(self, operands: tuple[torch.Tensor, ...], dim: int): # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(Cat.test_parameters) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_cat_u85_BI(self, operands: tuple[torch.Tensor, ...], dim: int): test_data = (operands, dim) diff --git a/backends/arm/test/ops/test_clone.py b/backends/arm/test/ops/test_clone.py index 0238d62bb48..2ec2f621fa9 100644 --- a/backends/arm/test/ops/test_clone.py +++ b/backends/arm/test/ops/test_clone.py @@ -11,6 +11,8 @@ import unittest from typing import Tuple +import pytest + import torch from executorch.backends.arm.quantizer.arm_quantizer import ( @@ -122,9 +124,11 @@ def test_clone_tosa_BI(self, test_tensor: torch.Tensor): self._test_clone_tosa_BI_pipeline(self.Clone(), (test_tensor,)) @parameterized.expand(Clone.test_parameters) + @pytest.mark.corstone_fvp def test_clone_u55_BI(self, test_tensor: torch.Tensor): self._test_clone_tosa_u55_pipeline(self.Clone(), (test_tensor,)) @parameterized.expand(Clone.test_parameters) + @pytest.mark.corstone_fvp def test_clone_u85_BI(self, test_tensor: torch.Tensor): self._test_clone_tosa_u85_pipeline(self.Clone(), (test_tensor,)) diff --git a/backends/arm/test/ops/test_conv1d.py b/backends/arm/test/ops/test_conv1d.py index 9a25da8aa12..593260ac56f 100644 --- a/backends/arm/test/ops/test_conv1d.py +++ b/backends/arm/test/ops/test_conv1d.py @@ -8,6 +8,8 @@ from typing import List, Optional, Tuple, Union +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -291,6 +293,7 @@ def test_conv1d_tosa_BI(self, test_name, model): # Expeted to fail as Conv1D requires transpoes which isn't supported on u55 @parameterized.expand(testsuite) + @pytest.mark.corstone_fvp @unittest.expectedFailure def test_conv1d_u55_BI(self, test_name, model): self._test_conv1d_ethosu_BI_pipeline( @@ -301,6 +304,7 @@ def test_conv1d_u55_BI(self, test_name, model): testsuite.remove(("5_3x2x128_st1", conv1d_5_3x2x128_st1)) @parameterized.expand(testsuite) + @pytest.mark.corstone_fvp def test_conv1d_u85_BI(self, test_name, model): self._test_conv1d_ethosu_BI_pipeline( model, common.get_u85_compile_spec(), model.get_inputs() diff --git a/backends/arm/test/ops/test_conv2d.py b/backends/arm/test/ops/test_conv2d.py index 19e3067344f..9ccac539408 100644 --- a/backends/arm/test/ops/test_conv2d.py +++ b/backends/arm/test/ops/test_conv2d.py @@ -8,6 +8,8 @@ from typing import List, Optional, Tuple, Union +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -325,6 +327,7 @@ def test_conv2d_tosa_BI(self, test_name, model): testsuite.remove(("5x5_3x2x128x128_st1", conv2d_5x5_3x2x128x128_st1)) @parameterized.expand(testsuite) + @pytest.mark.corstone_fvp def test_conv2d_u55_BI(self, test_name, model): self._test_conv2d_ethosu_BI_pipeline( common.get_u55_compile_spec(permute_memory_to_nhwc=True), @@ -333,6 +336,7 @@ def test_conv2d_u55_BI(self, test_name, model): ) @parameterized.expand(testsuite) + @pytest.mark.corstone_fvp def test_conv2d_u85_BI(self, test_name, model): self._test_conv2d_ethosu_BI_pipeline( common.get_u85_compile_spec(permute_memory_to_nhwc=True), diff --git a/backends/arm/test/ops/test_conv_combos.py b/backends/arm/test/ops/test_conv_combos.py index aad16709948..17083b129f0 100644 --- a/backends/arm/test/ops/test_conv_combos.py +++ b/backends/arm/test/ops/test_conv_combos.py @@ -267,6 +267,7 @@ def test_conv_meandim_tosa_BI(self): model = ComboConv2dMeandim() self._test_conv_combo_tosa_BI_pipeline(model, model.get_inputs()) + @pytest.mark.corstone_fvp def test_conv_meandim_u55_BI(self): model = ComboConv2dMeandim() self._test_conv_combo_ethos_BI_pipeline( @@ -275,6 +276,7 @@ def test_conv_meandim_u55_BI(self): model.get_inputs(), ) + @pytest.mark.corstone_fvp def test_conv_meandim_u85_BI(self): model = ComboConv2dMeandim() self._test_conv_combo_ethos_BI_pipeline( @@ -294,12 +296,14 @@ def test_conv_batchnorm_relu6_tosa_BI(self): model = ComboConvBatchnormRelu6() self._test_conv_combo_tosa_BI_pipeline(model, model.get_inputs()) + @pytest.mark.corstone_fvp def test_conv_batchnorm_relu6_u55_BI(self): model = ComboConvBatchnormRelu6() self._test_conv_combo_ethos_BI_pipeline( model, common.get_u55_compile_spec(), model.get_inputs() ) + @pytest.mark.corstone_fvp def test_conv_batchnorm_relu_u85_BI(self): model = ComboConvBatchnormRelu6() self._test_conv_combo_ethos_BI_pipeline( @@ -324,6 +328,7 @@ def test_conv_relu6_tosa_BI(self, test_data: torch.Tensor): self._test_conv_combo_tosa_BI_pipeline(model, test_data) @parameterized.expand(ComboConvRelu6.test_data) + @pytest.mark.corstone_fvp def test_conv_relu6_u55_BI(self, test_data: torch.Tensor): model = ComboConvRelu6() test_data = (test_data,) @@ -332,6 +337,7 @@ def test_conv_relu6_u55_BI(self, test_data: torch.Tensor): ) @parameterized.expand(ComboConvRelu6.test_data) + @pytest.mark.corstone_fvp def test_conv_relu6_u85_BI(self, test_data: torch.Tensor): model = ComboConvRelu6() test_data = (test_data,) @@ -347,11 +353,12 @@ def test_block_bottleneck_residual_tosa_MI(self): self._test_conv_combo_tosa_MI_pipeline(model, model.get_inputs()) # TODO: Investigate flakyness (MLTORCH-307) - @pytest.mark.flaky(reruns=3) + @unittest.skip(reason="Skiped due to flakyness (MLTORCH-307)") def test_block_bottleneck_residual_tosa_BI(self): model = ComboBlockBottleneckResidual() self._test_conv_combo_tosa_BI_pipeline(model, model.get_inputs()) + @pytest.mark.corstone_fvp def test_block_bottleneck_residual_u55_BI(self): model = ComboBlockBottleneckResidual() self._test_conv_combo_ethos_BI_pipeline( @@ -360,6 +367,7 @@ def test_block_bottleneck_residual_u55_BI(self): model.get_inputs(), ) + @pytest.mark.corstone_fvp def test_block_bottleneck_residual_u85_BI(self): model = ComboBlockBottleneckResidual() self._test_conv_combo_ethos_BI_pipeline( @@ -384,6 +392,7 @@ def test_conv_avgpool2d_tosa_BI(self, test_data: torch.Tensor): self._test_conv_combo_tosa_BI_pipeline(model, test_data) @parameterized.expand(ComboConvAvgPool2d.test_data) + @pytest.mark.corstone_fvp def test_conv_avgpool2d_u55_BI(self, test_data: torch.Tensor): model = ComboConvAvgPool2d() test_data = (test_data,) @@ -394,6 +403,7 @@ def test_conv_avgpool2d_u55_BI(self, test_data: torch.Tensor): ) @parameterized.expand(ComboConvAvgPool2d.test_data) + @pytest.mark.corstone_fvp def test_conv_avgpool2d_u85_BI(self, test_data: torch.Tensor): model = ComboConvAvgPool2d() test_data = (test_data,) diff --git a/backends/arm/test/ops/test_depthwise_conv.py b/backends/arm/test/ops/test_depthwise_conv.py index 5419332c378..3ce75840864 100644 --- a/backends/arm/test/ops/test_depthwise_conv.py +++ b/backends/arm/test/ops/test_depthwise_conv.py @@ -8,6 +8,8 @@ from typing import Tuple +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.ops.test_conv1d import Conv1d @@ -260,6 +262,7 @@ def test_dw_conv_tosa_BI(self, test_name: str, model: torch.nn.Module): ) # Works @parameterized.expand(testsuite_conv2d, skip_on_empty=True) + @pytest.mark.corstone_fvp @unittest.expectedFailure def test_dw_conv2d_u55_BI( self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False @@ -275,6 +278,7 @@ def test_dw_conv2d_u55_BI( # Expected to fail as conv1d needs transpose which is not supported # on u55. @parameterized.expand(testsuite_conv1d, skip_on_empty=True) + @pytest.mark.corstone_fvp @unittest.expectedFailure def test_dw_conv1d_u55_BI( self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False @@ -288,6 +292,7 @@ def test_dw_conv1d_u55_BI( ) @parameterized.expand(testsuite_conv1d + testsuite_conv2d_u85) + @pytest.mark.corstone_fvp def test_dw_conv_u85_BI( self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False ): @@ -301,6 +306,7 @@ def test_dw_conv_u85_BI( # All test cases except 3x3_1x3x256x256_gp3_st1 have numerical issues on FVP. MLETORCH-520 @parameterized.expand(testsuite_conv2d_u85_xfails) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_dw_conv_u85_BI_xfails( self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False diff --git a/backends/arm/test/ops/test_div.py b/backends/arm/test/ops/test_div.py index c28ba3acdf3..d5f61744692 100644 --- a/backends/arm/test/ops/test_div.py +++ b/backends/arm/test/ops/test_div.py @@ -10,6 +10,8 @@ from typing import Optional, Tuple, Union +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -184,6 +186,7 @@ def test_div_tosa_BI( self._test_div_tosa_BI_pipeline(self.Div(), test_data) @parameterized.expand(test_data_suite[:2]) + @pytest.mark.corstone_fvp def test_div_u55_BI( self, test_name: str, @@ -198,6 +201,7 @@ def test_div_u55_BI( # Numerical issues on FVP likely due to mul op, MLETORCH-521 @parameterized.expand(test_data_suite[2:]) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_div_u55_BI_xfails( self, @@ -212,6 +216,7 @@ def test_div_u55_BI_xfails( ) @parameterized.expand(test_data_suite[:2]) + @pytest.mark.corstone_fvp def test_div_u85_BI( self, test_name: str, @@ -226,6 +231,7 @@ def test_div_u85_BI( # Numerical issues on FVP likely due to mul op, MLETORCH-521 @parameterized.expand(test_data_suite[2:]) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_div_u85_BI_xfails( self, diff --git a/backends/arm/test/ops/test_exp.py b/backends/arm/test/ops/test_exp.py index 0bbd5097c31..3fa9f8c99fa 100644 --- a/backends/arm/test/ops/test_exp.py +++ b/backends/arm/test/ops/test_exp.py @@ -9,6 +9,8 @@ from typing import Tuple +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -111,12 +113,14 @@ def test_exp_tosa_BI(self, test_name: str, test_data: torch.Tensor): self._test_exp_tosa_BI_pipeline(self.Exp(), (test_data,)) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_exp_tosa_u55_BI(self, test_name: str, test_data: torch.Tensor): self._test_exp_ethosu_BI_pipeline( common.get_u55_compile_spec(), self.Exp(), (test_data,) ) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_exp_tosa_u85_BI(self, test_name: str, test_data: torch.Tensor): self._test_exp_ethosu_BI_pipeline( common.get_u85_compile_spec(), self.Exp(), (test_data,) diff --git a/backends/arm/test/ops/test_expand.py b/backends/arm/test/ops/test_expand.py index bd6b74d77b0..084eec138f9 100644 --- a/backends/arm/test/ops/test_expand.py +++ b/backends/arm/test/ops/test_expand.py @@ -9,8 +9,11 @@ # import unittest + from typing import Sequence, Tuple +import pytest + import torch from executorch.backends.arm.quantizer.arm_quantizer import ( @@ -111,6 +114,7 @@ def test_expand_tosa_BI(self, test_input, multiples): # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(Expand.test_parameters) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_expand_u55_BI(self, test_input, multiples): self._test_expand_ethosu_BI_pipeline( @@ -119,6 +123,7 @@ def test_expand_u55_BI(self, test_input, multiples): # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(Expand.test_parameters) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_expand_u85_BI(self, test_input, multiples): self._test_expand_ethosu_BI_pipeline( diff --git a/backends/arm/test/ops/test_full.py b/backends/arm/test/ops/test_full.py index 0c8afe1390b..1b6d6e6ae39 100644 --- a/backends/arm/test/ops/test_full.py +++ b/backends/arm/test/ops/test_full.py @@ -10,8 +10,11 @@ # import unittest + from typing import Tuple +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -145,6 +148,7 @@ def test_full_tosa_BI(self, test_tensor: Tuple): # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(AddVariableFull.test_parameters) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_full_u55_BI(self, test_tensor: Tuple): self._test_full_tosa_u55_pipeline( @@ -154,6 +158,7 @@ def test_full_u55_BI(self, test_tensor: Tuple): # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(AddVariableFull.test_parameters) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_full_u85_BI(self, test_tensor: Tuple): self._test_full_tosa_u85_pipeline( diff --git a/backends/arm/test/ops/test_hardtanh.py b/backends/arm/test/ops/test_hardtanh.py index 65d5ca92b26..7125920c8c9 100644 --- a/backends/arm/test/ops/test_hardtanh.py +++ b/backends/arm/test/ops/test_hardtanh.py @@ -6,8 +6,11 @@ # LICENSE file in the root directory of this source tree. import unittest + from typing import Tuple +import pytest + import torch from executorch.backends.arm.quantizer.arm_quantizer import ( @@ -124,12 +127,14 @@ def test_hardtanh_tosa_BI(self, test_name: str, test_data: torch.Tensor): self._test_hardtanh_tosa_BI_pipeline(self.HardTanh(), (test_data,)) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_hardtanh_tosa_u55_BI(self, test_name: str, test_data: torch.Tensor): self._test_hardtanh_tosa_ethosu_BI_pipeline( common.get_u55_compile_spec(), self.HardTanh(), (test_data,) ) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_hardtanh_tosa_u85_BI(self, test_name: str, test_data: torch.Tensor): self._test_hardtanh_tosa_ethosu_BI_pipeline( common.get_u85_compile_spec(), self.HardTanh(), (test_data,) diff --git a/backends/arm/test/ops/test_layer_norm.py b/backends/arm/test/ops/test_layer_norm.py index a1c17475ad5..2d88421fb56 100644 --- a/backends/arm/test/ops/test_layer_norm.py +++ b/backends/arm/test/ops/test_layer_norm.py @@ -5,8 +5,11 @@ # LICENSE file in the root directory of this source tree. import unittest + from typing import List, Tuple, Union +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -158,6 +161,7 @@ def test_layer_norm_tosa_BI( # Numerical issues on FVP likely due to mul op, MLETORCH-521 # Skip tests that require transposes. @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp @unittest.expectedFailure def test_layer_norm_u55_BI_xfails( self, @@ -171,6 +175,7 @@ def test_layer_norm_u55_BI_xfails( # Numerical issues on FVP likely due to mul op, MLETORCH-521 @parameterized.expand(test_data_suite[:-2]) + @pytest.mark.corstone_fvp @unittest.expectedFailure def test_layer_norm_u85_BI_xfails( self, @@ -183,6 +188,7 @@ def test_layer_norm_u85_BI_xfails( ) @parameterized.expand(test_data_suite[-2:]) + @pytest.mark.corstone_fvp def test_layer_norm_u85_BI( self, test_name: str, diff --git a/backends/arm/test/ops/test_linear.py b/backends/arm/test/ops/test_linear.py index 4385ca9864a..cd14b7801d2 100644 --- a/backends/arm/test/ops/test_linear.py +++ b/backends/arm/test/ops/test_linear.py @@ -10,6 +10,8 @@ from typing import Tuple +import pytest + import torch from executorch.backends.arm.test import common, conftest @@ -228,6 +230,7 @@ def test_linear_tosa_BI( ) @parameterized.expand(test_data_suite_rank1) + @pytest.mark.corstone_fvp def test_linear_tosa_u55_BI( self, test_name: str, diff --git a/backends/arm/test/ops/test_log.py b/backends/arm/test/ops/test_log.py index f9c34b1ff5e..0226a62328b 100644 --- a/backends/arm/test/ops/test_log.py +++ b/backends/arm/test/ops/test_log.py @@ -9,6 +9,8 @@ from typing import Tuple +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -111,12 +113,14 @@ def test_log_tosa_BI(self, test_name: str, test_data: torch.Tensor): self._test_log_tosa_BI_pipeline(self.Log(), (test_data,)) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_log_tosa_u55_BI(self, test_name: str, test_data: torch.Tensor): self._test_log_ethosu_BI_pipeline( common.get_u55_compile_spec(), self.Log(), (test_data,) ) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_log_tosa_u85_BI(self, test_name: str, test_data: torch.Tensor): self._test_log_ethosu_BI_pipeline( common.get_u85_compile_spec(), self.Log(), (test_data,) diff --git a/backends/arm/test/ops/test_max_pool.py b/backends/arm/test/ops/test_max_pool.py index ed2a3743198..a693c7d5497 100644 --- a/backends/arm/test/ops/test_max_pool.py +++ b/backends/arm/test/ops/test_max_pool.py @@ -10,6 +10,8 @@ from typing import Tuple +import pytest + import torch from executorch.backends.arm.quantizer.arm_quantizer import ( ArmQuantizer, @@ -160,6 +162,7 @@ def test_maxpool2d_tosa_BI( ) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_maxpool2d_tosa_u55_BI( self, test_name: str, @@ -177,6 +180,7 @@ def test_maxpool2d_tosa_u55_BI( ) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_maxpool2d_tosa_u85_BI( self, test_name: str, @@ -216,6 +220,7 @@ def test_maxpool2d_tosa_BI_mult_batches( ) @parameterized.expand(test_data_suite_mult_batches) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP # TODO: MLETORCH-433 def test_maxpool2d_tosa_u55_BI_mult_batches( self, @@ -234,6 +239,7 @@ def test_maxpool2d_tosa_u55_BI_mult_batches( ) @parameterized.expand(test_data_suite_mult_batches) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP # TODO: MLETORCH-433 def test_maxpool2d_tosa_u85_BI_mult_batches( self, diff --git a/backends/arm/test/ops/test_mul.py b/backends/arm/test/ops/test_mul.py index e18efc31a3b..9d789a8e336 100644 --- a/backends/arm/test/ops/test_mul.py +++ b/backends/arm/test/ops/test_mul.py @@ -7,6 +7,8 @@ import unittest +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -153,6 +155,7 @@ def test_mul_tosa_BI( self._test_mul_tosa_BI_pipeline(self.Mul(), test_data) @parameterized.expand(test_data_sute) + @pytest.mark.corstone_fvp def test_mul_u55_BI( self, test_name: str, @@ -165,6 +168,7 @@ def test_mul_u55_BI( ) @parameterized.expand(test_data_sute) + @pytest.mark.corstone_fvp def test_mul_u85_BI( self, test_name: str, diff --git a/backends/arm/test/ops/test_permute.py b/backends/arm/test/ops/test_permute.py index 72dd82f7262..b373af1401b 100644 --- a/backends/arm/test/ops/test_permute.py +++ b/backends/arm/test/ops/test_permute.py @@ -6,8 +6,11 @@ # LICENSE file in the root directory of this source tree. import unittest + from typing import Tuple +import pytest + import torch from executorch.backends.arm.quantizer.arm_quantizer import ( @@ -137,6 +140,7 @@ def test_permute_tosa_BI( # Expected to fail as TOSA.Transpose is not supported by Ethos-U55. @parameterized.expand(test_data_suite[0:1]) + @pytest.mark.corstone_fvp @unittest.expectedFailure def test_permute_u55_BI( self, test_name: str, test_data: torch.Tensor, dims: list[int] @@ -146,6 +150,7 @@ def test_permute_u55_BI( ) @parameterized.expand(test_data_suite[:-2]) + @pytest.mark.corstone_fvp def test_permute_u85_BI( self, test_name: str, test_data: torch.Tensor, dims: list[int] ): @@ -155,6 +160,7 @@ def test_permute_u85_BI( # Fails since on FVP since N > 1 is not supported. MLETORCH-517 @parameterized.expand(test_data_suite[-2:]) + @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP def test_permute_u85_BI_xfails( self, test_name: str, test_data: torch.Tensor, dims: list[int] diff --git a/backends/arm/test/ops/test_reciprocal.py b/backends/arm/test/ops/test_reciprocal.py index ccee3d39619..b3233d02a92 100644 --- a/backends/arm/test/ops/test_reciprocal.py +++ b/backends/arm/test/ops/test_reciprocal.py @@ -6,6 +6,8 @@ import unittest +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -112,6 +114,7 @@ def test_reciprocal_tosa_BI(self, test_name: str, input_: torch.Tensor): self._test_reciprocal_tosa_BI_pipeline(self.Reciprocal(), test_data) @parameterized.expand(test_data_suite) + @pytest.mark.corstone_fvp def test_reciprocal_u55_BI(self, test_name: str, input_: torch.Tensor): test_data = (input_,) self._test_reciprocal_u55_BI_pipeline(self.Reciprocal(), test_data) diff --git a/backends/arm/test/ops/test_sub.py b/backends/arm/test/ops/test_sub.py index 851516753da..0812f8a47a1 100644 --- a/backends/arm/test/ops/test_sub.py +++ b/backends/arm/test/ops/test_sub.py @@ -9,6 +9,8 @@ from typing import Tuple +import pytest + import torch from executorch.backends.arm.test import common, conftest from executorch.backends.arm.test.tester.arm_tester import ArmTester @@ -111,6 +113,7 @@ def test_sub_tosa_BI(self, test_data: torch.Tensor): self._test_sub_tosa_BI_pipeline(self.Sub(), test_data) @parameterized.expand(Sub.test_parameters) + @pytest.mark.corstone_fvp def test_sub_u55_BI(self, test_data: torch.Tensor): test_data = (test_data,) self._test_sub_ethosu_BI_pipeline( @@ -118,6 +121,7 @@ def test_sub_u55_BI(self, test_data: torch.Tensor): ) @parameterized.expand(Sub.test_parameters) + @pytest.mark.corstone_fvp def test_sub_u85_BI(self, test_data: torch.Tensor): test_data = (test_data,) self._test_sub_ethosu_BI_pipeline( @@ -135,6 +139,7 @@ def test_sub2_tosa_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): self._test_sub_tosa_BI_pipeline(self.Sub2(), test_data) @parameterized.expand(Sub2.test_parameters) + @pytest.mark.corstone_fvp def test_sub2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) self._test_sub_ethosu_BI_pipeline( @@ -142,6 +147,7 @@ def test_sub2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): ) @parameterized.expand(Sub2.test_parameters) + @pytest.mark.corstone_fvp def test_sub2_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) self._test_sub_ethosu_BI_pipeline( diff --git a/backends/arm/test/pytest.ini b/backends/arm/test/pytest.ini new file mode 100644 index 00000000000..3af1f0d0971 --- /dev/null +++ b/backends/arm/test/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +addopts = --strict-markers +markers = + slow: Tests that take long time + corstone_fvp: Tests that use Corstone300 or Corstone320 FVP \ No newline at end of file