From f5c7b1e845dea8cbf767544cd60a5005c87a8766 Mon Sep 17 00:00:00 2001 From: Erik Lundell Date: Fri, 15 Nov 2024 15:39:55 +0100 Subject: [PATCH] Add FVP testing to ops Add expected fails accordingly Signed-off-by: Erik Lundell Change-Id: Ic76626256ae4c53258536ffa747a7ee02832b168 --- backends/arm/test/ops/test_avg_pool.py | 13 +++++--- backends/arm/test/ops/test_bmm.py | 22 ++++++++++++-- backends/arm/test/ops/test_cat.py | 10 +++++- backends/arm/test/ops/test_clone.py | 5 ++- backends/arm/test/ops/test_conv1d.py | 8 ++++- backends/arm/test/ops/test_conv2d.py | 9 +++++- backends/arm/test/ops/test_conv_combos.py | 7 ++++- backends/arm/test/ops/test_depthwise_conv.py | 31 +++++++++++++++---- backends/arm/test/ops/test_div.py | 30 +++++++++++++++--- backends/arm/test/ops/test_exp.py | 7 +++-- backends/arm/test/ops/test_expand.py | 9 +++++- backends/arm/test/ops/test_full.py | 9 +++++- backends/arm/test/ops/test_hardtanh.py | 21 ++++++++++--- backends/arm/test/ops/test_layer_norm.py | 23 ++++++++++++-- backends/arm/test/ops/test_log.py | 5 ++- backends/arm/test/ops/test_mul.py | 32 ++++++++++++-------- backends/arm/test/ops/test_permute.py | 16 ++++++++-- backends/arm/test/ops/test_reciprocal.py | 19 ++++++------ backends/arm/test/ops/test_sub.py | 7 +++-- 19 files changed, 223 insertions(+), 60 deletions(-) diff --git a/backends/arm/test/ops/test_avg_pool.py b/backends/arm/test/ops/test_avg_pool.py index afd079fb957..ad3ddf8c0a9 100644 --- a/backends/arm/test/ops/test_avg_pool.py +++ b/backends/arm/test/ops/test_avg_pool.py @@ -23,10 +23,10 @@ test_data_suite = [ # (test_name, test_data, [kernel_size, stride, padding]) - ("zeros", torch.zeros(20, 16, 50, 32), [4, 2, 0]), - ("ones", torch.zeros(20, 16, 50, 32), [4, 2, 0]), - ("rand", torch.rand(20, 16, 50, 32), [4, 2, 0]), - ("randn", torch.randn(20, 16, 50, 32), [4, 2, 0]), + ("zeros", torch.zeros(1, 16, 50, 32), [4, 2, 0]), + ("ones", torch.zeros(1, 16, 50, 32), [4, 2, 0]), + ("rand", torch.rand(1, 16, 50, 32), [4, 2, 0]), + ("randn", torch.randn(1, 16, 50, 32), [4, 2, 0]), ] @@ -101,7 +101,7 @@ def _test_avgpool2d_tosa_ethos_BI_pipeline( test_data: Tuple[torch.tensor], ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -116,7 +116,10 @@ def _test_avgpool2d_tosa_ethos_BI_pipeline( .check_not(["executorch_exir_dialects_edge__ops_aten_avg_pool2d_default"]) .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(test_data_suite) def test_avgpool2d_tosa_MI( diff --git a/backends/arm/test/ops/test_bmm.py b/backends/arm/test/ops/test_bmm.py index 62466571209..824ec463724 100644 --- a/backends/arm/test/ops/test_bmm.py +++ b/backends/arm/test/ops/test_bmm.py @@ -41,7 +41,7 @@ def forward(self, x, y): class BMMSingleInput(torch.nn.Module): test_parameters = [ (torch.rand(20, 3, 3),), - (torch.ones(2, 128, 128),), + (torch.rand(2, 128, 128),), (10000 * torch.randn(4, 25, 25),), (5 + 5 * torch.randn(3, 64, 64),), ] @@ -96,7 +96,7 @@ def _test_bmm_ethosu_BI_pipeline( compile_spec: CompileSpec, test_data: Tuple[torch.Tensor, ...], ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -110,7 +110,10 @@ def _test_bmm_ethosu_BI_pipeline( .partition() .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(inputs=test_data, qtol=1) @parameterized.expand(BMM.test_parameters) def test_bmm_tosa_MI(self, operand1: torch.Tensor, operand2: torch.Tensor): @@ -143,9 +146,20 @@ def test_bmm_single_input_tosa_BI(self, operand1: torch.Tensor): self._test_bmm_tosa_BI_pipeline(self.BMMSingleInput(), test_data) @parameterized.expand(BMM.test_parameters) + @unittest.expectedFailure def test_bmm_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) - self._test_bmm_tosa_BI_pipeline(self.BMM(), test_data) + self._test_bmm_ethosu_BI_pipeline( + self.BMM(), common.get_u55_compile_spec(), test_data + ) + + @parameterized.expand(BMM.test_parameters) + @common.expectedFailureOnFVP + def test_bmm_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): + test_data = (operand1, operand2) + self._test_bmm_ethosu_BI_pipeline( + self.BMM(), common.get_u85_compile_spec(), test_data + ) # Expected to fail with error: Warning, unsupported fusing of TOSA Rescale previous operator is of type: Memcpy @parameterized.expand(BMMSingleInput.test_parameters) @@ -156,7 +170,9 @@ def test_bmm_single_input_u55_BI(self, operand1: torch.Tensor): self.BMMSingleInput(), common.get_u55_compile_spec(), test_data ) + # Numerical issues on FVP, MLETORCH 534 @parameterized.expand(BMMSingleInput.test_parameters) + @common.expectedFailureOnFVP def test_bmm_single_input_u85_BI(self, operand1: torch.Tensor): test_data = (operand1,) self._test_bmm_ethosu_BI_pipeline( diff --git a/backends/arm/test/ops/test_cat.py b/backends/arm/test/ops/test_cat.py index b380c44d52f..88846369d07 100644 --- a/backends/arm/test/ops/test_cat.py +++ b/backends/arm/test/ops/test_cat.py @@ -96,7 +96,7 @@ def _test_cat_ethosu_BI_pipeline( compile_spec: CompileSpec, test_data: Tuple[tuple[torch.Tensor, ...], int], ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -108,10 +108,14 @@ def _test_cat_ethosu_BI_pipeline( .check(["torch.ops.quantized_decomposed"]) .to_edge() .partition() + .dump_artifact() .check_not(["executorch_exir_dialects_edge__ops_aten_cat_default"]) .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(inputs=test_data) @parameterized.expand(Cat.test_parameters) def test_cat_tosa_MI(self, operands: tuple[torch.Tensor, ...], dim: int): @@ -129,14 +133,18 @@ def test_cat_tosa_BI(self, operands: tuple[torch.Tensor, ...], dim: int): test_data = (operands, dim) self._test_cat_tosa_BI_pipeline(self.Cat(), test_data) + # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(Cat.test_parameters) + @common.expectedFailureOnFVP def test_cat_u55_BI(self, operands: tuple[torch.Tensor, ...], dim: int): test_data = (operands, dim) self._test_cat_ethosu_BI_pipeline( self.Cat(), common.get_u55_compile_spec(), test_data ) + # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(Cat.test_parameters) + @common.expectedFailureOnFVP def test_cat_u85_BI(self, operands: tuple[torch.Tensor, ...], dim: int): test_data = (operands, dim) self._test_cat_ethosu_BI_pipeline( diff --git a/backends/arm/test/ops/test_clone.py b/backends/arm/test/ops/test_clone.py index 4721f257b0c..6b5216a8e1a 100644 --- a/backends/arm/test/ops/test_clone.py +++ b/backends/arm/test/ops/test_clone.py @@ -85,7 +85,7 @@ def _test_clone_tosa_ethos_pipeline( test_data: Tuple[torch.Tensor], ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) - ( + tester = ( ArmTester(module, example_inputs=test_data, compile_spec=compile_spec) .quantize(Quantize(quantizer, get_symmetric_quantization_config())) .export() @@ -94,7 +94,10 @@ def _test_clone_tosa_ethos_pipeline( .partition() .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) def _test_clone_tosa_u55_pipeline( self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] diff --git a/backends/arm/test/ops/test_conv1d.py b/backends/arm/test/ops/test_conv1d.py index 133148faef9..f00c7984a1a 100644 --- a/backends/arm/test/ops/test_conv1d.py +++ b/backends/arm/test/ops/test_conv1d.py @@ -268,7 +268,7 @@ def _test_conv1d_ethosu_BI_pipeline( compile_spec: CompileSpec, test_data: Tuple[torch.Tensor], ): - ( + tester = ( ArmTester(module, example_inputs=test_data, compile_spec=compile_spec) .quantize() .export() @@ -277,7 +277,10 @@ def _test_conv1d_ethosu_BI_pipeline( .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"]) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(testsuite) def test_conv1d_tosa_MI(self, test_name, model): @@ -295,6 +298,9 @@ def test_conv1d_u55_BI(self, test_name, model): model, common.get_u55_compile_spec(), model.get_inputs() ) + # This specific test case has numerical errors on FVP, MLETORCH-520. + testsuite.remove(("5_3x2x128_st1", conv1d_5_3x2x128_st1)) + @parameterized.expand(testsuite) def test_conv1d_u85_BI(self, test_name, model): self._test_conv1d_ethosu_BI_pipeline( diff --git a/backends/arm/test/ops/test_conv2d.py b/backends/arm/test/ops/test_conv2d.py index 43c3e851396..21df4bf0d52 100644 --- a/backends/arm/test/ops/test_conv2d.py +++ b/backends/arm/test/ops/test_conv2d.py @@ -295,7 +295,7 @@ def _test_conv2d_ethosu_BI_pipeline( module: torch.nn.Module, test_data: Tuple[torch.Tensor], ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -308,7 +308,10 @@ def _test_conv2d_ethosu_BI_pipeline( .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"]) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(testsuite) def test_conv2d_tosa_MI(self, test_name, model): @@ -318,6 +321,10 @@ def test_conv2d_tosa_MI(self, test_name, model): def test_conv2d_tosa_BI(self, test_name, model): self._test_conv2d_tosa_BI_pipeline(model, model.get_inputs()) + # These cases have numerical issues on FVP, MLETORCH-520 + testsuite.remove(("2x2_3x2x40x40_nobias", conv2d_2x2_3x2x40x40_nobias)) + testsuite.remove(("5x5_3x2x128x128_st1", conv2d_5x5_3x2x128x128_st1)) + @parameterized.expand(testsuite) def test_conv2d_u55_BI(self, test_name, model): self._test_conv2d_ethosu_BI_pipeline( diff --git a/backends/arm/test/ops/test_conv_combos.py b/backends/arm/test/ops/test_conv_combos.py index 3e9bdef958c..7555fff7206 100644 --- a/backends/arm/test/ops/test_conv_combos.py +++ b/backends/arm/test/ops/test_conv_combos.py @@ -238,7 +238,7 @@ def _test_conv_combo_ethos_BI_pipeline( compile_spec: CompileSpec, test_data: Tuple[torch.Tensor], ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -251,7 +251,10 @@ def _test_conv_combo_ethos_BI_pipeline( .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .check_not(list(module.edge_op_list)) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) #################### ## Conv + meandim ## @@ -272,6 +275,8 @@ def test_conv_meandim_u55_BI(self): model.get_inputs(), ) + # Numerical Issues on FVP, MLETORCH-520 + @common.expectedFailureOnFVP def test_conv_meandim_u85_BI(self): model = ComboConv2dMeandim() self._test_conv_combo_ethos_BI_pipeline( diff --git a/backends/arm/test/ops/test_depthwise_conv.py b/backends/arm/test/ops/test_depthwise_conv.py index 4bfa863c49f..28cb9ac8443 100644 --- a/backends/arm/test/ops/test_depthwise_conv.py +++ b/backends/arm/test/ops/test_depthwise_conv.py @@ -8,8 +8,6 @@ from typing import Tuple -import pytest - import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.ops.test_conv1d import Conv1d @@ -160,8 +158,8 @@ testsuite_conv1d = [ ("2_1x6x4_gp6_st1", dw_conv1d_2_1x6x4_gp6_st1), - ("3_1x3x256_gp3_st1", dw_conv1d_3_1x3x256_gp3_st1), ("two_dw_conv1d", two_dw_conv1d), + ("3_1x3x256_gp3_st1", dw_conv1d_3_1x3x256_gp3_st1), ("3_1x3x14_gp3_st1", dw_conv1d_3_1x3x14_gp3_st1), ] @@ -217,7 +215,7 @@ def _test_dw_conv_ethos_BI_pipeline( compile_spec: CompileSpec, test_data: Tuple[torch.Tensor], ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -230,7 +228,10 @@ def _test_dw_conv_ethos_BI_pipeline( .check_not(["executorch_exir_dialects_edge__ops_aten_convolution_default"]) .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(testsuite_conv1d + testsuite_conv2d) def test_dw_conv_tosa_MI(self, test_name: str, model: torch.nn.Module): @@ -238,11 +239,15 @@ def test_dw_conv_tosa_MI(self, test_name: str, model: torch.nn.Module): # TODO: Investigate flakyness (MLTORCH-307) @parameterized.expand(testsuite_conv1d + testsuite_conv2d) - @pytest.mark.flaky(reruns=3) def test_dw_conv_tosa_BI(self, test_name: str, model: torch.nn.Module): self._test_dw_conv_tosa_BI_pipeline(model, model.get_inputs()) + testsuite_conv2d.remove( + ("3x3_1x3x256x256_gp3_st1", dw_conv2d_3x3_1x3x256x256_gp3_st1) + ) # Works + @parameterized.expand(testsuite_conv2d, skip_on_empty=True) + @common.expectedFailureOnFVP def test_dw_conv2d_u55_BI( self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False ): @@ -269,7 +274,21 @@ def test_dw_conv1d_u55_BI( model.get_inputs(), ) - @parameterized.expand(testsuite_conv1d + testsuite_conv2d) + # All test cases except 3x3_1x3x256x256_gp3_st1 have numerical issues on FVP. MLETORCH-520 + @parameterized.expand(testsuite_conv1d[:-2] + testsuite_conv2d) + @common.expectedFailureOnFVP + def test_dw_conv_u85_BI_xfails( + self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False + ): + self._test_dw_conv_ethos_BI_pipeline( + model, + common.get_u85_compile_spec( + permute_memory_to_nhwc=True, quantize_io=set_quantize_io + ), + model.get_inputs(), + ) + + @parameterized.expand(testsuite_conv1d[-2:]) def test_dw_conv_u85_BI( self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False ): diff --git a/backends/arm/test/ops/test_div.py b/backends/arm/test/ops/test_div.py index 28cc6866906..b3815f3e7c6 100644 --- a/backends/arm/test/ops/test_div.py +++ b/backends/arm/test/ops/test_div.py @@ -136,10 +136,10 @@ def _test_div_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, atol=1, rtol=0.1) ) - def _test_div_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + def _test_div_ethos_BI_pipeline( + self, module: torch.nn.Module, compile_spec, test_data: Tuple[torch.Tensor] ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -155,7 +155,10 @@ def _test_div_u55_BI_pipeline( .partition() .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(test_data_suite) def test_div_tosa_MI( @@ -180,7 +183,9 @@ def test_div_tosa_BI( test_data = (input_, other_) self._test_div_tosa_BI_pipeline(self.Div(), test_data) + # Numerical issues on FVP likely due to mul op, MLETORCH-521 @parameterized.expand(test_data_suite) + @common.expectedFailureOnFVP def test_div_u55_BI( self, test_name: str, @@ -189,4 +194,21 @@ def test_div_u55_BI( rounding_mode: Optional[str] = None, ): test_data = (input_, other_) - self._test_div_u55_BI_pipeline(self.Div(), test_data) + self._test_div_ethos_BI_pipeline( + self.Div(), common.get_u55_compile_spec(), test_data + ) + + # Numerical issues on FVP likely due to mul op, MLETORCH-521 + @parameterized.expand(test_data_suite) + @common.expectedFailureOnFVP + def test_div_u85_BI( + self, + test_name: str, + input_: Union[torch.Tensor, torch.types.Number], + other_: Union[torch.Tensor, torch.types.Number], + rounding_mode: Optional[str] = None, + ): + test_data = (input_, other_) + self._test_div_ethos_BI_pipeline( + self.Div(), common.get_u85_compile_spec(), test_data + ) diff --git a/backends/arm/test/ops/test_exp.py b/backends/arm/test/ops/test_exp.py index c706b7b206d..f33e0a9058d 100644 --- a/backends/arm/test/ops/test_exp.py +++ b/backends/arm/test/ops/test_exp.py @@ -20,7 +20,7 @@ ("zeros", torch.zeros(1, 10, 10, 10)), ("ones", torch.ones(10, 10, 10)), ("rand", torch.rand(10, 10) - 0.5), - ("randn_pos", torch.randn(10) + 10), + ("randn_pos", torch.randn(1, 4, 4, 4) + 10), ("randn_neg", torch.randn(10) - 10), ("ramp", torch.arange(-16, 16, 0.2)), ] @@ -78,7 +78,7 @@ def _test_exp_ethosu_BI_pipeline( module: torch.nn.Module, test_data: Tuple[torch.tensor], ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -93,7 +93,10 @@ def _test_exp_ethosu_BI_pipeline( .check_not(["executorch_exir_dialects_edge__ops_aten_exp_default"]) .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(test_data_suite) def test_exp_tosa_MI( diff --git a/backends/arm/test/ops/test_expand.py b/backends/arm/test/ops/test_expand.py index effa7ce713b..27f311b5463 100644 --- a/backends/arm/test/ops/test_expand.py +++ b/backends/arm/test/ops/test_expand.py @@ -81,7 +81,7 @@ def _test_expand_ethosu_BI_pipeline( self, compile_spec: CompileSpec, module: torch.nn.Module, test_data: Tuple ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -95,7 +95,10 @@ def _test_expand_ethosu_BI_pipeline( .check_not(["torch.ops.aten.expand.default"]) .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(Expand.test_parameters) def test_expand_tosa_MI(self, test_input, multiples): @@ -105,13 +108,17 @@ def test_expand_tosa_MI(self, test_input, multiples): def test_expand_tosa_BI(self, test_input, multiples): self._test_expand_tosa_BI_pipeline(self.Expand(), (test_input, multiples)) + # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(Expand.test_parameters) + @common.expectedFailureOnFVP def test_expand_u55_BI(self, test_input, multiples): self._test_expand_ethosu_BI_pipeline( common.get_u55_compile_spec(), self.Expand(), (test_input, multiples) ) + # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(Expand.test_parameters) + @common.expectedFailureOnFVP def test_expand_u85_BI(self, test_input, multiples): self._test_expand_ethosu_BI_pipeline( common.get_u85_compile_spec(), self.Expand(), (test_input, multiples) diff --git a/backends/arm/test/ops/test_full.py b/backends/arm/test/ops/test_full.py index d4cfc5c3692..9857a7b87be 100644 --- a/backends/arm/test/ops/test_full.py +++ b/backends/arm/test/ops/test_full.py @@ -97,7 +97,7 @@ def _test_full_tosa_BI_pipeline( def _test_full_tosa_ethos_pipeline( self, compile_spec: list[CompileSpec], module: torch.nn.Module, test_data: Tuple ): - ( + tester = ( ArmTester(module, example_inputs=test_data, compile_spec=compile_spec) .quantize() .export() @@ -107,7 +107,10 @@ def _test_full_tosa_ethos_pipeline( .check_not(["executorch_exir_dialects_edge__ops_aten_full_default"]) .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) def _test_full_tosa_u55_pipeline(self, module: torch.nn.Module, test_data: Tuple): self._test_full_tosa_ethos_pipeline( @@ -140,14 +143,18 @@ def test_full_tosa_MI(self, test_tensor: Tuple): def test_full_tosa_BI(self, test_tensor: Tuple): self._test_full_tosa_BI_pipeline(self.AddVariableFull(), test_tensor, False) + # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(AddVariableFull.test_parameters) + @common.expectedFailureOnFVP def test_full_u55_BI(self, test_tensor: Tuple): self._test_full_tosa_u55_pipeline( self.AddVariableFull(), test_tensor, ) + # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(AddVariableFull.test_parameters) + @common.expectedFailureOnFVP def test_full_u85_BI(self, test_tensor: Tuple): self._test_full_tosa_u85_pipeline( self.AddVariableFull(), diff --git a/backends/arm/test/ops/test_hardtanh.py b/backends/arm/test/ops/test_hardtanh.py index a9f12abdf02..10073c50956 100644 --- a/backends/arm/test/ops/test_hardtanh.py +++ b/backends/arm/test/ops/test_hardtanh.py @@ -87,15 +87,15 @@ def _test_hardtanh_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data) ) - def _test_hardtanh_tosa_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + def _test_hardtanh_tosa_ethosu_BI_pipeline( + self, compile_spec, module: torch.nn.Module, test_data: Tuple[torch.tensor] ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) - ( + tester = ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize(Quantize(quantizer, get_symmetric_quantization_config())) .export() @@ -106,7 +106,10 @@ def _test_hardtanh_tosa_u55_BI_pipeline( .check_not(["executorch_exir_dialects_edge__ops_aten_hardtanh_default"]) .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(test_data_suite) def test_hardtanh_tosa_MI( @@ -122,4 +125,12 @@ def test_hardtanh_tosa_BI(self, test_name: str, test_data: torch.Tensor): @parameterized.expand(test_data_suite) def test_hardtanh_tosa_u55_BI(self, test_name: str, test_data: torch.Tensor): - self._test_hardtanh_tosa_u55_BI_pipeline(self.HardTanh(), (test_data,)) + self._test_hardtanh_tosa_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.HardTanh(), (test_data,) + ) + + @parameterized.expand(test_data_suite) + def test_hardtanh_tosa_u85_BI(self, test_name: str, test_data: torch.Tensor): + self._test_hardtanh_tosa_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.HardTanh(), (test_data,) + ) diff --git a/backends/arm/test/ops/test_layer_norm.py b/backends/arm/test/ops/test_layer_norm.py index f059d71eba8..0b06044a59a 100644 --- a/backends/arm/test/ops/test_layer_norm.py +++ b/backends/arm/test/ops/test_layer_norm.py @@ -115,7 +115,7 @@ def _test_layernorm_ethosu_BI_pipeline( compile_spec: CompileSpec, test_data: Tuple[torch.Tensor], ): - ( + tester = ( ArmTester( model=module, example_inputs=test_data, @@ -128,7 +128,10 @@ def _test_layernorm_ethosu_BI_pipeline( .partition() .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(test_data_suite) def test_layer_norm_tosa_MI( @@ -152,8 +155,10 @@ def test_layer_norm_tosa_BI( self.LayerNorm(*model_params), (test_data,) ) + # Numerical issues on FVP likely due to mul op, MLETORCH-521 # Skip tests that require transposes. @parameterized.expand(test_data_suite[:-2]) + @common.expectedFailureOnFVP def test_layer_norm_u55_BI( self, test_name: str, @@ -164,7 +169,21 @@ def test_layer_norm_u55_BI( self.LayerNorm(*model_params), common.get_u55_compile_spec(), (test_data,) ) - @parameterized.expand(test_data_suite) + # Numerical issues on FVP likely due to mul op, MLETORCH-521 + @parameterized.expand(test_data_suite[:-1]) + @common.expectedFailureOnFVP + def test_layer_norm_u85_BI_fvp_xfails( + self, + test_name: str, + test_data: torch.Tensor, + model_params, + ): + self._test_layernorm_ethosu_BI_pipeline( + self.LayerNorm(*model_params), common.get_u85_compile_spec(), (test_data,) + ) + + @parameterized.expand(test_data_suite[-1:]) + @unittest.skip # Flaky def test_layer_norm_u85_BI( self, test_name: str, diff --git a/backends/arm/test/ops/test_log.py b/backends/arm/test/ops/test_log.py index 847635ea361..10175d27fbe 100644 --- a/backends/arm/test/ops/test_log.py +++ b/backends/arm/test/ops/test_log.py @@ -78,7 +78,7 @@ def _test_log_ethosu_BI_pipeline( module: torch.nn.Module, test_data: Tuple[torch.tensor], ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -93,7 +93,10 @@ def _test_log_ethosu_BI_pipeline( .check_not(["executorch_exir_dialects_edge__ops_aten_log_default"]) .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(test_data_suite) def test_log_tosa_MI( diff --git a/backends/arm/test/ops/test_mul.py b/backends/arm/test/ops/test_mul.py index 7fa20c25667..8f0321ea5f7 100644 --- a/backends/arm/test/ops/test_mul.py +++ b/backends/arm/test/ops/test_mul.py @@ -16,9 +16,9 @@ test_data_sute = [ # (test_name, input, other,) See torch.mul() for info ( - "op_mul_rank1_ones", - torch.ones(5), - torch.ones(5), + "op_mul_rank1_rand", + torch.rand(5) * 3.7, + torch.rand(5) * 1.5, ), ( "op_mul_rank2_rand", @@ -32,23 +32,23 @@ ), ( "op_mul_rank4_randn", - torch.randn(5, 10, 25, 20), - torch.randn(5, 10, 25, 20), + torch.randn(1, 10, 25, 20), + torch.randn(1, 10, 25, 20), ), ( "op_mul_rank4_ones_mul_negative", torch.ones(1, 10, 25, 20), - (-1) * torch.ones(5, 10, 25, 20), + (-1) * torch.ones(1, 10, 25, 20), ), ( "op_mul_rank4_negative_large_rand", - (-200) * torch.rand(5, 10, 25, 20), - torch.rand(5, 1, 1, 20), + (-200) * torch.rand(1, 10, 25, 20), + torch.rand(1, 1, 1, 20), ), ( "op_mul_rank4_large_randn", - 200 * torch.randn(5, 10, 25, 20), - torch.rand(5, 10, 25, 1), + 200 * torch.randn(1, 10, 25, 20), + torch.rand(1, 10, 25, 1), ), ] @@ -112,7 +112,7 @@ def _test_mul_ethosu_BI_pipeline( module: torch.nn.Module, test_data: tuple[torch.Tensor, torch.Tensor], ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -126,7 +126,10 @@ def _test_mul_ethosu_BI_pipeline( .partition() .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(test_data_sute) def test_mul_tosa_MI( @@ -149,7 +152,9 @@ def test_mul_tosa_BI( test_data = (input_, other_) self._test_mul_tosa_BI_pipeline(self.Mul(), test_data) + # Numerical issues on FVP, MLETORCH-521 @parameterized.expand(test_data_sute) + @common.expectedFailureOnFVP def test_mul_u55_BI( self, test_name: str, @@ -161,7 +166,10 @@ def test_mul_u55_BI( common.get_u55_compile_spec(), self.Mul(), test_data ) - @parameterized.expand(test_data_sute) + # Numerical issues on FVP, MLETORCH-521 + # test_data_sute[0] works on U85 + @parameterized.expand(test_data_sute[1:]) + @common.expectedFailureOnFVP def test_mul_u85_BI( self, test_name: str, diff --git a/backends/arm/test/ops/test_permute.py b/backends/arm/test/ops/test_permute.py index 62b6b823de7..92400215b76 100644 --- a/backends/arm/test/ops/test_permute.py +++ b/backends/arm/test/ops/test_permute.py @@ -100,7 +100,7 @@ def _test_permute_ethos_BI_pipeline( test_data: Tuple[torch.Tensor], ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -117,6 +117,8 @@ def _test_permute_ethos_BI_pipeline( .to_executorch() .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(test_data_suite) def test_permute_tosa_MI( @@ -143,10 +145,20 @@ def test_permute_u55_BI( self.Permute(dims=dims), common.get_u55_compile_spec(), (test_data,) ) - @parameterized.expand(test_data_suite) + @parameterized.expand(test_data_suite[:-2]) def test_permute_u85_BI( self, test_name: str, test_data: torch.Tensor, dims: list[int] ): self._test_permute_ethos_BI_pipeline( self.Permute(dims=dims), common.get_u85_compile_spec(), (test_data,) ) + + # Fails since on FVP since N > 1 is not supported. MLETORCH-517 + @parameterized.expand(test_data_suite[-2:]) + @common.expectedFailureOnFVP + def test_permute_u85_BI_xfails( + self, test_name: str, test_data: torch.Tensor, dims: list[int] + ): + self._test_permute_ethos_BI_pipeline( + self.Permute(dims=dims), common.get_u85_compile_spec(), (test_data,) + ) diff --git a/backends/arm/test/ops/test_reciprocal.py b/backends/arm/test/ops/test_reciprocal.py index 7745a614e6b..876f063c76c 100644 --- a/backends/arm/test/ops/test_reciprocal.py +++ b/backends/arm/test/ops/test_reciprocal.py @@ -22,12 +22,12 @@ torch.rand(5) * 5, ), ("op_reciprocal_rank1_negative_ones", torch.ones(5) * (-1)), - ("op_reciprocal_rank4_ones", torch.ones(5, 10, 25, 20)), - ("op_reciprocal_rank4_negative_ones", (-1) * torch.ones(5, 10, 25, 20)), - ("op_reciprocal_rank4_ones_reciprocal_negative", torch.ones(5, 10, 25, 20)), - ("op_reciprocal_rank4_large_rand", 200 * torch.rand(5, 10, 25, 20)), - ("op_reciprocal_rank4_negative_large_rand", (-200) * torch.rand(5, 10, 25, 20)), - ("op_reciprocal_rank4_large_randn", 200 * torch.randn(5, 10, 25, 20) + 1), + ("op_reciprocal_rank4_ones", torch.ones(1, 10, 25, 20)), + ("op_reciprocal_rank4_negative_ones", (-1) * torch.ones(1, 10, 25, 20)), + ("op_reciprocal_rank4_ones_reciprocal_negative", torch.ones(1, 10, 25, 20)), + ("op_reciprocal_rank4_large_rand", 200 * torch.rand(1, 10, 25, 20)), + ("op_reciprocal_rank4_negative_large_rand", (-200) * torch.rand(1, 10, 25, 20)), + ("op_reciprocal_rank4_large_randn", 200 * torch.randn(1, 10, 25, 20) + 1), ] @@ -81,7 +81,7 @@ def _test_reciprocal_tosa_BI_pipeline( def _test_reciprocal_u55_BI_pipeline( self, module: torch.nn.Module, test_data: tuple[torch.Tensor] ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -95,15 +95,16 @@ def _test_reciprocal_u55_BI_pipeline( .partition() .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(test_data_suite) def test_reciprocal_tosa_MI(self, test_name: str, input_: torch.Tensor): test_data = (input_,) self._test_reciprocal_tosa_MI_pipeline(self.Reciprocal(), test_data) - # Expected to fail since ArmQuantizer cannot quantize a Reciprocal layer - # TODO(MLETORCH-129) @parameterized.expand(test_data_suite) def test_reciprocal_tosa_BI(self, test_name: str, input_: torch.Tensor): diff --git a/backends/arm/test/ops/test_sub.py b/backends/arm/test/ops/test_sub.py index 5c67240e521..327a8de994f 100644 --- a/backends/arm/test/ops/test_sub.py +++ b/backends/arm/test/ops/test_sub.py @@ -17,7 +17,7 @@ from parameterized import parameterized -class TestSimpleSub(unittest.TestCase): +class TestSub(unittest.TestCase): class Sub(torch.nn.Module): test_parameters = [ (torch.ones(5),), @@ -82,7 +82,7 @@ def _test_sub_ethosu_BI_pipeline( module: torch.nn.Module, test_data: Tuple[torch.Tensor], ): - ( + tester = ( ArmTester( module, example_inputs=test_data, @@ -96,7 +96,10 @@ def _test_sub_ethosu_BI_pipeline( .partition() .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() + .serialize() ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) @parameterized.expand(Sub.test_parameters) def test_sub_tosa_MI(self, test_data: torch.Tensor):