diff --git a/backends/arm/arm_backend.py b/backends/arm/arm_backend.py index a5f47c222fe..b83280763c2 100644 --- a/backends/arm/arm_backend.py +++ b/backends/arm/arm_backend.py @@ -52,8 +52,8 @@ def __init__(self): def ethosu_compile_spec( self, config: str, - system_config: Optional[str] = None, - memory_mode: Optional[str] = None, + system_config: str, + memory_mode: str, extra_flags: Optional[str] = None, config_ini: Optional[str] = "Arm/vela.ini", ) -> "ArmCompileSpecBuilder": diff --git a/backends/arm/test/models/test_mobilenet_v2_arm.py b/backends/arm/test/models/test_mobilenet_v2_arm.py index bb2c0d103f5..f9d408c1bae 100644 --- a/backends/arm/test/models/test_mobilenet_v2_arm.py +++ b/backends/arm/test/models/test_mobilenet_v2_arm.py @@ -102,3 +102,18 @@ def test_mv2_u55_BI(self): tester.run_method_and_compare_outputs( atol=1.0, qtol=1, inputs=self.model_inputs ) + + def test_mv2_u85_BI(self): + ( + ArmTester( + self.mv2, + example_inputs=self.model_inputs, + compile_spec=common.get_u85_compile_spec(permute_memory_to_nhwc=True), + ) + .quantize() + .export() + .to_edge(config=self._edge_compile_config) + .check(list(self.operators_after_quantization)) + .partition() + .to_executorch() + ) diff --git a/backends/arm/test/ops/test_add.py b/backends/arm/test/ops/test_add.py index 63023327f79..cff8af11654 100644 --- a/backends/arm/test/ops/test_add.py +++ b/backends/arm/test/ops/test_add.py @@ -13,6 +13,7 @@ from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester from executorch.exir import EdgeCompileConfig +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized @@ -92,16 +93,17 @@ def _test_add_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_add_u55_BI_pipeline( + def _test_add_ethos_BI_pipeline( self, module: torch.nn.Module, + compile_spec: CompileSpec, test_data: Tuple[torch.Tensor], ): tester = ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True), + compile_spec=compile_spec, ) .quantize() .export() @@ -114,8 +116,7 @@ def _test_add_u55_BI_pipeline( .serialize() ) - if common.is_option_enabled("corstone300"): - tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) + return tester @parameterized.expand(Add.test_parameters) def test_add_tosa_MI(self, test_data: torch.Tensor): @@ -130,7 +131,22 @@ def test_add_tosa_BI(self, test_data: torch.Tensor): @parameterized.expand(Add.test_parameters) def test_add_u55_BI(self, test_data: torch.Tensor): test_data = (test_data,) - self._test_add_u55_BI_pipeline(self.Add(), test_data) + tester = self._test_add_ethos_BI_pipeline( + self.Add(), + common.get_u55_compile_spec(permute_memory_to_nhwc=True), + test_data, + ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) + + @parameterized.expand(Add.test_parameters) + def test_add_u85_BI(self, test_data: torch.Tensor): + test_data = (test_data,) + self._test_add_ethos_BI_pipeline( + self.Add(), + common.get_u85_compile_spec(permute_memory_to_nhwc=True), + test_data, + ) @parameterized.expand(Add2.test_parameters) def test_add2_tosa_MI(self, operand1: torch.Tensor, operand2: torch.Tensor): @@ -145,4 +161,15 @@ def test_add2_tosa_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): @parameterized.expand(Add2.test_parameters) def test_add2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) - self._test_add_u55_BI_pipeline(self.Add2(), test_data) + tester = self._test_add_ethos_BI_pipeline( + self.Add2(), common.get_u55_compile_spec(), test_data + ) + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) + + @parameterized.expand(Add2.test_parameters) + def test_add2_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): + test_data = (operand1, operand2) + self._test_add_ethos_BI_pipeline( + self.Add2(), common.get_u85_compile_spec(), test_data + ) diff --git a/backends/arm/test/ops/test_avg_pool.py b/backends/arm/test/ops/test_avg_pool.py index 32a0e5555a3..6c14420dbcf 100644 --- a/backends/arm/test/ops/test_avg_pool.py +++ b/backends/arm/test/ops/test_avg_pool.py @@ -13,6 +13,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.backend_details import CompileSpec from parameterized import parameterized logger = logging.getLogger(__name__) @@ -86,14 +87,17 @@ def _test_avgpool2d_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_avgpool2d_tosa_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + def _test_avgpool2d_tosa_ethos_BI_pipeline( + self, + module: torch.nn.Module, + compile_spec: CompileSpec, + test_data: Tuple[torch.tensor], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True), + compile_spec=compile_spec, ) .quantize() .export() @@ -141,6 +145,22 @@ def test_avgpool2d_tosa_u55_BI( test_data: torch.Tensor, model_params: int | Tuple[int, int], ): - self._test_avgpool2d_tosa_u55_BI_pipeline( - self.AvgPool2d(*model_params), (test_data,) + self._test_avgpool2d_tosa_ethos_BI_pipeline( + self.AvgPool2d(*model_params), + common.get_u55_compile_spec(permute_memory_to_nhwc=True), + (test_data,), + ) + + @parameterized.expand(test_data_suite) + @unittest.expectedFailure + def test_avgpool2d_tosa_u85_BI( + self, + test_name: str, + test_data: torch.Tensor, + model_params: int | Tuple[int, int], + ): + self._test_avgpool2d_tosa_ethos_BI_pipeline( + self.AvgPool2d(*model_params), + common.get_u85_compile_spec(permute_memory_to_nhwc=True), + (test_data,), ) diff --git a/backends/arm/test/ops/test_bmm.py b/backends/arm/test/ops/test_bmm.py index 30f45261247..e4e6abb7bb3 100644 --- a/backends/arm/test/ops/test_bmm.py +++ b/backends/arm/test/ops/test_bmm.py @@ -11,6 +11,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized torch.manual_seed(1) @@ -83,14 +84,17 @@ def _test_bmm_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data) ) - def _test_bmm_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor, ...] + def _test_bmm_ethosu_BI_pipeline( + self, + module: torch.nn.Module, + compile_spec: CompileSpec, + test_data: Tuple[torch.Tensor, ...], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize() .export() @@ -132,4 +136,13 @@ def test_bmm_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): @unittest.expectedFailure def test_bmm_single_input_u55_BI(self, operand1: torch.Tensor): test_data = (operand1,) - self._test_bmm_u55_BI_pipeline(self.BMMSingleInput(), test_data) + self._test_bmm_ethosu_BI_pipeline( + self.BMMSingleInput(), common.get_u55_compile_spec(), test_data + ) + + @parameterized.expand(BMMSingleInput.test_parameters) + def test_bmm_single_input_u85_BI(self, operand1: torch.Tensor): + test_data = (operand1,) + self._test_bmm_ethosu_BI_pipeline( + self.BMMSingleInput(), common.get_u85_compile_spec(), test_data + ) diff --git a/backends/arm/test/ops/test_cat.py b/backends/arm/test/ops/test_cat.py index a40ae43b673..9723ba0f0c0 100644 --- a/backends/arm/test/ops/test_cat.py +++ b/backends/arm/test/ops/test_cat.py @@ -13,6 +13,7 @@ from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized @@ -89,14 +90,17 @@ def _test_cat_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_cat_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[tuple[torch.Tensor, ...], int] + def _test_cat_ethosu_BI_pipeline( + self, + module: torch.nn.Module, + compile_spec: CompileSpec, + test_data: Tuple[tuple[torch.Tensor, ...], int], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize() .export() @@ -125,9 +129,16 @@ def test_cat_tosa_BI(self, operands: tuple[torch.Tensor, ...], dim: int): test_data = (operands, dim) self._test_cat_tosa_BI_pipeline(self.Cat(), test_data) - # TODO: Remove @unittest.expectedFailure when this issue is fixed in Regor @parameterized.expand(Cat.test_parameters) - @unittest.expectedFailure def test_cat_u55_BI(self, operands: tuple[torch.Tensor, ...], dim: int): test_data = (operands, dim) - self._test_cat_u55_BI_pipeline(self.Cat(), test_data) + self._test_cat_ethosu_BI_pipeline( + self.Cat(), common.get_u55_compile_spec(), test_data + ) + + @parameterized.expand(Cat.test_parameters) + def test_cat_u85_BI(self, operands: tuple[torch.Tensor, ...], dim: int): + test_data = (operands, dim) + self._test_cat_ethosu_BI_pipeline( + self.Cat(), common.get_u85_compile_spec(), test_data + ) diff --git a/backends/arm/test/ops/test_clone.py b/backends/arm/test/ops/test_clone.py index 8386283f24e..9852c5c4520 100644 --- a/backends/arm/test/ops/test_clone.py +++ b/backends/arm/test/ops/test_clone.py @@ -21,6 +21,8 @@ from executorch.backends.arm.test.tester.arm_tester import ArmTester from executorch.backends.xnnpack.test.tester.tester import Quantize + +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized @@ -76,16 +78,15 @@ def _test_clone_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_clone_tosa_u55_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + def _test_clone_tosa_ethos_pipeline( + self, + compile_spec: list[CompileSpec], + module: torch.nn.Module, + test_data: Tuple[torch.Tensor], ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) ( - ArmTester( - module, - example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), - ) + ArmTester(module, example_inputs=test_data, compile_spec=compile_spec) .quantize(Quantize(quantizer, get_symmetric_quantization_config())) .export() .check_count({"torch.ops.aten.clone.default": 1}) @@ -95,6 +96,20 @@ def _test_clone_tosa_u55_pipeline( .to_executorch() ) + def _test_clone_tosa_u55_pipeline( + self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + ): + self._test_clone_tosa_ethos_pipeline( + common.get_u55_compile_spec(), module, test_data + ) + + def _test_clone_tosa_u85_pipeline( + self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + ): + self._test_clone_tosa_ethos_pipeline( + common.get_u85_compile_spec(), module, test_data + ) + @parameterized.expand(Clone.test_parameters) def test_clone_tosa_MI(self, test_tensor: torch.Tensor): self._test_clone_tosa_MI_pipeline(self.Clone(), (test_tensor,)) @@ -106,3 +121,7 @@ def test_clone_tosa_BI(self, test_tensor: torch.Tensor): @parameterized.expand(Clone.test_parameters) def test_clone_u55_BI(self, test_tensor: torch.Tensor): self._test_clone_tosa_u55_pipeline(self.Clone(), (test_tensor,)) + + @parameterized.expand(Clone.test_parameters) + def test_clone_u85_BI(self, test_tensor: torch.Tensor): + self._test_clone_tosa_u85_pipeline(self.Clone(), (test_tensor,)) diff --git a/backends/arm/test/ops/test_conv.py b/backends/arm/test/ops/test_conv.py index 82748799533..286404922f2 100644 --- a/backends/arm/test/ops/test_conv.py +++ b/backends/arm/test/ops/test_conv.py @@ -12,6 +12,7 @@ from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized @@ -297,14 +298,17 @@ def _test_conv2d_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_conv2d_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + def _test_conv2d_ethosu_BI_pipeline( + self, + compile_spec: CompileSpec, + module: torch.nn.Module, + test_data: Tuple[torch.Tensor], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True), + compile_spec=compile_spec, ) .quantize() .export() @@ -325,4 +329,16 @@ def test_conv2d_tosa_BI(self, test_name, model): @parameterized.expand(testsuite_u55) def test_conv2d_u55_BI(self, test_name, model): - self._test_conv2d_u55_BI_pipeline(model, model.get_inputs()) + self._test_conv2d_ethosu_BI_pipeline( + common.get_u55_compile_spec(permute_memory_to_nhwc=True), + model, + model.get_inputs(), + ) + + @parameterized.expand(testsuite_u55) + def test_conv2d_u85_BI(self, test_name, model): + self._test_conv2d_ethosu_BI_pipeline( + common.get_u85_compile_spec(permute_memory_to_nhwc=True), + model, + model.get_inputs(), + ) diff --git a/backends/arm/test/ops/test_conv_combos.py b/backends/arm/test/ops/test_conv_combos.py index 31051ef8f7d..1fe4f1b5a57 100644 --- a/backends/arm/test/ops/test_conv_combos.py +++ b/backends/arm/test/ops/test_conv_combos.py @@ -12,6 +12,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.backend_details import CompileSpec from parameterized import parameterized logger = logging.getLogger(__name__) @@ -199,14 +200,17 @@ def _test_conv_combo_tosa_BI_pipeline( ) ) - def _test_conv_combo_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + def _test_conv_combo_ethos_BI_pipeline( + self, + module: torch.nn.Module, + compile_spec: CompileSpec, + test_data: Tuple[torch.Tensor], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True), + compile_spec=compile_spec, ) .quantize() .export() @@ -230,7 +234,19 @@ def test_conv_meandim_tosa_BI(self): def test_conv_meandim_u55_BI(self): model = ComboConv2dMeandim() - self._test_conv_combo_u55_BI_pipeline(model, model.get_inputs()) + self._test_conv_combo_ethos_BI_pipeline( + model, + common.get_u55_compile_spec(permute_memory_to_nhwc=True), + model.get_inputs(), + ) + + def test_conv_meandim_u85_BI(self): + model = ComboConv2dMeandim() + self._test_conv_combo_ethos_BI_pipeline( + model, + common.get_u85_compile_spec(permute_memory_to_nhwc=True), + model.get_inputs(), + ) ############################## ## Conv + batch norm + relu ## @@ -245,7 +261,17 @@ def test_conv_batchnorm_relu6_tosa_BI(self): def test_conv_batchnorm_relu6_u55_BI(self): model = ComboConvBatchnormRelu6() - self._test_conv_combo_u55_BI_pipeline(model, model.get_inputs()) + self._test_conv_combo_ethos_BI_pipeline( + model, common.get_u55_compile_spec(), model.get_inputs() + ) + + def test_conv_batchnorm_relu_u85_BI(self): + model = ComboConvBatchnormRelu6() + self._test_conv_combo_ethos_BI_pipeline( + model, + common.get_u85_compile_spec(), + model.get_inputs(), + ) ################## ## Conv + ReLU6 ## @@ -266,7 +292,17 @@ def test_conv_relu6_tosa_BI(self, test_data: torch.Tensor): def test_conv_relu6_u55_BI(self, test_data: torch.Tensor): model = ComboConvRelu6() test_data = (test_data,) - self._test_conv_combo_u55_BI_pipeline(model, test_data) + self._test_conv_combo_ethos_BI_pipeline( + model, common.get_u55_compile_spec(permute_memory_to_nhwc=True), test_data + ) + + @parameterized.expand(ComboConvRelu6.test_data) + def test_conv_relu6_u85_BI(self, test_data: torch.Tensor): + model = ComboConvRelu6() + test_data = (test_data,) + self._test_conv_combo_ethos_BI_pipeline( + model, common.get_u85_compile_spec(permute_memory_to_nhwc=True), test_data + ) ############################### ## Block bottleneck residual ## @@ -281,4 +317,16 @@ def test_block_bottleneck_residual_tosa_BI(self): def test_block_bottleneck_residual_u55_BI(self): model = ComboBlockBottleneckResidual() - self._test_conv_combo_u55_BI_pipeline(model, model.get_inputs()) + self._test_conv_combo_ethos_BI_pipeline( + model, + common.get_u55_compile_spec(permute_memory_to_nhwc=True), + model.get_inputs(), + ) + + def test_block_bottleneck_residual_u85_BI(self): + model = ComboBlockBottleneckResidual() + self._test_conv_combo_ethos_BI_pipeline( + model, + common.get_u85_compile_spec(permute_memory_to_nhwc=True), + model.get_inputs(), + ) diff --git a/backends/arm/test/ops/test_depthwise_conv.py b/backends/arm/test/ops/test_depthwise_conv.py index 5545a5ce4a8..11b9e4876bb 100644 --- a/backends/arm/test/ops/test_depthwise_conv.py +++ b/backends/arm/test/ops/test_depthwise_conv.py @@ -16,6 +16,7 @@ from executorch.backends.arm.test.ops.test_conv import Conv2d from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.backend_details import CompileSpec from parameterized import parameterized logger = logging.getLogger(__name__) @@ -130,11 +131,6 @@ # Fails when enabling CompileSpec.set_quantize_io(True). MLETORCH-191. testsuite_u55.remove(("3x3_1x3x256x256_gp3_st1", dw_conv2d_3x3_1x3x256x256_gp3_st1)) -# Add failing test (set_quantize_io=True) temporarily to investigate -testsuite_u55.append( - ("3x3_1x3x256x256_gp3_st1", dw_conv2d_3x3_1x3x256x256_gp3_st1, True) -) - class TestDepthwiseConv2D(unittest.TestCase): """Tests Conv2D where groups == in_channels and out_channels = K * in_channels. This @@ -177,19 +173,17 @@ def _test_dw_conv2d_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_dw_conv2d_u55_BI_pipeline( + def _test_dw_conv2d_ethos_BI_pipeline( self, module: torch.nn.Module, + compile_spec: CompileSpec, test_data: Tuple[torch.Tensor], - set_quantize_io: bool = False, ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec( - permute_memory_to_nhwc=True, quantize_io=set_quantize_io - ), + compile_spec=compile_spec, ) .quantize() .export() @@ -201,18 +195,35 @@ def _test_dw_conv2d_u55_BI_pipeline( ) @parameterized.expand(testsuite) - def test_dw_conv2d_tosa_MI(self, test_name, model): + def test_dw_conv2d_tosa_MI(self, test_name: str, model: torch.nn.Module): self._test_dw_conv2d_tosa_MI_pipeline(model, model.get_inputs()) # TODO: Investigate flakyness (MLTORCH-307) @parameterized.expand(testsuite) @pytest.mark.flaky(reruns=3) - def test_dw_conv2d_tosa_BI(self, test_name, model): + def test_dw_conv2d_tosa_BI(self, test_name: str, model: torch.nn.Module): self._test_dw_conv2d_tosa_BI_pipeline(model, model.get_inputs()) @parameterized.expand(testsuite_u55, skip_on_empty=True) - @unittest.expectedFailure - def test_dw_conv2d_u55_BI(self, test_name, model, set_quantize_io=False): - self._test_dw_conv2d_u55_BI_pipeline( - model, model.get_inputs(), set_quantize_io=set_quantize_io + def test_dw_conv2d_u55_BI( + self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False + ): + self._test_dw_conv2d_ethos_BI_pipeline( + model, + common.get_u55_compile_spec( + permute_memory_to_nhwc=True, quantize_io=set_quantize_io + ), + model.get_inputs(), + ) + + @parameterized.expand(testsuite) + def test_dw_conv2d_u85_BI( + self, test_name: str, model: torch.nn.Module, set_quantize_io: bool = False + ): + self._test_dw_conv2d_ethos_BI_pipeline( + model, + common.get_u85_compile_spec( + permute_memory_to_nhwc=True, quantize_io=set_quantize_io + ), + model.get_inputs(), ) diff --git a/backends/arm/test/ops/test_exp.py b/backends/arm/test/ops/test_exp.py index 4f4935d482c..6e85d8fe49b 100644 --- a/backends/arm/test/ops/test_exp.py +++ b/backends/arm/test/ops/test_exp.py @@ -12,6 +12,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.backend_details import CompileSpec from parameterized import parameterized test_data_suite = [ @@ -71,8 +72,11 @@ def _test_exp_tosa_BI_pipeline(self, module: torch.nn.Module, test_data: Tuple): .run_method_and_compare_outputs(inputs=test_data) ) - def _test_exp_tosa_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + def _test_exp_ethosu_BI_pipeline( + self, + compile_spec: CompileSpec, + module: torch.nn.Module, + test_data: Tuple[torch.tensor], ): ( ArmTester( @@ -105,4 +109,12 @@ def test_exp_tosa_BI(self, test_name: str, test_data: torch.Tensor): @parameterized.expand(test_data_suite) def test_exp_tosa_u55_BI(self, test_name: str, test_data: torch.Tensor): - self._test_exp_tosa_u55_BI_pipeline(self.Exp(), (test_data,)) + self._test_exp_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.Exp(), (test_data,) + ) + + @parameterized.expand(test_data_suite) + def test_exp_tosa_u85_BI(self, test_name: str, test_data: torch.Tensor): + self._test_exp_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.Exp(), (test_data,) + ) diff --git a/backends/arm/test/ops/test_expand.py b/backends/arm/test/ops/test_expand.py index 66c081a544c..e9bbea9a5e5 100644 --- a/backends/arm/test/ops/test_expand.py +++ b/backends/arm/test/ops/test_expand.py @@ -76,7 +76,9 @@ def _test_expand_tosa_BI_pipeline(self, module: torch.nn.Module, test_data: Tupl .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_expand_tosa_u55_pipeline(self, module: torch.nn.Module, test_data: Tuple): + def _test_expand_ethosu_BI_pipeline( + self, module: torch.nn.Module, test_data: Tuple + ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) ( ArmTester( @@ -104,6 +106,15 @@ def test_expand_tosa_BI(self, test_input, multiples): # Expected failure since tosa.TILE is unsupported by Vela. @parameterized.expand(Expand.test_parameters) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: MLBEDSW-9386 def test_expand_u55_BI(self, test_input, multiples): - self._test_expand_tosa_u55_pipeline(self.Expand(), (test_input, multiples)) + self._test_expand_ethosu_BI_pipeline( + self.Expand(), common.get_u55_compile_spec(), (test_input, multiples) + ) + + @parameterized.expand(Expand.test_parameters) + @unittest.expectedFailure # TODO: MLBEDSW-9386 + def test_expand_u85_BI(self, test_input, multiples): + self._test_expand_ethosu_BI_pipeline( + self.Expand(), common.get_u85_compile_spec(), (test_input, multiples) + ) diff --git a/backends/arm/test/ops/test_full.py b/backends/arm/test/ops/test_full.py index 1be7f59ab8f..2722edef328 100644 --- a/backends/arm/test/ops/test_full.py +++ b/backends/arm/test/ops/test_full.py @@ -15,6 +15,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized @@ -93,13 +94,11 @@ def _test_full_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data) ) - def _test_full_tosa_u55_pipeline(self, module: torch.nn.Module, test_data: Tuple): + def _test_full_tosa_ethos_pipeline( + self, compile_spec: list[CompileSpec], module: torch.nn.Module, test_data: Tuple + ): ( - ArmTester( - module, - example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), - ) + ArmTester(module, example_inputs=test_data, compile_spec=compile_spec) .quantize() .export() .check_count({"torch.ops.aten.full.default": 1}) @@ -110,6 +109,16 @@ def _test_full_tosa_u55_pipeline(self, module: torch.nn.Module, test_data: Tuple .to_executorch() ) + def _test_full_tosa_u55_pipeline(self, module: torch.nn.Module, test_data: Tuple): + self._test_full_tosa_ethos_pipeline( + common.get_u55_compile_spec(), module, test_data + ) + + def _test_full_tosa_u85_pipeline(self, module: torch.nn.Module, test_data: Tuple): + self._test_full_tosa_ethos_pipeline( + common.get_u85_compile_spec(), module, test_data + ) + def test_only_full_tosa_MI(self): self._test_full_tosa_MI_pipeline(self.Full(), ()) @@ -138,6 +147,13 @@ def test_full_u55_BI(self, test_tensor: Tuple): test_tensor, ) + @parameterized.expand(AddVariableFull.test_parameters) + def test_full_u85_BI(self, test_tensor: Tuple): + self._test_full_tosa_u85_pipeline( + self.AddVariableFull(), + test_tensor, + ) + # This fails since full outputs int64 by default if 'fill_value' is integer, which our backend doesn't support. @unittest.expectedFailure def test_integer_value(self): diff --git a/backends/arm/test/ops/test_linear.py b/backends/arm/test/ops/test_linear.py index 6fdbb2127e0..3f68ab0251a 100644 --- a/backends/arm/test/ops/test_linear.py +++ b/backends/arm/test/ops/test_linear.py @@ -15,6 +15,7 @@ from executorch.backends.arm.test.tester.arm_tester import ArmTester from executorch.exir import EdgeCompileConfig +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized logger = logging.getLogger(__name__) @@ -153,14 +154,17 @@ def _test_linear_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=True) ) - def _test_linear_tosa_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] - ): + def _test_linear_tosa_ethosu_BI_pipeline( + self, + module: torch.nn.Module, + compile_spec: CompileSpec, + test_data: Tuple[torch.Tensor], + ) -> ArmTester: tester = ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=False), + compile_spec=compile_spec, ) .quantize() .export() @@ -172,9 +176,7 @@ def _test_linear_tosa_u55_BI_pipeline( .to_executorch() .serialize() ) - - if common.is_option_enabled("corstone300"): - tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) + return tester @parameterized.expand(test_data_suite_rank1 + test_data_suite_rank4) def test_linear_tosa_MI( @@ -215,10 +217,32 @@ def test_linear_tosa_u55_BI( ): in_features = test_data.shape[-1] test_data = (test_data,) - self._test_linear_tosa_u55_BI_pipeline( + tester = self._test_linear_tosa_ethosu_BI_pipeline( + self.Linear( + in_features=in_features, + out_features=out_features, + ), + common.get_u55_compile_spec(permute_memory_to_nhwc=False), + test_data, + ) + + if common.is_option_enabled("corstone300"): + tester.run_method_and_compare_outputs(qtol=1, inputs=test_data) + + @parameterized.expand(test_data_suite_rank1) + def test_linear_tosa_u85_BI( + self, + test_name: str, + test_data: torch.Tensor, + out_features: int, + ): + in_features = test_data.shape[-1] + test_data = (test_data,) + self._test_linear_tosa_ethosu_BI_pipeline( self.Linear( in_features=in_features, out_features=out_features, ), + common.get_u85_compile_spec(permute_memory_to_nhwc=False), test_data, ) diff --git a/backends/arm/test/ops/test_log.py b/backends/arm/test/ops/test_log.py index 90066b3a63b..269b7be25f5 100644 --- a/backends/arm/test/ops/test_log.py +++ b/backends/arm/test/ops/test_log.py @@ -12,6 +12,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.backend_details import CompileSpec from parameterized import parameterized test_data_suite = [ @@ -71,14 +72,17 @@ def _test_log_tosa_BI_pipeline(self, module: torch.nn.Module, test_data: Tuple): .run_method_and_compare_outputs(inputs=test_data) ) - def _test_log_tosa_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + def _test_log_ethosu_BI_pipeline( + self, + compile_spec: CompileSpec, + module: torch.nn.Module, + test_data: Tuple[torch.tensor], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize() .export() @@ -105,4 +109,12 @@ def test_log_tosa_BI(self, test_name: str, test_data: torch.Tensor): @parameterized.expand(test_data_suite) def test_log_tosa_u55_BI(self, test_name: str, test_data: torch.Tensor): - self._test_log_tosa_u55_BI_pipeline(self.Log(), (test_data,)) + self._test_log_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.Log(), (test_data,) + ) + + @parameterized.expand(test_data_suite) + def test_log_tosa_u85_BI(self, test_name: str, test_data: torch.Tensor): + self._test_log_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.Log(), (test_data,) + ) diff --git a/backends/arm/test/ops/test_mean_dim.py b/backends/arm/test/ops/test_mean_dim.py index e48d749c194..0653e84e704 100644 --- a/backends/arm/test/ops/test_mean_dim.py +++ b/backends/arm/test/ops/test_mean_dim.py @@ -13,6 +13,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.backend_details import CompileSpec from parameterized import parameterized logger = logging.getLogger(__name__) @@ -91,14 +92,17 @@ def _test_meandim_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_meandim_tosa_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + def _test_meandim_tosa_ethosu_BI_pipeline( + self, + module: torch.nn.Module, + compile_spec: CompileSpec, + test_data: Tuple[torch.tensor], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize() .export() @@ -138,4 +142,20 @@ def test_meandim_tosa_u55_BI( test_name: str, test_data: torch.Tensor, ): - self._test_meandim_tosa_u55_BI_pipeline(self.MeanDim(), (test_data,)) + self._test_meandim_tosa_ethosu_BI_pipeline( + self.MeanDim(), + common.get_u55_compile_spec(), + (test_data,), + ) + + @parameterized.expand(test_data_suite) + def test_meandim_tosa_u85_BI( + self, + test_name: str, + test_data: torch.Tensor, + ): + self._test_meandim_tosa_ethosu_BI_pipeline( + self.MeanDim(), + common.get_u85_compile_spec(), + (test_data,), + ) diff --git a/backends/arm/test/ops/test_mm.py b/backends/arm/test/ops/test_mm.py index 9a9b3ef579b..4271496eaa9 100644 --- a/backends/arm/test/ops/test_mm.py +++ b/backends/arm/test/ops/test_mm.py @@ -12,6 +12,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.backend_details import CompileSpec from parameterized import parameterized logger = logging.getLogger(__name__) @@ -87,14 +88,17 @@ def _test_mm_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data) ) - def _test_mm_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + def _test_mm_ethosu_BI_pipeline( + self, + compile_spec: CompileSpec, + module: torch.nn.Module, + test_data: Tuple[torch.Tensor], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize() .export() @@ -131,11 +135,29 @@ def test_mm_single_input_tosa_BI(self, operand1: torch.Tensor): @unittest.expectedFailure def test_mm_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) - self._test_mm_u55_BI_pipeline(self.MM(), test_data) + self._test_mm_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.MM(), test_data + ) # Expected to fail with error: Warning, unsupported fusing of TOSA Rescale previous operator is of type: Memcpy @parameterized.expand(MMSingleInput.test_parameters) @unittest.expectedFailure def test_mm_single_input_u55_BI(self, operand1: torch.Tensor): test_data = (operand1,) - self._test_mm_u55_BI_pipeline(self.MMSingleInput(), test_data) + self._test_mm_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.MMSingleInput(), test_data + ) + + @parameterized.expand(MM.test_parameters) + def test_mm_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): + test_data = (operand1, operand2) + self._test_mm_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.MM(), test_data + ) + + @parameterized.expand(MMSingleInput.test_parameters) + def test_mm_single_input_u85_BI(self, operand1: torch.Tensor): + test_data = (operand1,) + self._test_mm_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.MMSingleInput(), test_data + ) diff --git a/backends/arm/test/ops/test_mul.py b/backends/arm/test/ops/test_mul.py index 2aac3f22f13..a1c2dba5fed 100644 --- a/backends/arm/test/ops/test_mul.py +++ b/backends/arm/test/ops/test_mul.py @@ -10,6 +10,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.backend_details import CompileSpec from parameterized import parameterized test_data_sute = [ @@ -101,14 +102,17 @@ def _test_mul_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1.0) ) - def _test_mul_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: tuple[torch.Tensor, torch.Tensor] + def _test_mul_ethosu_BI_pipeline( + self, + compile_spec: CompileSpec, + module: torch.nn.Module, + test_data: tuple[torch.Tensor, torch.Tensor], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(permute_memory_to_nhwc=True), + compile_spec=compile_spec, ) .quantize() .export() @@ -149,4 +153,18 @@ def test_mul_u55_BI( other_: torch.Tensor, ): test_data = (input_, other_) - self._test_mul_u55_BI_pipeline(self.Mul(), test_data) + self._test_mul_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.Mul(), test_data + ) + + @parameterized.expand(test_data_sute) + def test_mul_u85_BI( + self, + test_name: str, + input_: torch.Tensor, + other_: torch.Tensor, + ): + test_data = (input_, other_) + self._test_mul_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.Mul(), test_data + ) diff --git a/backends/arm/test/ops/test_relu.py b/backends/arm/test/ops/test_relu.py index d2ca8540f4c..effbccc74d5 100644 --- a/backends/arm/test/ops/test_relu.py +++ b/backends/arm/test/ops/test_relu.py @@ -17,6 +17,7 @@ from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester from executorch.backends.xnnpack.test.tester.tester import Quantize +from executorch.exir.backend.backend_details import CompileSpec from parameterized import parameterized @@ -82,15 +83,18 @@ def _test_relu_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data) ) - def _test_relu_tosa_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + def _test_relu_ethosu_BI_pipeline( + self, + compile_spec: CompileSpec, + module: torch.nn.Module, + test_data: Tuple[torch.tensor], ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize(Quantize(quantizer, get_symmetric_quantization_config())) .export() @@ -116,5 +120,13 @@ def test_relu_tosa_BI(self, test_name: str, test_data: torch.Tensor): self._test_relu_tosa_BI_pipeline(self.Relu(), (test_data,)) @parameterized.expand(test_data_suite) - def test_relu_tosa_u55_BI(self, test_name: str, test_data: torch.Tensor): - self._test_relu_tosa_u55_BI_pipeline(self.Relu(), (test_data,)) + def test_relu_u55_BI(self, test_name: str, test_data: torch.Tensor): + self._test_relu_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.Relu(), (test_data,) + ) + + @parameterized.expand(test_data_suite) + def test_relu_u85_BI(self, test_name: str, test_data: torch.Tensor): + self._test_relu_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.Relu(), (test_data,) + ) diff --git a/backends/arm/test/ops/test_repeat.py b/backends/arm/test/ops/test_repeat.py index a6fad033456..542f0d6256b 100644 --- a/backends/arm/test/ops/test_repeat.py +++ b/backends/arm/test/ops/test_repeat.py @@ -21,6 +21,7 @@ from executorch.backends.arm.test.tester.arm_tester import ArmTester from executorch.backends.xnnpack.test.tester.tester import Quantize +from executorch.exir.backend.backend_details import CompileSpec from parameterized import parameterized @@ -77,13 +78,15 @@ def _test_repeat_tosa_BI_pipeline(self, module: torch.nn.Module, test_data: Tupl .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_repeat_tosa_u55_pipeline(self, module: torch.nn.Module, test_data: Tuple): + def _test_repeat_ethosu_pipeline( + self, compile_spec: CompileSpec, module: torch.nn.Module, test_data: Tuple + ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize(Quantize(quantizer, get_symmetric_quantization_config())) .export() @@ -103,8 +106,16 @@ def test_repeat_tosa_MI(self, test_input, multiples): def test_repeat_tosa_BI(self, test_input, multiples): self._test_repeat_tosa_BI_pipeline(self.Repeat(), (test_input, multiples)) - # Expected failure since tosa.TILE is unsupported by Vela. @parameterized.expand(Repeat.test_parameters) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: MLBEDSW-9386 def test_repeat_u55_BI(self, test_input, multiples): - self._test_repeat_tosa_u55_pipeline(self.Repeat(), (test_input, multiples)) + self._test_repeat_ethosu_pipeline( + common.get_u55_compile_spec(), self.Repeat(), (test_input, multiples) + ) + + @parameterized.expand(Repeat.test_parameters) + @unittest.expectedFailure # TODO: MLBEDSW-9386 + def test_repeat_u85_BI(self, test_input, multiples): + self._test_repeat_ethosu_pipeline( + common.get_u85_compile_spec(), self.Repeat(), (test_input, multiples) + ) diff --git a/backends/arm/test/ops/test_sigmoid.py b/backends/arm/test/ops/test_sigmoid.py index 369019774fa..f75583164c1 100644 --- a/backends/arm/test/ops/test_sigmoid.py +++ b/backends/arm/test/ops/test_sigmoid.py @@ -13,6 +13,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized logger = logging.getLogger(__name__) @@ -102,14 +103,17 @@ def _test_sigmoid_tosa_BI_pipeline(self, module: torch.nn.Module, test_data: Tup .run_method_and_compare_outputs(inputs=test_data) ) - def _test_sigmoid_tosa_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + def _test_sigmoid_tosa_ethos_BI_pipeline( + self, + compile_spec: list[CompileSpec], + module: torch.nn.Module, + test_data: Tuple[torch.tensor], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize() .export() @@ -122,6 +126,20 @@ def _test_sigmoid_tosa_u55_BI_pipeline( .to_executorch() ) + def _test_sigmoid_tosa_u55_BI_pipeline( + self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + ): + self._test_sigmoid_tosa_ethos_BI_pipeline( + common.get_u55_compile_spec(), module, test_data + ) + + def _test_sigmoid_tosa_u85_BI_pipeline( + self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + ): + self._test_sigmoid_tosa_ethos_BI_pipeline( + common.get_u85_compile_spec(), module, test_data + ) + @parameterized.expand(test_data_suite) def test_sigmoid_tosa_MI( self, @@ -148,3 +166,7 @@ def test_sigmoid_add_sigmoid_tosa_BI(self): @parameterized.expand(test_data_suite) def test_sigmoid_tosa_u55_BI(self, test_name: str, test_data: torch.Tensor): self._test_sigmoid_tosa_u55_BI_pipeline(self.Sigmoid(), (test_data,)) + + @parameterized.expand(test_data_suite) + def test_sigmoid_tosa_u85_BI(self, test_name: str, test_data: torch.Tensor): + self._test_sigmoid_tosa_u85_BI_pipeline(self.Sigmoid(), (test_data,)) diff --git a/backends/arm/test/ops/test_slice.py b/backends/arm/test/ops/test_slice.py index 14874df156e..ca026c7f420 100644 --- a/backends/arm/test/ops/test_slice.py +++ b/backends/arm/test/ops/test_slice.py @@ -15,6 +15,7 @@ from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester from executorch.backends.xnnpack.test.tester.tester import Quantize +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized @@ -77,8 +78,11 @@ def _test_slice_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_slice_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + def _test_slice_ethos_BI_pipeline( + self, + compile_spec: list[CompileSpec], + module: torch.nn.Module, + test_data: Tuple[torch.Tensor], ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) ( @@ -96,6 +100,20 @@ def _test_slice_u55_BI_pipeline( .to_executorch() ) + def _test_slice_u55_BI_pipeline( + self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + ): + self._test_slice_ethos_BI_pipeline( + common.get_u55_compile_spec(), module, test_data + ) + + def _test_slice_u85_BI_pipeline( + self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + ): + self._test_slice_ethos_BI_pipeline( + common.get_u85_compile_spec(), module, test_data + ) + @parameterized.expand(Slice.test_tensors) def test_slice_tosa_MI(self, tensor): self._test_slice_tosa_MI_pipeline(self.Slice(), (tensor,)) @@ -108,9 +126,10 @@ def test_slice_nchw_tosa_BI(self, test_tensor: torch.Tensor): def test_slice_nhwc_tosa_BI(self, test_tensor: torch.Tensor): self._test_slice_tosa_BI_pipeline(self.Slice(), (test_tensor,), True) - # Fails during Vela compilation when trying to use a Tuple as a Named tuple, - # Could be Vela Issue, wait until Regor. @parameterized.expand(Slice.test_tensors) - @unittest.expectedFailure def test_slice_u55_BI(self, test_tensor: torch.Tensor): self._test_slice_u55_BI_pipeline(self.Slice(), (test_tensor,)) + + @parameterized.expand(Slice.test_tensors) + def test_slice_u85_BI(self, test_tensor: torch.Tensor): + self._test_slice_u85_BI_pipeline(self.Slice(), (test_tensor,)) diff --git a/backends/arm/test/ops/test_softmax.py b/backends/arm/test/ops/test_softmax.py index 20da65b687f..a7d25d266de 100644 --- a/backends/arm/test/ops/test_softmax.py +++ b/backends/arm/test/ops/test_softmax.py @@ -12,6 +12,7 @@ import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized @@ -80,14 +81,17 @@ def _test_softmax_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_softmax_tosa_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + def _test_softmax_tosa_ethos_BI_pipeline( + self, + compile_spec: list[CompileSpec], + module: torch.nn.Module, + test_data: Tuple[torch.tensor], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize() .export() @@ -100,6 +104,20 @@ def _test_softmax_tosa_u55_BI_pipeline( .to_executorch() ) + def _test_softmax_tosa_u55_BI_pipeline( + self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + ): + self._test_softmax_tosa_ethos_BI_pipeline( + common.get_u55_compile_spec(), module, test_data + ) + + def _test_softmax_tosa_u85_BI_pipeline( + self, module: torch.nn.Module, test_data: Tuple[torch.tensor] + ): + self._test_softmax_tosa_ethos_BI_pipeline( + common.get_u85_compile_spec(), module, test_data + ) + @parameterized.expand(test_data_suite) def test_softmax_tosa_MI( self, @@ -132,3 +150,13 @@ def test_softmax_tosa_u55_BI( dim: int, ): self._test_softmax_tosa_u55_BI_pipeline(self.Softmax(dim=dim), (test_data,)) + + @parameterized.expand(test_data_suite) + @unittest.expectedFailure + def test_softmax_tosa_u85_BI( + self, + test_name: str, + test_data: torch.Tensor, + dim: int, + ): + self._test_softmax_tosa_u85_BI_pipeline(self.Softmax(dim=dim), (test_data,)) diff --git a/backends/arm/test/ops/test_split.py b/backends/arm/test/ops/test_split.py index bc998179c0c..02133d4e7f4 100644 --- a/backends/arm/test/ops/test_split.py +++ b/backends/arm/test/ops/test_split.py @@ -14,6 +14,7 @@ from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester from executorch.backends.xnnpack.test.tester.tester import Quantize +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized test_data_t = tuple[torch.Tensor, int | list[int], int] @@ -94,15 +95,15 @@ def _test_split_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_split_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: test_data_t + def _test_split_ethosu_BI_pipeline( + self, compile_spec: CompileSpec, module: torch.nn.Module, test_data: test_data_t ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize(Quantize(quantizer, get_symmetric_quantization_config())) .export() @@ -131,9 +132,33 @@ def test_split_n_out_tosa_MI(self, test_data: test_data_t): def test_split_tosa_BI(self, test_data: test_data_t): self._test_split_tosa_BI_pipeline(self.Split(), test_data) - # Fails during Vela compilation when trying to use a Tuple as a Named tuple, - # Could be Vela Issue, wait until Regor. - @parameterized.expand(Split.test_data) - @unittest.expectedFailure + @parameterized.expand( + [Split.test_data[0], Split.test_data[1], Split.test_data[2], Split.test_data[4]] + ) def test_split_u55_BI(self, test_data: test_data_t): - self._test_split_u55_BI_pipeline(self.Split(), test_data) + self._test_split_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.Split(), test_data + ) + + # TODO MLETORCH-350 + @parameterized.expand([Split.test_data[3], Split.test_data[5]]) + @unittest.expectedFailure + def test_split_u55_BI_skip(self, test_data: test_data_t): + self._test_split_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.Split(), test_data + ) + + @parameterized.expand( + [Split.test_data[0], Split.test_data[1], Split.test_data[2], Split.test_data[4]] + ) + def test_split_u85_BI(self, test_data: test_data_t): + self._test_split_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.Split(), test_data + ) + + @parameterized.expand([Split.test_data[3], Split.test_data[5]]) + @unittest.expectedFailure + def test_split_u85_BI_skip(self, test_data: test_data_t): + self._test_split_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.Split(), test_data + ) diff --git a/backends/arm/test/ops/test_sub.py b/backends/arm/test/ops/test_sub.py index 0a9f159f365..e80c0436989 100644 --- a/backends/arm/test/ops/test_sub.py +++ b/backends/arm/test/ops/test_sub.py @@ -13,6 +13,7 @@ from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.arm_tester import ArmTester +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized @@ -75,14 +76,17 @@ def _test_sub_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_sub_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + def _test_sub_ethosu_BI_pipeline( + self, + compile_spec: list[CompileSpec], + module: torch.nn.Module, + test_data: Tuple[torch.Tensor], ): ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize() .export() @@ -107,9 +111,37 @@ def test_sub_tosa_BI(self, test_data: torch.Tensor): @parameterized.expand(Sub.test_parameters) def test_sub_u55_BI(self, test_data: torch.Tensor): test_data = (test_data,) - self._test_sub_u55_BI_pipeline(self.Sub(), test_data) + self._test_sub_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.Sub(), test_data + ) + + @parameterized.expand(Sub.test_parameters) + def test_sub_u85_BI(self, test_data: torch.Tensor): + test_data = (test_data,) + self._test_sub_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.Sub(), test_data + ) @parameterized.expand(Sub2.test_parameters) def test_sub2_tosa_MI(self, operand1: torch.Tensor, operand2: torch.Tensor): test_data = (operand1, operand2) self._test_sub_tosa_MI_pipeline(self.Sub2(), test_data) + + @parameterized.expand(Sub2.test_parameters) + def test_sub2_tosa_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): + test_data = (operand1, operand2) + self._test_sub_tosa_BI_pipeline(self.Sub2(), test_data) + + @parameterized.expand(Sub2.test_parameters) + def test_sub2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): + test_data = (operand1, operand2) + self._test_sub_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.Sub2(), test_data + ) + + @parameterized.expand(Sub2.test_parameters) + def test_sub2_u85_BI(self, operand1: torch.Tensor, operand2: torch.Tensor): + test_data = (operand1, operand2) + self._test_sub_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.Sub2(), test_data + ) diff --git a/backends/arm/test/ops/test_unsqueeze.py b/backends/arm/test/ops/test_unsqueeze.py index 6da6a196c07..9c79d4371c3 100644 --- a/backends/arm/test/ops/test_unsqueeze.py +++ b/backends/arm/test/ops/test_unsqueeze.py @@ -21,6 +21,7 @@ from executorch.backends.arm.test.tester.arm_tester import ArmTester from executorch.backends.xnnpack.test.tester.tester import Quantize +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized @@ -70,15 +71,18 @@ def _test_unsqueeze_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_unsqueeze_tosa_u55_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor, int] + def _test_unsqueeze_ethosu_BI_pipeline( + self, + compile_spec: CompileSpec, + module: torch.nn.Module, + test_data: Tuple[torch.Tensor, int], ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) ( ArmTester( module, example_inputs=test_data, - compile_spec=common.get_u55_compile_spec(), + compile_spec=compile_spec, ) .quantize(Quantize(quantizer, get_symmetric_quantization_config())) .export() @@ -100,4 +104,12 @@ def test_unsqueeze_tosa_BI(self, test_tensor: torch.Tensor): @parameterized.expand(Unsqueeze.test_parameters) def test_unsqueeze_u55_BI(self, test_tensor: torch.Tensor): - self._test_unsqueeze_tosa_u55_pipeline(self.Unsqueeze(), (test_tensor, 0)) + self._test_unsqueeze_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.Unsqueeze(), (test_tensor, 0) + ) + + @parameterized.expand(Unsqueeze.test_parameters) + def test_unsqueeze_u85_BI(self, test_tensor: torch.Tensor): + self._test_unsqueeze_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.Unsqueeze(), (test_tensor, 0) + ) diff --git a/backends/arm/test/ops/test_view.py b/backends/arm/test/ops/test_view.py index 1f51261bf7a..53025c0ac08 100644 --- a/backends/arm/test/ops/test_view.py +++ b/backends/arm/test/ops/test_view.py @@ -21,6 +21,7 @@ from executorch.backends.arm.test.tester.arm_tester import ArmTester from executorch.backends.xnnpack.test.tester.tester import Quantize +from executorch.exir.backend.compile_spec_schema import CompileSpec from parameterized import parameterized @@ -73,8 +74,11 @@ def _test_view_tosa_BI_pipeline( .run_method_and_compare_outputs(inputs=test_data, qtol=1) ) - def _test_view_u55_BI_pipeline( - self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + def _test_view_ethos_BI_pipeline( + self, + compile_spec: list[CompileSpec], + module: torch.nn.Module, + test_data: Tuple[torch.Tensor], ): quantizer = ArmQuantizer().set_io(get_symmetric_quantization_config()) ( @@ -92,6 +96,20 @@ def _test_view_u55_BI_pipeline( .to_executorch() ) + def _test_view_u55_BI_pipeline( + self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + ): + self._test_view_ethos_BI_pipeline( + common.get_u55_compile_spec(), module, test_data + ) + + def _test_view_u85_BI_pipeline( + self, module: torch.nn.Module, test_data: Tuple[torch.Tensor] + ): + self._test_view_ethos_BI_pipeline( + common.get_u85_compile_spec(), module, test_data + ) + @parameterized.expand(View.test_parameters) def test_view_tosa_MI(self, test_tensor: torch.Tensor): self._test_view_tosa_MI_pipeline(self.View(), (test_tensor,)) @@ -103,3 +121,7 @@ def test_view_tosa_BI(self, test_tensor: torch.Tensor): @parameterized.expand(View.test_parameters) def test_view_u55_BI(self, test_tensor: torch.Tensor): self._test_view_u55_BI_pipeline(self.View(), (test_tensor,)) + + @parameterized.expand(View.test_parameters) + def test_view_u85_BI(self, test_tensor: torch.Tensor): + self._test_view_u85_BI_pipeline(self.View(), (test_tensor,)) diff --git a/examples/arm/aot_arm_compiler.py b/examples/arm/aot_arm_compiler.py index 4d77e819089..9a45195e58f 100644 --- a/examples/arm/aot_arm_compiler.py +++ b/examples/arm/aot_arm_compiler.py @@ -214,7 +214,11 @@ def forward(self, x): edge = edge.to_backend( ArmPartitioner( ArmCompileSpecBuilder() - .ethosu_compile_spec("ethos-u55-128") + .ethosu_compile_spec( + "ethos-u55-128", + system_config="Ethos_U55_High_End_Embedded", + memory_mode="Shared_Sram", + ) .set_permute_memory_format( args.model_name in MODEL_NAME_TO_MODEL.keys() ) diff --git a/examples/arm/setup.sh b/examples/arm/setup.sh index 8c39a3a8668..3d99143d27b 100755 --- a/examples/arm/setup.sh +++ b/examples/arm/setup.sh @@ -216,7 +216,7 @@ function setup_vela() { if [[ ! -e ethos-u-vela ]]; then git clone https://review.mlplatform.org/ml/ethos-u/ethos-u-vela repo_dir="${root_dir}/ethos-u-vela" - base_rev=d362f5443f67b1e6213a9d8f124edff758efac96 + base_rev=fe0eaa55c5ed319f78c01978f3b40eb11a9bcb38 patch_repo fi cd "${root_dir}/ethos-u-vela"