From 5ce6f1d3ca2719a40eb1a8b668d785d545622d93 Mon Sep 17 00:00:00 2001 From: Erik Lundell Date: Mon, 27 Jan 2025 14:47:40 +0100 Subject: [PATCH] Fix helper method for getting input names The previous version missed ConstantInputs which caused problems in the runtime when the provided inputs did not match the GraphSignature. You could argue that having to provide constant inputs that are already encoded in the graph as arguments is unnecessary. However, this solution is more general and does not stop us from adding a pass that prunes unwanted inputs from the graph signature in the future. (if that is possible) Encountered an issue with too long input names, had to rename some parameters in testcases to shorter names (added ticket to adress this, #MLETORCH-628) Signed-off-by: Erik Lundell Change-Id: Idf39d94aa1340ace655c1393b97f4616aec250db --- backends/arm/test/ops/test_cat.py | 34 +++++++++++++++++++++------- backends/arm/test/ops/test_expand.py | 34 ++++++++++++++++++++-------- backends/arm/test/ops/test_full.py | 4 ---- backends/arm/test/runner_utils.py | 22 +++++++----------- 4 files changed, 59 insertions(+), 35 deletions(-) diff --git a/backends/arm/test/ops/test_cat.py b/backends/arm/test/ops/test_cat.py index 115b4402f5b..a1613d1d04b 100644 --- a/backends/arm/test/ops/test_cat.py +++ b/backends/arm/test/ops/test_cat.py @@ -1,5 +1,5 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. -# Copyright 2024 Arm Limited and/or its affiliates. +# Copyright 2024-2025 Arm Limited and/or its affiliates. # All rights reserved. # # This source code is licensed under the BSD-style license found in the @@ -33,6 +33,8 @@ class Cat(torch.nn.Module): ), -1, ), + ((torch.randn(1, 2, 4, 4), torch.randn(1, 2, 4, 1)), 3), + ((torch.randn(1, 2, 4, 4), torch.randn(1, 2, 4, 4)), 0), ((torch.randn(2, 2, 4, 4), torch.randn(2, 2, 4, 1)), 3), ( ( @@ -47,8 +49,8 @@ class Cat(torch.nn.Module): def __init__(self): super().__init__() - def forward(self, tensors: tuple[torch.Tensor, ...], dim: int) -> torch.Tensor: - return torch.cat(tensors, dim=dim) + def forward(self, t: tuple[torch.Tensor, ...], dim: int) -> torch.Tensor: + return torch.cat(t, dim=dim) def _test_cat_tosa_MI_pipeline( self, module: torch.nn.Module, test_data: Tuple[tuple[torch.Tensor, ...], int] @@ -134,22 +136,38 @@ def test_cat_tosa_BI(self, operands: tuple[torch.Tensor, ...], dim: int): test_data = (operands, dim) self._test_cat_tosa_BI_pipeline(self.Cat(), test_data) - # Mismatch in provided number of inputs and model signature, MLETORCH 519 - @parameterized.expand(Cat.test_parameters) + @parameterized.expand(Cat.test_parameters[:-3]) @pytest.mark.corstone_fvp - @conftest.expectedFailureOnFVP def test_cat_u55_BI(self, operands: tuple[torch.Tensor, ...], dim: int): test_data = (operands, dim) self._test_cat_ethosu_BI_pipeline( self.Cat(), common.get_u55_compile_spec(), test_data ) - # Mismatch in provided number of inputs and model signature, MLETORCH 519 - @parameterized.expand(Cat.test_parameters) + # MLETORCH-630 Cat does not work on FVP with batch>1 + @parameterized.expand(Cat.test_parameters[-3:]) @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP + def test_cat_u55_BI_xfails(self, operands: tuple[torch.Tensor, ...], dim: int): + test_data = (operands, dim) + self._test_cat_ethosu_BI_pipeline( + self.Cat(), common.get_u55_compile_spec(), test_data + ) + + @parameterized.expand(Cat.test_parameters[:-3]) + @pytest.mark.corstone_fvp def test_cat_u85_BI(self, operands: tuple[torch.Tensor, ...], dim: int): test_data = (operands, dim) self._test_cat_ethosu_BI_pipeline( self.Cat(), common.get_u85_compile_spec(), test_data ) + + # MLETORCH-630 Cat does not work on FVP with batch>1 + @parameterized.expand(Cat.test_parameters[-3:]) + @pytest.mark.corstone_fvp + @conftest.expectedFailureOnFVP + def test_cat_u85_BI_xfails(self, operands: tuple[torch.Tensor, ...], dim: int): + test_data = (operands, dim) + self._test_cat_ethosu_BI_pipeline( + self.Cat(), common.get_u85_compile_spec(), test_data + ) diff --git a/backends/arm/test/ops/test_expand.py b/backends/arm/test/ops/test_expand.py index 116f5d64e87..d0807f3db0b 100644 --- a/backends/arm/test/ops/test_expand.py +++ b/backends/arm/test/ops/test_expand.py @@ -37,15 +37,17 @@ class Expand(torch.nn.Module): test_parameters = [ (torch.rand(1), (2,)), (torch.randn(1, 4), (1, -1)), - (torch.rand(1, 1, 2, 2), (4, 3, -1, 2)), (torch.randn(1), (2, 2, 4)), - (torch.rand(3, 2, 4, 1), (-1, -1, -1, 3)), + (torch.randn(1, 1, 1, 5), (1, 4, -1, -1)), (torch.randn(1, 1, 192), (1, -1, -1)), + (torch.randn(1, 1), (1, 2, 2, 4)), + (torch.randn(1, 1), (2, 2, 2, 4)), (torch.randn(10, 1, 1, 97), (-1, 4, -1, -1)), + (torch.rand(1, 1, 2, 2), (4, 3, -1, 2)), ] - def forward(self, x: torch.Tensor, multiples: Sequence): - return x.expand(multiples) + def forward(self, x: torch.Tensor, m: Sequence): + return x.expand(m) def _test_expand_tosa_MI_pipeline(self, module: torch.nn.Module, test_data: Tuple): ( @@ -113,20 +115,34 @@ def test_expand_tosa_MI(self, test_input, multiples): def test_expand_tosa_BI(self, test_input, multiples): self._test_expand_tosa_BI_pipeline(self.Expand(), (test_input, multiples)) - # Mismatch in provided number of inputs and model signature, MLETORCH 519 - @parameterized.expand(Expand.test_parameters) + @parameterized.expand(Expand.test_parameters[:-3]) @pytest.mark.corstone_fvp - @conftest.expectedFailureOnFVP def test_expand_u55_BI(self, test_input, multiples): self._test_expand_ethosu_BI_pipeline( common.get_u55_compile_spec(), self.Expand(), (test_input, multiples) ) - # Mismatch in provided number of inputs and model signature, MLETORCH 519 - @parameterized.expand(Expand.test_parameters) + # MLETORCH-629: Expand does not work on FVP with batch>1 + @parameterized.expand(Expand.test_parameters[-3:]) @pytest.mark.corstone_fvp @conftest.expectedFailureOnFVP + def test_expand_u55_BI_xfails(self, test_input, multiples): + self._test_expand_ethosu_BI_pipeline( + common.get_u55_compile_spec(), self.Expand(), (test_input, multiples) + ) + + @parameterized.expand(Expand.test_parameters[:-3]) + @pytest.mark.corstone_fvp def test_expand_u85_BI(self, test_input, multiples): self._test_expand_ethosu_BI_pipeline( common.get_u85_compile_spec(), self.Expand(), (test_input, multiples) ) + + # MLETORCH-629: Expand does not work on FVP with batch>1 + @parameterized.expand(Expand.test_parameters[-3:]) + @pytest.mark.corstone_fvp + @conftest.expectedFailureOnFVP + def test_expand_u85_BI_xfails(self, test_input, multiples): + self._test_expand_ethosu_BI_pipeline( + common.get_u85_compile_spec(), self.Expand(), (test_input, multiples) + ) diff --git a/backends/arm/test/ops/test_full.py b/backends/arm/test/ops/test_full.py index fc82fa4dd71..586e6bd4db2 100644 --- a/backends/arm/test/ops/test_full.py +++ b/backends/arm/test/ops/test_full.py @@ -143,20 +143,16 @@ def test_full_tosa_MI(self, test_tensor: Tuple): def test_full_tosa_BI(self, test_tensor: Tuple): self._test_full_tosa_BI_pipeline(self.AddVariableFull(), test_tensor) - # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(AddVariableFull.test_parameters) @pytest.mark.corstone_fvp - @conftest.expectedFailureOnFVP def test_full_u55_BI(self, test_tensor: Tuple): self._test_full_tosa_u55_pipeline( self.AddVariableFull(), test_tensor, ) - # Mismatch in provided number of inputs and model signature, MLETORCH 519 @parameterized.expand(AddVariableFull.test_parameters) @pytest.mark.corstone_fvp - @conftest.expectedFailureOnFVP def test_full_u85_BI(self, test_tensor: Tuple): self._test_full_tosa_u85_pipeline( self.AddVariableFull(), diff --git a/backends/arm/test/runner_utils.py b/backends/arm/test/runner_utils.py index 6996d53e913..7131b723205 100644 --- a/backends/arm/test/runner_utils.py +++ b/backends/arm/test/runner_utils.py @@ -65,16 +65,7 @@ def get_input_names(program: ExportedProgram) -> list[str]: Returns: A list of strings with the names of the model input. """ - input_names = [] - - # E.g. bias and weights are 'placeholders' as well. This is used to - # get only the use inputs. - usr_inputs = program.graph_signature.user_inputs - for node in program.graph.nodes: - if node.op == "placeholder" and node.name in usr_inputs: - input_names.append(node.name) - - return input_names + return [spec.arg.name for spec in program.graph_signature.input_specs] def get_input_quantization_params( @@ -334,13 +325,16 @@ def run_corstone( def prep_data_for_save( - data: torch.Tensor, + data, input_name: str, quant_param: Optional[QuantizationParams] = None, ): - data_np = np.array(data.detach(), order="C").astype( - torch_to_numpy_dtype_dict[data.dtype] - ) + if isinstance(data, torch.Tensor): + data_np = np.array(data.detach(), order="C").astype( + torch_to_numpy_dtype_dict[data.dtype] + ) + else: + data_np = np.array(data) if quant_param is not None: assert quant_param.node_name in input_name, ( f"The quantization params name '{quant_param.node_name}' does not "