From 6c5b7e5cbe5774b316989522c907f47e02ff5ee7 Mon Sep 17 00:00:00 2001 From: Tom Allsop Date: Fri, 21 Feb 2025 14:38:43 +0000 Subject: [PATCH 1/2] Arm: Add MobileNet v3 testcase * Add missing inplace operators to quantization annotator. * Add atol, rtol & qtol to test_pipeline classes. Co-authored-by: Oscar Andersson Signed-off-by: Tom Allsop Change-Id: Idca2bcfdaf8039430a73148e9742f747b03b3e6b --- .../arm/quantizer/quantization_annotator.py | 2 + .../arm/test/models/test_mobilenet_v3_arm.py | 75 +++++++++++++++++++ backends/arm/test/tester/test_pipeline.py | 32 +++++++- 3 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 backends/arm/test/models/test_mobilenet_v3_arm.py diff --git a/backends/arm/quantizer/quantization_annotator.py b/backends/arm/quantizer/quantization_annotator.py index 271210172c4..3e98df7f6f5 100644 --- a/backends/arm/quantizer/quantization_annotator.py +++ b/backends/arm/quantizer/quantization_annotator.py @@ -137,6 +137,7 @@ def _match_pattern( torch.ops.aten.sum.dim_IntList, torch.ops.aten.hardsigmoid.default, torch.ops.aten.hardswish.default, + torch.ops.aten.hardswish_.default, torch.ops.aten.full_like.default, ] @@ -196,6 +197,7 @@ def _match_pattern( torch.ops.aten.full.default, torch.ops.aten.flatten.using_ints, torch.ops.aten.dropout.default, + torch.ops.aten.dropout_.default, torch.ops.aten.clamp.default, torch.ops.aten.clamp.Tensor, operator.getitem, diff --git a/backends/arm/test/models/test_mobilenet_v3_arm.py b/backends/arm/test/models/test_mobilenet_v3_arm.py new file mode 100644 index 00000000000..66a4096d665 --- /dev/null +++ b/backends/arm/test/models/test_mobilenet_v3_arm.py @@ -0,0 +1,75 @@ +# Copyright 2025 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Tuple + +import torch + +from executorch.backends.arm.test.tester.test_pipeline import ( + EthosU55PipelineBI, + EthosU85PipelineBI, + TosaPipelineBI, + TosaPipelineMI, +) + +from torchvision import models, transforms + +mv3 = models.mobilenet_v3_small(weights=models.MobileNet_V3_Small_Weights) +mv3 = mv3.eval() + +normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + +input_tensor = torch.rand(1, 3, 232, 232) + +model_inputs = (normalize(input_tensor),) +input_t = Tuple[torch.Tensor] + + +def test_mv3_tosa_MI(): + pipeline = TosaPipelineMI[input_t]( + mv3, model_inputs, aten_op=[], exir_op=[], use_to_edge_transform_and_lower=True + ) + pipeline.run() + + +def test_mv3_tosa_BI(): + pipeline = TosaPipelineBI[input_t]( + mv3, + model_inputs, + aten_op=[], + exir_op=[], + use_to_edge_transform_and_lower=True, + atol=0.3, + qtol=1, + ) + pipeline.run() + + +def test_mv3_u55_BI(): + pipeline = EthosU55PipelineBI[input_t]( + mv3, + model_inputs, + aten_ops=[], + exir_ops=[], + run_on_fvp=True, + use_to_edge_transform_and_lower=True, + atol=0.3, + qtol=1, + ) + pipeline.run() + + +def test_mv3_u85_BI(): + pipeline = EthosU85PipelineBI[input_t]( + mv3, + model_inputs, + aten_ops=[], + exir_ops=[], + run_on_fvp=True, + use_to_edge_transform_and_lower=True, + atol=0.3, + qtol=1, + ) + pipeline.run() diff --git a/backends/arm/test/tester/test_pipeline.py b/backends/arm/test/tester/test_pipeline.py index 01707b6364a..cad62c021f6 100644 --- a/backends/arm/test/tester/test_pipeline.py +++ b/backends/arm/test/tester/test_pipeline.py @@ -274,6 +274,9 @@ def __init__( symmetric_io_quantization: bool = False, use_to_edge_transform_and_lower: bool = True, custom_path: str = None, + atol: float = 1e-03, + rtol: float = 1e-03, + qtol: int = 0, ): compile_spec = common.get_tosa_compile_spec( tosa_version, custom_path=custom_path @@ -322,7 +325,11 @@ def __init__( ) self.add_stage( - self.tester.run_method_and_compare_outputs, inputs=self.test_data + self.tester.run_method_and_compare_outputs, + atol=atol, + rtol=rtol, + qtol=qtol, + inputs=self.test_data, ) @@ -353,6 +360,9 @@ def __init__( tosa_version: str = "TOSA-0.80+MI", use_to_edge_transform_and_lower: bool = True, custom_path: str = None, + atol: float = 1e-03, + rtol: float = 1e-03, + qtol: int = 0, ): compile_spec = common.get_tosa_compile_spec( tosa_version, custom_path=custom_path @@ -376,7 +386,11 @@ def __init__( ) self.add_stage( - self.tester.run_method_and_compare_outputs, inputs=self.test_data + self.tester.run_method_and_compare_outputs, + atol=atol, + rtol=rtol, + qtol=qtol, + inputs=self.test_data, ) @@ -406,6 +420,9 @@ def __init__( symmetric_io_quantization: bool = False, use_to_edge_transform_and_lower: bool = False, custom_path: str = None, + atol: float = 1e-03, + rtol: float = 1e-03, + qtol: int = 1, ): compile_spec = common.get_u55_compile_spec(custom_path=custom_path) quant_stage = ( @@ -458,7 +475,9 @@ def __init__( self.add_stage(self.tester.serialize) self.add_stage( self.tester.run_method_and_compare_outputs, - qtol=1, + atol=atol, + rtol=rtol, + qtol=qtol, inputs=self.test_data, ) @@ -489,6 +508,9 @@ def __init__( symmetric_io_quantization: bool = False, use_to_edge_transform_and_lower: bool = False, custom_path: str = None, + atol: float = 1e-03, + rtol: float = 1e-03, + qtol: int = 1, ): compile_spec = common.get_u85_compile_spec(custom_path=custom_path) quant_stage = ( @@ -541,7 +563,9 @@ def __init__( self.add_stage(self.tester.serialize) self.add_stage( self.tester.run_method_and_compare_outputs, - qtol=1, + atol=atol, + rtol=rtol, + qtol=qtol, inputs=self.test_data, ) From d775706b3a9e290130294c98c1572bfdb54e35b9 Mon Sep 17 00:00:00 2001 From: Tom Allsop Date: Fri, 21 Feb 2025 14:38:43 +0000 Subject: [PATCH 2/2] Add XfailIfNoCorstone* to tests Change-Id: Ie2307d9f4238ce75aa816946b132330490c3a441 --- backends/arm/test/models/test_mobilenet_v3_arm.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backends/arm/test/models/test_mobilenet_v3_arm.py b/backends/arm/test/models/test_mobilenet_v3_arm.py index 66a4096d665..7e0afe4a54f 100644 --- a/backends/arm/test/models/test_mobilenet_v3_arm.py +++ b/backends/arm/test/models/test_mobilenet_v3_arm.py @@ -5,6 +5,9 @@ from typing import Tuple +import common +import pytest + import torch from executorch.backends.arm.test.tester.test_pipeline import ( @@ -27,6 +30,7 @@ input_t = Tuple[torch.Tensor] +@pytest.mark.slow def test_mv3_tosa_MI(): pipeline = TosaPipelineMI[input_t]( mv3, model_inputs, aten_op=[], exir_op=[], use_to_edge_transform_and_lower=True @@ -34,6 +38,7 @@ def test_mv3_tosa_MI(): pipeline.run() +@pytest.mark.slow def test_mv3_tosa_BI(): pipeline = TosaPipelineBI[input_t]( mv3, @@ -47,6 +52,9 @@ def test_mv3_tosa_BI(): pipeline.run() +@pytest.mark.slow +@pytest.mark.corstone_fvp +@common.XfailIfNoCorstone300 def test_mv3_u55_BI(): pipeline = EthosU55PipelineBI[input_t]( mv3, @@ -61,6 +69,9 @@ def test_mv3_u55_BI(): pipeline.run() +@pytest.mark.slow +@pytest.mark.corstone_fvp +@common.XfailIfNoCorstone320 def test_mv3_u85_BI(): pipeline = EthosU85PipelineBI[input_t]( mv3,