Arm backend: Fix sigmoid int16 and int32 flakyness

MLBEDSW-10642 has now been resolved. Step vela pin and remove all
related pytest.mark_flaky in sigmoid int16 and int32 tests. Also updates
dw_conv2d tests as new constraints were introduced in vela compilation.

Signed-off-by: Oscar Andersson <[email protected]>
Change-Id: I8c1aff31fb173fa6fdebc1d5b757623f9c2e1321

Author: oscarandersson8218

backends/arm/test/ops/test_depthwise_conv.py

@@ -1,5 +1,4 @@
 # Copyright 2024-2025 Arm Limited and/or its affiliates.
-# All rights reserved.
 #
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
@@ -165,11 +164,11 @@
     ("3x3_1x4x256x256_gp4_nobias", dw_conv2d_3x3_1x4x256x256_gp4_nobias),
 ]
 
-testsuite_conv2d_u85_xfails = [
+testsuite_conv2d_u55_xfails = [
     ("3x3_2x8x198x198_gp8_st3", dw_conv2d_3x3_2x8x198x198_gp8_st3),
     ("two_dw_conv2d", two_dw_conv2d),
 ]
-
+testsuite_conv2d_u85_xfails = testsuite_conv2d_u55_xfails
 
 testsuite_conv1d = [
     ("2_1x6x4_gp6_st1", dw_conv1d_2_1x6x4_gp6_st1),
@@ -266,9 +265,8 @@ def test_dw_conv2d_u55_BI(self, test_name: str, model: torch.nn.Module):
             model.get_inputs(),
         )
 
-    @parameterized.expand(testsuite_conv2d[4:], skip_on_empty=True)
-    @pytest.mark.corstone_fvp
-    @conftest.expectedFailureOnFVP  # TODO: MLETORCH-516
+    @parameterized.expand(testsuite_conv2d_u55_xfails)
+    @pytest.mark.xfail(reason="MLBEDSW-10752: Vela issues with batches > 1 for dw_conv")
     def test_dw_conv2d_u55_BI_xfails(self, test_name: str, model: torch.nn.Module):
         self._test_dw_conv_ethos_BI_pipeline(
             model,
@@ -294,11 +292,9 @@ def test_dw_conv_u85_BI(self, test_name: str, model: torch.nn.Module):
             model.get_inputs(),
         )
 
-    # All test cases except 3x3_1x3x256x256_gp3_st1 have numerical issues on FVP. MLETORCH-520
     @parameterized.expand(testsuite_conv2d_u85_xfails)
-    @pytest.mark.corstone_fvp
-    @conftest.expectedFailureOnFVP
-    def test_dw_conv_u85_BI_xfails(self, test_name: str, model: torch.nn.Module):
+    @pytest.mark.xfail(reason="MLBEDSW-10752: Vela issues with batches > 1 for dw_conv")
+    def test_dw_conv2d_u85_BI_xfails(self, test_name: str, model: torch.nn.Module):
         self._test_dw_conv_ethos_BI_pipeline(
             model,
             common.get_u85_compile_spec(),

backends/arm/test/ops/test_sigmoid_16bit.py

@@ -4,7 +4,6 @@
 # LICENSE file in the root directory of this source tree.
 
 import pytest
-
 import torch
 from executorch.backends.arm.quantizer import (
     get_symmetric_quantization_config,
@@ -81,10 +80,13 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
-        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
+        Sigmoid(),
+        (test_data(),),
+        Sigmoid.aten_op,
+        Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
@@ -97,10 +99,14 @@ def test_sigmoid_tosa_BI(test_data):
         "ramp": "AssertionError: Output 0 does not match reference output. MLETORCH-787"
     },
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
+@pytest.mark.flaky(reruns=5)  # MLETORCH-787: Investigate int16-int8 rescaling precision
 def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
-        SigmoidAddSigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
+        SigmoidAddSigmoid(),
+        (test_data(),),
+        Sigmoid.aten_op,
+        Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
@@ -110,7 +116,6 @@ def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     "test_data",
     test_data_suite,
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         Sigmoid(), (test_data(),), "TOSA-0.80+BI+u55", {Sigmoid.exir_op: 1}
@@ -123,7 +128,6 @@ def test_sigmoid_tosa_u55(test_data):
     "test_data",
     test_data_suite,
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_add_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         SigmoidAddSigmoid(),
@@ -137,7 +141,6 @@ def test_sigmoid_add_sigmoid_tosa_u55(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
 def test_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
@@ -151,10 +154,10 @@ def test_sigmoid_tosa_u85(test_data):
     "test_data",
     test_data_suite,
     xfails={
-        "ramp": "AssertionError: Output 0 does not match reference output.",
+        "ramp": "AssertionError: Output 0 does not match reference output. MLETORCH-787"
     },
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
+@pytest.mark.flaky(reruns=5)  # MLETORCH-787: Investigate int16-int8 rescaling precision
 @common.XfailIfNoCorstone320
 def test_sigmoid_add_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(

backends/arm/test/ops/test_sigmoid_32bit.py

@@ -3,7 +3,6 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-import pytest
 import torch
 from executorch.backends.arm.quantizer import TOSAQuantizer
 from executorch.backends.arm.quantizer.quantization_config import QuantizationConfig
@@ -97,33 +96,32 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         Sigmoid(),
         (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         SigmoidAddSigmoid(),
         (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
+        qtol=1,
     )
     pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         Sigmoid(), (test_data(),), "TOSA-0.80+BI+u55", {Sigmoid.exir_op: 1}
@@ -133,7 +131,6 @@ def test_sigmoid_tosa_u55(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 def test_sigmoid_add_sigmoid_tosa_u55(test_data):
     pipeline = OpNotSupportedPipeline(
         SigmoidAddSigmoid(),
@@ -147,7 +144,6 @@ def test_sigmoid_add_sigmoid_tosa_u55(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
 def test_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
@@ -160,11 +156,7 @@ def test_sigmoid_tosa_u85(test_data):
 @common.parametrize(
     "test_data",
     test_data_suite,
-    xfails={
-        "ramp": "AssertionError: Output 0 does not match reference output.",
-    },
 )
-@pytest.mark.flaky(reruns=32)  # Flaky due to Vela bug: MLBEDSW-10642
 @common.XfailIfNoCorstone320
 def test_sigmoid_add_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(

examples/arm/setup.sh

@@ -60,7 +60,7 @@ fi
 
 # vela
 vela_repo_url="https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela"
-vela_rev="859cc066178a87ff28230c1ce9bd370f1e98aa5a"
+vela_rev="677651add50745478367da009936e4505d627399"
 
 ########
 ### Functions