Skip to content

Commit a0d9c7e

Browse files
oscarandersson8218oscarandersson8218
authored
Arm backend: Fix sigmoid int16 and int32 flakyness (#10548)
Step vela pin and remove all related pytest.mark_flaky in sigmoid int16 and int32 tests. Signed-off-by: Oscar Andersson <[email protected]>
1 parent 8953279 commit a0d9c7e

File tree

6 files changed

+41
-36
lines changed

6 files changed

+41
-36
lines changed

‎backends/arm/operator_support/tosa_supported_operators.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,7 @@ class CheckProperQuantization(OperatorSupportBase):
306306
exir_ops.edge.aten.sub.Tensor,
307307
exir_ops.edge.aten.upsample_bilinear2d.vec,
308308
exir_ops.edge.aten.upsample_nearest2d.vec,
309+
torch.ops.aten.scalar_tensor.default,
309310
*TableOps.included_ops(),
310311
)
311312

‎backends/arm/test/models/test_conformer.py‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ def test_conformer_tosa_MI(self):
6363
)
6464
)
6565

66-
@unittest.expectedFailure # TODO(MLETORCH-635)
6766
def test_conformer_tosa_BI(self):
6867
(
6968
ArmTester(
@@ -111,7 +110,6 @@ def test_conformer_u55_BI(self):
111110
except Exception:
112111
pass
113112

114-
@unittest.expectedFailure # TODO(MLETORCH-635)
115113
def test_conformer_u85_BI(self):
116114
tester = (
117115
ArmTester(
@@ -126,9 +124,15 @@ def test_conformer_u85_BI(self):
126124
.serialize()
127125
)
128126
if conftest.is_option_enabled("corstone_fvp"):
129-
tester.run_method_and_compare_outputs(
130-
qtol=1.0,
131-
rtol=1.0,
132-
atol=5.0,
133-
inputs=get_test_inputs(self.dim, self.lengths, self.num_examples),
134-
)
127+
try:
128+
tester.run_method_and_compare_outputs(
129+
qtol=1.0,
130+
rtol=1.0,
131+
atol=5.0,
132+
inputs=get_test_inputs(self.dim, self.lengths, self.num_examples),
133+
)
134+
self.fail(
135+
"TODO(MLETORCH-635): Expected failure under FVP option, but test passed."
136+
)
137+
except Exception:
138+
pass

‎backends/arm/test/ops/test_depthwise_conv.py‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -202,13 +202,13 @@ def test_convolution_2d_tosa_BI_depth_wise(test_module: torch.nn.Module):
202202

203203

204204
x_fails = {
205-
"3x3_2x8x198x198_gp8_st3": "MLETORCH-516: AssertionError: Output 0 does not match reference output.",
206-
"two_dw_conv2d": "MLETORCH-516: AssertionError: Output 0 does not match reference output.",
205+
"3x3_2x8x198x198_gp8_st3": "MLETORCH-517: Operators fail with batches > 1",
206+
"two_dw_conv2d": "MLETORCH-517: Operators fail with batches > 1",
207207
}
208208

209209

210-
@common.parametrize("test_module", testsuite_conv2d, x_fails)
211210
@common.XfailIfNoCorstone300 # TODO: MLETORCH-516
211+
@common.parametrize("test_module", testsuite_conv2d, x_fails)
212212
def test_convolution_2d_u55_BI_depth_wise(test_module: torch.nn.Module):
213213
pipeline = EthosU55PipelineBI[input_t](
214214
test_module(),
@@ -233,8 +233,8 @@ def test_convolution_1d_u55_BI_depth_wise(test_module: torch.nn.Module):
233233
pipeline.run()
234234

235235

236-
@common.parametrize("test_module", testsuite_conv1d | testsuite_conv2d, x_fails)
237236
@common.XfailIfNoCorstone320 # TODO: MLETORCH-516
237+
@common.parametrize("test_module", testsuite_conv2d, x_fails)
238238
def test_convolution_2d_u85_BI_depth_wise(test_module: torch.nn.Module):
239239
pipeline = EthosU85PipelineBI[input_t](
240240
test_module(),
@@ -244,3 +244,16 @@ def test_convolution_2d_u85_BI_depth_wise(test_module: torch.nn.Module):
244244
run_on_fvp=True,
245245
)
246246
pipeline.run()
247+
248+
249+
@common.XfailIfNoCorstone320 # TODO: MLETORCH-516
250+
@common.parametrize("test_module", testsuite_conv1d, x_fails)
251+
def test_convolution_1d_u85_BI_depth_wise(test_module: torch.nn.Module):
252+
pipeline = EthosU85PipelineBI[input_t](
253+
test_module(),
254+
test_module().get_inputs(),
255+
aten_ops=[],
256+
exir_ops=exir_op,
257+
run_on_fvp=True,
258+
)
259+
pipeline.run()

‎backends/arm/test/ops/test_sigmoid_16bit.py‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
# LICENSE file in the root directory of this source tree.
55

66
import pytest
7-
87
import torch
98
from executorch.backends.arm.quantizer import (
109
get_symmetric_quantization_config,
@@ -91,10 +90,13 @@ def forward(self, x):
9190

9291

9392
@common.parametrize("test_data", test_data_suite)
94-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
9593
def test_sigmoid_tosa_BI(test_data):
9694
pipeline = TosaPipelineBI(
97-
Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
95+
Sigmoid(),
96+
(test_data(),),
97+
Sigmoid.aten_op,
98+
Sigmoid.exir_op,
99+
qtol=1,
98100
)
99101
pipeline.change_args("quantize", get_16bit_sigmoid_quantizer())
100102
pipeline.run()
@@ -108,13 +110,13 @@ def test_sigmoid_tosa_BI(test_data):
108110
},
109111
strict=False,
110112
)
111-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
112113
def test_sigmoid_tosa_BI_add_sigmoid(test_data):
113114
pipeline = TosaPipelineBI(
114115
SigmoidAddSigmoid(),
115116
(test_data(),),
116117
Sigmoid.aten_op,
117118
Sigmoid.exir_op,
119+
qtol=1,
118120
)
119121
pipeline.run()
120122

@@ -131,7 +133,6 @@ def test_sigmoid_tosa_BI_add_sigmoid(test_data):
131133
"test_data",
132134
test_data_suite,
133135
)
134-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
135136
def test_sigmoid_u55_BI(test_data):
136137
pipeline = OpNotSupportedPipeline(
137138
Sigmoid(),
@@ -148,7 +149,6 @@ def test_sigmoid_u55_BI(test_data):
148149
"test_data",
149150
test_data_suite,
150151
)
151-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
152152
def test_sigmoid_u55_BI_add_sigmoid(test_data):
153153
pipeline = OpNotSupportedPipeline(
154154
SigmoidAddSigmoid(),
@@ -163,7 +163,6 @@ def test_sigmoid_u55_BI_add_sigmoid(test_data):
163163

164164

165165
@common.parametrize("test_data", test_data_suite)
166-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
167166
@common.XfailIfNoCorstone320
168167
def test_sigmoid_u85_BI(test_data):
169168
pipeline = EthosU85PipelineBI(
@@ -181,10 +180,10 @@ def test_sigmoid_u85_BI(test_data):
181180
"test_data",
182181
test_data_suite,
183182
xfails={
184-
"ramp": "AssertionError: Output 0 does not match reference output.",
183+
"ramp": "AssertionError: Output 0 does not match reference output. MLETORCH-787"
185184
},
186185
)
187-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
186+
@pytest.mark.flaky(reruns=5) # MLETORCH-787: Investigate int16-int8 rescaling precision
188187
@common.XfailIfNoCorstone320
189188
def test_sigmoid_u85_BI_add_sigmoid(test_data):
190189
pipeline = EthosU85PipelineBI(

‎backends/arm/test/ops/test_sigmoid_32bit.py‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
55

6-
import pytest
76
import torch
87
from executorch.backends.arm.quantizer import TOSAQuantizer
98
from executorch.backends.arm.quantizer.quantization_config import QuantizationConfig
@@ -107,33 +106,32 @@ def forward(self, x):
107106

108107

109108
@common.parametrize("test_data", test_data_suite)
110-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
111109
def test_sigmoid_tosa_BI(test_data):
112110
pipeline = TosaPipelineBI(
113111
Sigmoid(),
114112
(test_data(),),
115113
Sigmoid.aten_op,
116114
Sigmoid.exir_op,
115+
qtol=1,
117116
)
118117
pipeline.change_args("quantize", get_32bit_sigmoid_quantizer())
119118
pipeline.run()
120119

121120

122121
@common.parametrize("test_data", test_data_suite)
123-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
124122
def test_sigmoid_tosa_BI_add_sigmoid(test_data):
125123
pipeline = TosaPipelineBI(
126124
SigmoidAddSigmoid(),
127125
(test_data(),),
128126
Sigmoid.aten_op,
129127
Sigmoid.exir_op,
128+
qtol=1,
130129
)
131130
pipeline.change_args("quantize", get_32bit_sigmoid_quantizer())
132131
pipeline.run()
133132

134133

135134
@common.parametrize("test_data", test_data_suite)
136-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
137135
def test_sigmoid_u55_BI(test_data):
138136
pipeline = OpNotSupportedPipeline(
139137
Sigmoid(),
@@ -147,7 +145,6 @@ def test_sigmoid_u55_BI(test_data):
147145

148146

149147
@common.parametrize("test_data", test_data_suite)
150-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
151148
def test_sigmoid_u55_BI_add_sigmoid(test_data):
152149
pipeline = OpNotSupportedPipeline(
153150
SigmoidAddSigmoid(),
@@ -162,9 +159,7 @@ def test_sigmoid_u55_BI_add_sigmoid(test_data):
162159

163160

164161
@common.parametrize("test_data", test_data_suite)
165-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
166162
@common.XfailIfNoCorstone320
167-
@pytest.mark.flaky(reruns=5)
168163
def test_sigmoid_u85_BI(test_data):
169164
pipeline = EthosU85PipelineBI(
170165
Sigmoid(),
@@ -180,15 +175,8 @@ def test_sigmoid_u85_BI(test_data):
180175
@common.parametrize(
181176
"test_data",
182177
test_data_suite,
183-
xfails={
184-
"ramp": "AssertionError: Output 0 does not match reference output.",
185-
"rand": "AssertionError: Output 0 does not match reference output.",
186-
"rand_4d": "AssertionError: Output 0 does not match reference output.",
187-
},
188178
)
189-
@pytest.mark.flaky(reruns=32) # Flaky due to Vela bug: MLBEDSW-10642
190179
@common.XfailIfNoCorstone320
191-
@pytest.mark.flaky(reruns=5)
192180
def test_sigmoid_u85_BI_add_sigmoid(test_data):
193181
pipeline = EthosU85PipelineBI(
194182
SigmoidAddSigmoid(),

‎examples/arm/setup.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ fi
6060

6161
# vela
6262
vela_repo_url="https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela"
63-
vela_rev="859cc066178a87ff28230c1ce9bd370f1e98aa5a"
63+
vela_rev="8cac2b9a7204b57125a8718049519b091a98846c"
6464

6565
########
6666
### Functions

0 commit comments

Comments
 (0)