Skip to content

Commit cc922da

Browse files
oscarandersson8218oscarandersson8218
authored
Arm backend: Enable FVP tests for ops lowered to REDUCE_SUM and fix numerical issues (#8617)
Enable FVP tests for ops lowered to REDUCE_SUM vela version is stepped to fix numerical issues with REDUCE_SUM. Add ExpectedFailureOnFvp for BMM U55 tests Signed-off-by: Oscar Andersson <[email protected]>
1 parent 282242e commit cc922da

File tree

10 files changed

+119
-147
lines changed

10 files changed

+119
-147
lines changed

‎backends/arm/test/misc/test_multiple_outputs.py‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,23 +76,21 @@ def _test_ethosu_BI_pipeline(
7676
tester.run_method_and_compare_outputs(qtol=1, inputs=test_data)
7777

7878
@pytest.mark.corstone_fvp
79-
def test_u85_BI(self):
79+
def test_u55_BI(self):
8080
module = self.MultipleOutputsModule()
8181
test_data = module.get_inputs()
8282
self._test_ethosu_BI_pipeline(
8383
module,
8484
test_data,
85-
common.get_u85_compile_spec(),
85+
common.get_u55_compile_spec(),
8686
)
8787

8888
@pytest.mark.corstone_fvp
89-
@conftest.expectedFailureOnFVP
90-
# TODO MLETORCH-598
91-
def test_u55_BI(self):
89+
def test_u85_BI(self):
9290
module = self.MultipleOutputsModule()
9391
test_data = module.get_inputs()
9492
self._test_ethosu_BI_pipeline(
9593
module,
9694
test_data,
97-
common.get_u55_compile_spec(),
95+
common.get_u85_compile_spec(),
9896
)

‎backends/arm/test/ops/test_bmm.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,9 +150,10 @@ def test_bmm_single_input_tosa_BI(self, test_data_generator: Callable[[], Tuple]
150150
test_data = test_data_generator()
151151
self._test_bmm_tosa_BI_pipeline(self.BMMSingleInput(), test_data)
152152

153+
# Expected to fail on FVP as TOSA.MATMUL is not supported on U55
153154
@parameterized.expand(BMM.test_data_generators)
154155
@pytest.mark.corstone_fvp
155-
@unittest.expectedFailure
156+
@conftest.expectedFailureOnFVP
156157
def test_bmm_u55_BI_xfails(self, test_data_generator: Callable[[], Tuple]):
157158
test_data = test_data_generator()
158159
self._test_bmm_ethosu_BI_pipeline(
@@ -167,10 +168,10 @@ def test_bmm_u85_BI(self, test_data_generator: Callable[[], Tuple]):
167168
self.BMM(), common.get_u85_compile_spec(), test_data
168169
)
169170

170-
# Expected to fail with error: Warning, unsupported fusing of TOSA Rescale previous operator is of type: Memcpy
171+
# Expected to fail on FVP as TOSA.MATMUL is not supported on U55
171172
@parameterized.expand(BMMSingleInput.test_data_generators)
172173
@pytest.mark.corstone_fvp
173-
@unittest.expectedFailure
174+
@conftest.expectedFailureOnFVP
174175
def test_bmm_single_input_u55_BI_xfails(
175176
self, test_data_generator: Callable[[], Tuple]
176177
):

‎backends/arm/test/ops/test_layer_norm.py‎

Lines changed: 2 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ def test_layer_norm_tosa_BI(
158158
self.LayerNorm(*model_params), (test_data,)
159159
)
160160

161-
@parameterized.expand(test_data_suite[4:])
161+
@parameterized.expand(test_data_suite)
162162
@pytest.mark.corstone_fvp
163163
def test_layer_norm_u55_BI(
164164
self,
@@ -170,36 +170,7 @@ def test_layer_norm_u55_BI(
170170
self.LayerNorm(*model_params), common.get_u55_compile_spec(), (test_data,)
171171
)
172172

173-
# Numerical issues on FVP likely due to mul op, MLETORCH-521
174-
# Skip tests that require transposes.
175-
@parameterized.expand(test_data_suite[:4])
176-
@pytest.mark.corstone_fvp
177-
@conftest.expectedFailureOnFVP
178-
def test_layer_norm_u55_BI_xfails(
179-
self,
180-
test_name: str,
181-
test_data: torch.Tensor,
182-
model_params,
183-
):
184-
self._test_layernorm_ethosu_BI_pipeline(
185-
self.LayerNorm(*model_params), common.get_u55_compile_spec(), (test_data,)
186-
)
187-
188-
# Numerical issues on FVP likely due to mul op, MLETORCH-521
189-
@parameterized.expand(test_data_suite[:-2])
190-
@pytest.mark.corstone_fvp
191-
@conftest.expectedFailureOnFVP
192-
def test_layer_norm_u85_BI_xfails(
193-
self,
194-
test_name: str,
195-
test_data: torch.Tensor,
196-
model_params,
197-
):
198-
self._test_layernorm_ethosu_BI_pipeline(
199-
self.LayerNorm(*model_params), common.get_u85_compile_spec(), (test_data,)
200-
)
201-
202-
@parameterized.expand(test_data_suite[-2:])
173+
@parameterized.expand(test_data_suite)
203174
@pytest.mark.corstone_fvp
204175
def test_layer_norm_u85_BI(
205176
self,

‎backends/arm/test/ops/test_logsoftmax.py‎

Lines changed: 18 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
import pytest
1212

1313
import torch
14-
from executorch.backends.arm.test import common
14+
from executorch.backends.arm.test import common, conftest
1515
from executorch.backends.arm.test.tester.arm_tester import ArmTester
1616
from executorch.exir.backend.compile_spec_schema import CompileSpec
1717
from parameterized import parameterized
@@ -28,16 +28,17 @@
2828
lambda: ("randn", torch.randn(10, 10, 10, 10), 3),
2929
lambda: ("randn_neg_dim", torch.randn(10, 5, 8, 7), -3),
3030
]
31-
test_data_generators_u55 = [
31+
32+
test_data_generators_FVP = [
3233
# (test_name, test_data, dim)
3334
lambda: ("ones", torch.ones(10, 10), 1),
3435
lambda: ("ones_neg_dim", torch.ones(10, 3, 4), -1),
35-
lambda: ("randn_neg_dim", torch.randn(10, 5, 8, 7), -3),
36-
lambda: ("zeros", torch.zeros(10, 8, 5, 2), 0),
37-
lambda: ("zeros_neg_dim", torch.zeros(10, 7, 8, 9), -4),
36+
lambda: ("randn_neg_dim", torch.randn(1, 5, 8, 7), -3),
37+
lambda: ("zeros", torch.zeros(1, 8, 5, 2), 0),
38+
lambda: ("zeros_neg_dim", torch.zeros(1, 7, 8, 9), -4),
3839
lambda: ("rand", torch.rand(1, 2, 5, 8), 2),
39-
lambda: ("rand_neg_dim", torch.rand(2, 10, 8, 10), -2),
40-
lambda: ("randn", torch.randn(10, 10, 10, 10), 3),
40+
lambda: ("rand_neg_dim", torch.rand(1, 10, 8, 10), -2),
41+
lambda: ("randn", torch.randn(1, 10, 10, 10), 3),
4142
]
4243

4344

@@ -99,7 +100,7 @@ def _test_logsoftmax_tosa_ethos_BI_pipeline(
99100
module: torch.nn.Module,
100101
test_data: Tuple[torch.tensor],
101102
):
102-
(
103+
tester = (
103104
ArmTester(
104105
module,
105106
example_inputs=test_data,
@@ -114,21 +115,10 @@ def _test_logsoftmax_tosa_ethos_BI_pipeline(
114115
.check_not(["executorch_exir_dialects_edge__ops_aten__logsoftmax_default"])
115116
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
116117
.to_executorch()
118+
.serialize()
117119
)
118-
119-
def _test_logsoftmax_tosa_u55_BI_pipeline(
120-
self, module: torch.nn.Module, test_data: Tuple[torch.tensor]
121-
):
122-
self._test_logsoftmax_tosa_ethos_BI_pipeline(
123-
common.get_u55_compile_spec(), module, test_data
124-
)
125-
126-
def _test_logsoftmax_tosa_u85_BI_pipeline(
127-
self, module: torch.nn.Module, test_data: Tuple[torch.tensor]
128-
):
129-
self._test_logsoftmax_tosa_ethos_BI_pipeline(
130-
common.get_u85_compile_spec(), module, test_data
131-
)
120+
if conftest.is_option_enabled("corstone_fvp"):
121+
tester.run_method_and_compare_outputs(inputs=test_data, qtol=1)
132122

133123
@parameterized.expand(test_data_generators)
134124
def test_logsoftmax_tosa_MI(self, test_data_generator: Callable[[], Tuple]):
@@ -141,18 +131,18 @@ def test_logsoftmax_tosa_BI(self, test_data_generator: Callable[[], Tuple]):
141131
test_name, test_data, dim = test_data_generator()
142132
self._test_logsoftmax_tosa_BI_pipeline(self.LogSoftmax(dim=dim), (test_data,))
143133

144-
@parameterized.expand(test_data_generators_u55)
134+
@parameterized.expand(test_data_generators_FVP)
145135
@pytest.mark.flaky # TODO: MLETORCH-460 - Numerically stabler (log)softmax implementation
146136
def test_logsoftmax_tosa_u55_BI(self, test_data_generator: Callable[[], Tuple]):
147137
test_name, test_data, dim = test_data_generator()
148-
self._test_logsoftmax_tosa_u55_BI_pipeline(
149-
self.LogSoftmax(dim=dim), (test_data,)
138+
self._test_logsoftmax_tosa_ethos_BI_pipeline(
139+
common.get_u55_compile_spec(), self.LogSoftmax(dim=dim), (test_data,)
150140
)
151141

152-
@parameterized.expand(test_data_generators)
142+
@parameterized.expand(test_data_generators_FVP)
153143
@pytest.mark.flaky # TODO: MLETORCH-460 - Numerically stabler (log)softmax implementation
154144
def test_logsoftmax_tosa_u85_BI(self, test_data_generator: Callable[[], Tuple]):
155145
test_name, test_data, dim = test_data_generator()
156-
self._test_logsoftmax_tosa_u85_BI_pipeline(
157-
self.LogSoftmax(dim=dim), (test_data,)
146+
self._test_logsoftmax_tosa_ethos_BI_pipeline(
147+
common.get_u85_compile_spec(), self.LogSoftmax(dim=dim), (test_data,)
158148
)

‎backends/arm/test/ops/test_mean_dim.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from typing import Tuple
1111

1212
import torch
13-
from executorch.backends.arm.test import common
13+
from executorch.backends.arm.test import common, conftest
1414
from executorch.backends.arm.test.tester.arm_tester import ArmTester
1515
from executorch.exir.backend.backend_details import CompileSpec
1616
from parameterized import parameterized
@@ -121,7 +121,7 @@ def _test_adaptive_avg_pool2d_tosa_ethosu_BI_pipeline(
121121
compile_spec: CompileSpec,
122122
test_data: Tuple[torch.tensor],
123123
):
124-
(
124+
tester = (
125125
ArmTester(
126126
module,
127127
example_inputs=test_data,
@@ -141,7 +141,10 @@ def _test_adaptive_avg_pool2d_tosa_ethosu_BI_pipeline(
141141
)
142142
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
143143
.to_executorch()
144+
.serialize()
144145
)
146+
if conftest.is_option_enabled("corstone_fvp"):
147+
tester.run_method_and_compare_outputs(inputs=test_data)
145148

146149
def _test_meandim_tosa_MI_pipeline(
147150
self, module: torch.nn.Module, test_data: Tuple[torch.tensor]
@@ -188,7 +191,7 @@ def _test_meandim_tosa_ethosu_BI_pipeline(
188191
compile_spec: CompileSpec,
189192
test_data: Tuple[torch.tensor],
190193
):
191-
(
194+
tester = (
192195
ArmTester(
193196
module,
194197
example_inputs=test_data,
@@ -207,7 +210,10 @@ def _test_meandim_tosa_ethosu_BI_pipeline(
207210
)
208211
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
209212
.to_executorch()
213+
.serialize()
210214
)
215+
if conftest.is_option_enabled("corstone_fvp"):
216+
tester.run_method_and_compare_outputs(inputs=test_data, qtol=1)
211217

212218
@parameterized.expand(AdaptiveAveragePool2d.test_data_suite)
213219
def test_adaptive_avg_pool2d_tosa_MI(

‎backends/arm/test/ops/test_mm.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,18 +132,16 @@ def test_mm_single_input_tosa_BI(self, test_data_generator: Callable[[], Tuple])
132132
test_data = test_data_generator()
133133
self._test_mm_tosa_BI_pipeline(self.MMSingleInput(), test_data)
134134

135-
# Expected to fail with error: CPU performance estimation for "MatMul" not implemented
135+
# TODO: Enable numerical testing
136136
@parameterized.expand(MM.test_data_generators)
137-
@unittest.expectedFailure
138137
def test_mm_u55_BI(self, test_data_generator: Callable[[], Tuple]):
139138
test_data = test_data_generator()
140139
self._test_mm_ethosu_BI_pipeline(
141140
common.get_u55_compile_spec(), self.MM(), test_data
142141
)
143142

144-
# Expected to fail with error: Warning, unsupported fusing of TOSA Rescale previous operator is of type: Memcpy
143+
# TODO: Enable numerical testing
145144
@parameterized.expand(MMSingleInput.test_data_generators)
146-
@unittest.expectedFailure
147145
def test_mm_single_input_u55_BI(self, test_data_generator: Callable[[], Tuple]):
148146
test_data = test_data_generator()
149147
self._test_mm_ethosu_BI_pipeline(

‎backends/arm/test/ops/test_softmax.py‎

Lines changed: 23 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import pytest
1313

1414
import torch
15-
from executorch.backends.arm.test import common
15+
from executorch.backends.arm.test import common, conftest
1616
from executorch.backends.arm.test.tester.arm_tester import ArmTester
1717
from executorch.exir.backend.compile_spec_schema import CompileSpec
1818
from parameterized import parameterized
@@ -30,16 +30,16 @@
3030
lambda: ("randn_neg_dim", torch.randn(10, 5, 8, 7), -3),
3131
]
3232

33-
test_data_generators_u55 = [
33+
test_data_generators_FVP = [
3434
# (test_name, test_data, dim)
3535
lambda: ("ones", torch.ones(10, 10), 1),
36-
lambda: ("ones_neg_dim", torch.ones(10, 3, 4), -1),
37-
lambda: ("randn_neg_dim", torch.randn(10, 5, 8, 7), -3),
38-
lambda: ("zeros", torch.zeros(10, 8, 5, 2), 0),
39-
lambda: ("zeros_neg_dim", torch.zeros(10, 7, 8, 9), -4),
36+
lambda: ("ones_neg_dim", torch.ones(1, 3, 4), -1),
37+
lambda: ("randn_neg_dim", torch.randn(1, 5, 8, 7), -3),
38+
lambda: ("zeros", torch.zeros(1, 8, 5, 2), 0),
39+
lambda: ("zeros_neg_dim", torch.zeros(1, 7, 8, 9), -4),
4040
lambda: ("rand", torch.rand(1, 2, 5, 8), 2),
41-
lambda: ("rand_neg_dim", torch.rand(2, 10, 8, 10), -2),
42-
lambda: ("randn", torch.randn(10, 10, 10, 10), 3),
41+
lambda: ("rand_neg_dim", torch.rand(1, 10, 8, 10), -2),
42+
lambda: ("randn", torch.randn(1, 10, 10, 10), 3),
4343
]
4444

4545

@@ -95,13 +95,13 @@ def _test_softmax_tosa_BI_pipeline(
9595
.run_method_and_compare_outputs(inputs=test_data)
9696
)
9797

98-
def _test_softmax_tosa_ethos_BI_pipeline(
98+
def _test_softmax_ethosu_BI_pipeline(
9999
self,
100100
compile_spec: list[CompileSpec],
101101
module: torch.nn.Module,
102102
test_data: Tuple[torch.tensor],
103103
):
104-
(
104+
tester = (
105105
ArmTester(
106106
module,
107107
example_inputs=test_data,
@@ -116,21 +116,10 @@ def _test_softmax_tosa_ethos_BI_pipeline(
116116
.check_not(["executorch_exir_dialects_edge__ops_aten__softmax_default"])
117117
.check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
118118
.to_executorch()
119+
.serialize()
119120
)
120-
121-
def _test_softmax_tosa_u55_BI_pipeline(
122-
self, module: torch.nn.Module, test_data: Tuple[torch.tensor]
123-
):
124-
self._test_softmax_tosa_ethos_BI_pipeline(
125-
common.get_u55_compile_spec(), module, test_data
126-
)
127-
128-
def _test_softmax_tosa_u85_BI_pipeline(
129-
self, module: torch.nn.Module, test_data: Tuple[torch.tensor]
130-
):
131-
self._test_softmax_tosa_ethos_BI_pipeline(
132-
common.get_u85_compile_spec(), module, test_data
133-
)
121+
if conftest.is_option_enabled("corstone_fvp"):
122+
tester.run_method_and_compare_outputs(inputs=test_data, qtol=1)
134123

135124
@parameterized.expand(test_data_generators)
136125
def test_softmax_tosa_MI(self, test_data_generator: Callable[[], Tuple]):
@@ -143,14 +132,18 @@ def test_softmax_tosa_BI(self, test_data_generator: Callable[[], Tuple]):
143132
test_name, test_data, dim = test_data_generator()
144133
self._test_softmax_tosa_BI_pipeline(self.Softmax(dim=dim), (test_data,))
145134

146-
@parameterized.expand(test_data_generators_u55)
135+
@parameterized.expand(test_data_generators_FVP)
147136
@pytest.mark.flaky # TODO: MLETORCH-460 - Numerically stabler (log)softmax implementation
148-
def test_softmax_tosa_u55_BI(self, test_data_generator: Callable[[], Tuple]):
137+
def test_softmax_u55_BI(self, test_data_generator: Callable[[], Tuple]):
149138
test_name, test_data, dim = test_data_generator()
150-
self._test_softmax_tosa_u55_BI_pipeline(self.Softmax(dim=dim), (test_data,))
139+
self._test_softmax_ethosu_BI_pipeline(
140+
common.get_u55_compile_spec(), self.Softmax(dim=dim), (test_data,)
141+
)
151142

152-
@parameterized.expand(test_data_generators)
143+
@parameterized.expand(test_data_generators_FVP)
153144
@pytest.mark.flaky # TODO: MLETORCH-460 - Numerically stabler (log)softmax implementation
154-
def test_softmax_tosa_u85_BI(self, test_data_generator: Callable[[], Tuple]):
145+
def test_softmax_u85_BI(self, test_data_generator: Callable[[], Tuple]):
155146
test_name, test_data, dim = test_data_generator()
156-
self._test_softmax_tosa_u85_BI_pipeline(self.Softmax(dim=dim), (test_data,))
147+
self._test_softmax_ethosu_BI_pipeline(
148+
common.get_u85_compile_spec(), self.Softmax(dim=dim), (test_data,)
149+
)

0 commit comments

Comments
 (0)