Skip to content

Commit e376338

Browse files
GregoryComerGregoryComer
authored
[Backend Tester] Clean up a few test issues (#13258)
There are a few broken tests that need cleaning up. Some are failing due to missing portable kernels. These tests are now skipped if any unsupported portable ops remain post-delegation. I also fixed a few other small issues and bumped the element-wise tolerance to reduce false positives. SNR should hopefully catch most blatant correctness issues. The fp16 and quantized tests can generate occasional high element-wise error but still have decent SNR (~60+).
1 parent 1d5a214 commit e376338

File tree

9 files changed

+64
-47
lines changed

9 files changed

+64
-47
lines changed

‎backends/test/harness/stages/to_edge_transform_and_lower.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@ def __init__(
2323
if default_partitioner_cls is not None
2424
else []
2525
)
26-
self.edge_compile_conf = edge_compile_config or EdgeCompileConfig()
26+
self.edge_compile_conf = edge_compile_config or EdgeCompileConfig(
27+
_check_ir_validity=False
28+
)
2729
self.edge_dialect_program = None
2830

2931
def stage_type(self) -> StageType:

‎backends/test/suite/operators/test_amax.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -207,39 +207,39 @@ def test_amax_edge_cases(self, flow: TestFlow) -> None:
207207
AmaxModel(),
208208
(x,),
209209
flow,
210-
use_random_test_inputs=False,
210+
generate_random_test_inputs=False,
211211
)
212212
self._test_op(
213213
AmaxModel(dim=0),
214214
(x,),
215215
flow,
216-
use_random_test_inputs=False,
216+
generate_random_test_inputs=False,
217217
)
218218
self._test_op(
219219
AmaxModel(dim=1),
220220
(x,),
221221
flow,
222-
use_random_test_inputs=False,
222+
generate_random_test_inputs=False,
223223
)
224224

225225
x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]])
226226
self._test_op(
227227
AmaxModel(),
228228
(x,),
229229
flow,
230-
use_random_test_inputs=False,
230+
generate_random_test_inputs=False,
231231
)
232232
self._test_op(
233233
AmaxModel(dim=0),
234234
(x,),
235235
flow,
236-
use_random_test_inputs=False,
236+
generate_random_test_inputs=False,
237237
)
238238
self._test_op(
239239
AmaxModel(dim=1),
240240
(x,),
241241
flow,
242-
use_random_test_inputs=False,
242+
generate_random_test_inputs=False,
243243
)
244244

245245
def test_amax_scalar(self, flow: TestFlow) -> None:

‎backends/test/suite/operators/test_amin.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -209,39 +209,39 @@ def test_amin_edge_cases(self, flow: TestFlow) -> None:
209209
AminModel(),
210210
(x,),
211211
flow,
212-
use_random_test_inputs=False,
212+
generate_random_test_inputs=False,
213213
)
214214
self._test_op(
215215
AminModel(dim=0),
216216
(x,),
217217
flow,
218-
use_random_test_inputs=False,
218+
generate_random_test_inputs=False,
219219
)
220220
self._test_op(
221221
AminModel(dim=1),
222222
(x,),
223223
flow,
224-
use_random_test_inputs=False,
224+
generate_random_test_inputs=False,
225225
)
226226

227227
x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]])
228228
self._test_op(
229229
AminModel(),
230230
(x,),
231231
flow,
232-
use_random_test_inputs=False,
232+
generate_random_test_inputs=False,
233233
)
234234
self._test_op(
235235
AminModel(dim=0),
236236
(x,),
237237
flow,
238-
use_random_test_inputs=False,
238+
generate_random_test_inputs=False,
239239
)
240240
self._test_op(
241241
AminModel(dim=1),
242242
(x,),
243243
flow,
244-
use_random_test_inputs=False,
244+
generate_random_test_inputs=False,
245245
)
246246

247247
def test_amin_scalar(self, flow: TestFlow) -> None:

‎backends/test/suite/operators/test_argmax.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -149,39 +149,39 @@ def test_argmax_edge_cases(self, flow: TestFlow) -> None:
149149
ArgmaxModel(),
150150
(x,),
151151
flow,
152-
use_random_test_inputs=False,
152+
generate_random_test_inputs=False,
153153
)
154154
self._test_op(
155155
ArgmaxModel(dim=0),
156156
(x,),
157157
flow,
158-
use_random_test_inputs=False,
158+
generate_random_test_inputs=False,
159159
)
160160
self._test_op(
161161
ArgmaxModel(dim=1),
162162
(x,),
163163
flow,
164-
use_random_test_inputs=False,
164+
generate_random_test_inputs=False,
165165
)
166166

167167
x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]])
168168
self._test_op(
169169
ArgmaxModel(),
170170
(x,),
171171
flow,
172-
use_random_test_inputs=False,
172+
generate_random_test_inputs=False,
173173
)
174174
self._test_op(
175175
ArgmaxModel(dim=0),
176176
(x,),
177177
flow,
178-
use_random_test_inputs=False,
178+
generate_random_test_inputs=False,
179179
)
180180
self._test_op(
181181
ArgmaxModel(dim=1),
182182
(x,),
183183
flow,
184-
use_random_test_inputs=False,
184+
generate_random_test_inputs=False,
185185
)
186186

187187
x = torch.tensor([5.0])

‎backends/test/suite/operators/test_argmin.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -149,39 +149,39 @@ def test_argmin_edge_cases(self, flow: TestFlow) -> None:
149149
ArgminModel(),
150150
(x,),
151151
flow,
152-
use_random_test_inputs=False,
152+
generate_random_test_inputs=False,
153153
)
154154
self._test_op(
155155
ArgminModel(dim=0),
156156
(x,),
157157
flow,
158-
use_random_test_inputs=False,
158+
generate_random_test_inputs=False,
159159
)
160160
self._test_op(
161161
ArgminModel(dim=1),
162162
(x,),
163163
flow,
164-
use_random_test_inputs=False,
164+
generate_random_test_inputs=False,
165165
)
166166

167167
x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]])
168168
self._test_op(
169169
ArgminModel(),
170170
(x,),
171171
flow,
172-
use_random_test_inputs=False,
172+
generate_random_test_inputs=False,
173173
)
174174
self._test_op(
175175
ArgminModel(dim=0),
176176
(x,),
177177
flow,
178-
use_random_test_inputs=False,
178+
generate_random_test_inputs=False,
179179
)
180180
self._test_op(
181181
ArgminModel(dim=1),
182182
(x,),
183183
flow,
184-
use_random_test_inputs=False,
184+
generate_random_test_inputs=False,
185185
)
186186

187187
x = torch.tensor([5.0])

‎backends/test/suite/operators/test_floor.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818

1919

2020
class FloorModel(torch.nn.Module):
21-
def __init__(self):
22-
super().__init__()
21+
def forward(self, x):
22+
return torch.floor(x)
2323

2424

2525
@operator_test

‎backends/test/suite/reporting.py‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,35 +28,32 @@ class TestResult(IntEnum):
2828
SUCCESS_UNDELEGATED = 1
2929
""" The test succeeded without the backend delegating anything. """
3030

31-
EAGER_FAIL = 2
32-
""" The test failed due to the model failing to run in eager mode. """
31+
SKIPPED = 2
32+
""" The test was skipped due to a non-backend failure. """
3333

3434
QUANTIZE_FAIL = 3
3535
""" The test failed due to the quantization stage failing. """
3636

37-
EXPORT_FAIL = 4
38-
""" The test failed due to the model failing to export. """
39-
40-
LOWER_FAIL = 5
37+
LOWER_FAIL = 4
4138
""" The test failed due to a failure in partitioning or lowering. """
4239

43-
PTE_LOAD_FAIL = 6
40+
PTE_LOAD_FAIL = 5
4441
""" The test failed due to the resulting PTE failing to load. """
4542

46-
PTE_RUN_FAIL = 7
43+
PTE_RUN_FAIL = 6
4744
""" The test failed due to the resulting PTE failing to run. """
4845

49-
OUTPUT_MISMATCH_FAIL = 8
46+
OUTPUT_MISMATCH_FAIL = 7
5047
""" The test failed due to a mismatch between runtime and reference outputs. """
5148

52-
UNKNOWN_FAIL = 9
49+
UNKNOWN_FAIL = 8
5350
""" The test failed in an unknown or unexpected manner. """
5451

5552
def is_success(self):
5653
return self in {TestResult.SUCCESS, TestResult.SUCCESS_UNDELEGATED}
5754

5855
def is_non_backend_failure(self):
59-
return self in {TestResult.EAGER_FAIL, TestResult.EAGER_FAIL}
56+
return self in {TestResult.SKIPPED}
6057

6158
def is_backend_failure(self):
6259
return not self.is_success() and not self.is_non_backend_failure()
@@ -66,12 +63,10 @@ def display_name(self):
6663
return "Success (Delegated)"
6764
elif self == TestResult.SUCCESS_UNDELEGATED:
6865
return "Success (Undelegated)"
69-
elif self == TestResult.EAGER_FAIL:
70-
return "Fail (Eager)"
66+
elif self == TestResult.SKIPPED:
67+
return "Skipped"
7168
elif self == TestResult.QUANTIZE_FAIL:
7269
return "Fail (Quantize)"
73-
elif self == TestResult.EXPORT_FAIL:
74-
return "Fail (Export)"
7570
elif self == TestResult.LOWER_FAIL:
7671
return "Fail (Lowering)"
7772
elif self == TestResult.PTE_LOAD_FAIL:

‎backends/test/suite/runner.py‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,14 @@
99

1010
import torch
1111

12+
# Set of unsupported ops that should cause tests to be skipped
13+
UNSUPPORTED_PORTABLE_OPS = {
14+
"aten::_embedding_bag",
15+
"aten::median",
16+
"aten::median.dim",
17+
"aten::round.decimals",
18+
}
19+
1220
from executorch.backends.test.harness.error_statistics import ErrorStatistics
1321
from executorch.backends.test.harness.stages import StageType
1422
from executorch.backends.test.suite.discovery import discover_tests, TestFilter
@@ -70,7 +78,7 @@ def build_result(
7078
try:
7179
model(*inputs)
7280
except Exception as e:
73-
return build_result(TestResult.EAGER_FAIL, e)
81+
return build_result(TestResult.SKIPPED, e)
7482

7583
try:
7684
tester = flow.tester_factory(model, inputs)
@@ -96,7 +104,7 @@ def build_result(
96104
tester._get_default_stage(StageType.EXPORT, dynamic_shapes=dynamic_shapes),
97105
)
98106
except Exception as e:
99-
return build_result(TestResult.EXPORT_FAIL, e)
107+
return build_result(TestResult.SKIPPED, e)
100108

101109
lower_start_time = time.perf_counter()
102110
try:
@@ -125,7 +133,16 @@ def build_result(
125133
if n.op == "call_function"
126134
)
127135

128-
# Only run the runtime portion if something was delegated (or the flow doesn't delegate).
136+
# Check if any undelegated ops are in the unsupported ops set.
137+
has_unsupported_ops = any(
138+
op in UNSUPPORTED_PORTABLE_OPS for op in undelegated_op_counts.keys()
139+
)
140+
141+
# Skip the test if there are unsupported portable ops remaining.
142+
if has_unsupported_ops:
143+
return build_result(TestResult.SKIPPED)
144+
145+
# Only run the runtime portion if something was delegated (or the flow doesn't delegate)
129146
if is_delegated or not flow.is_delegated:
130147
try:
131148
tester.to_executorch().serialize()
@@ -142,12 +159,15 @@ def build_result(
142159
tester.run_method_and_compare_outputs(
143160
inputs=None if generate_random_test_inputs else inputs,
144161
statistics_callback=lambda stats: error_statistics.append(stats),
162+
atol=1e-1,
163+
rtol=4e-2,
145164
)
146165
except AssertionError as e:
147166
return build_result(TestResult.OUTPUT_MISMATCH_FAIL, e)
148167
except Exception as e:
149168
return build_result(TestResult.PTE_RUN_FAIL, e)
150169
else:
170+
# Skip the test if nothing is delegated
151171
return build_result(TestResult.SUCCESS_UNDELEGATED)
152172

153173
return build_result(TestResult.SUCCESS)

‎backends/test/suite/tests/test_reporting.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@
5454
flow="flow1",
5555
name="test2_backend2_flow1",
5656
params={"use_dynamic_shapes": True},
57-
result=TestResult.EXPORT_FAIL,
57+
result=TestResult.SKIPPED,
5858
error=None,
5959
tensor_error_statistics=[],
6060
),
@@ -108,7 +108,7 @@ def test_csv_report_simple(self):
108108
self.assertEqual(records[3]["Test Case"], "test2")
109109
self.assertEqual(records[3]["Backend"], "backend2")
110110
self.assertEqual(records[3]["Flow"], "flow1")
111-
self.assertEqual(records[3]["Result"], "Fail (Export)")
111+
self.assertEqual(records[3]["Result"], "Skipped")
112112
self.assertEqual(records[3]["Dtype"], "")
113113
self.assertEqual(records[3]["Use_dynamic_shapes"], "True")
114114

0 commit comments

Comments
 (0)