diff --git a/backends/test/harness/stages/to_edge_transform_and_lower.py b/backends/test/harness/stages/to_edge_transform_and_lower.py index 16b5ad086aa..19a6b6033c5 100644 --- a/backends/test/harness/stages/to_edge_transform_and_lower.py +++ b/backends/test/harness/stages/to_edge_transform_and_lower.py @@ -23,7 +23,9 @@ def __init__( if default_partitioner_cls is not None else [] ) - self.edge_compile_conf = edge_compile_config or EdgeCompileConfig() + self.edge_compile_conf = edge_compile_config or EdgeCompileConfig( + _check_ir_validity=False + ) self.edge_dialect_program = None def stage_type(self) -> StageType: diff --git a/backends/test/suite/operators/test_amax.py b/backends/test/suite/operators/test_amax.py index aff33476e69..0c9a8c06f0d 100644 --- a/backends/test/suite/operators/test_amax.py +++ b/backends/test/suite/operators/test_amax.py @@ -207,19 +207,19 @@ def test_amax_edge_cases(self, flow: TestFlow) -> None: AmaxModel(), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( AmaxModel(dim=0), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( AmaxModel(dim=1), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]]) @@ -227,19 +227,19 @@ def test_amax_edge_cases(self, flow: TestFlow) -> None: AmaxModel(), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( AmaxModel(dim=0), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( AmaxModel(dim=1), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) def test_amax_scalar(self, flow: TestFlow) -> None: diff --git a/backends/test/suite/operators/test_amin.py b/backends/test/suite/operators/test_amin.py index ab59d77d0be..f4b88b1dade 100644 --- a/backends/test/suite/operators/test_amin.py +++ b/backends/test/suite/operators/test_amin.py @@ -209,19 +209,19 @@ def test_amin_edge_cases(self, flow: TestFlow) -> None: AminModel(), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( AminModel(dim=0), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( AminModel(dim=1), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]]) @@ -229,19 +229,19 @@ def test_amin_edge_cases(self, flow: TestFlow) -> None: AminModel(), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( AminModel(dim=0), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( AminModel(dim=1), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) def test_amin_scalar(self, flow: TestFlow) -> None: diff --git a/backends/test/suite/operators/test_argmax.py b/backends/test/suite/operators/test_argmax.py index adf1e43a340..dc8b57fc214 100644 --- a/backends/test/suite/operators/test_argmax.py +++ b/backends/test/suite/operators/test_argmax.py @@ -149,19 +149,19 @@ def test_argmax_edge_cases(self, flow: TestFlow) -> None: ArgmaxModel(), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( ArgmaxModel(dim=0), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( ArgmaxModel(dim=1), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]]) @@ -169,19 +169,19 @@ def test_argmax_edge_cases(self, flow: TestFlow) -> None: ArgmaxModel(), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( ArgmaxModel(dim=0), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( ArgmaxModel(dim=1), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) x = torch.tensor([5.0]) diff --git a/backends/test/suite/operators/test_argmin.py b/backends/test/suite/operators/test_argmin.py index 0613c74a3ee..d7a24e24f5a 100644 --- a/backends/test/suite/operators/test_argmin.py +++ b/backends/test/suite/operators/test_argmin.py @@ -149,19 +149,19 @@ def test_argmin_edge_cases(self, flow: TestFlow) -> None: ArgminModel(), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( ArgminModel(dim=0), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( ArgminModel(dim=1), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) x = torch.tensor([[1.0, float("nan"), 3.0], [4.0, 5.0, float("nan")]]) @@ -169,19 +169,19 @@ def test_argmin_edge_cases(self, flow: TestFlow) -> None: ArgminModel(), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( ArgminModel(dim=0), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) self._test_op( ArgminModel(dim=1), (x,), flow, - use_random_test_inputs=False, + generate_random_test_inputs=False, ) x = torch.tensor([5.0]) diff --git a/backends/test/suite/operators/test_floor.py b/backends/test/suite/operators/test_floor.py index e5da5da63df..fcc834afa16 100644 --- a/backends/test/suite/operators/test_floor.py +++ b/backends/test/suite/operators/test_floor.py @@ -18,8 +18,8 @@ class FloorModel(torch.nn.Module): - def __init__(self): - super().__init__() + def forward(self, x): + return torch.floor(x) @operator_test diff --git a/backends/test/suite/reporting.py b/backends/test/suite/reporting.py index 6981047b580..93a93f76283 100644 --- a/backends/test/suite/reporting.py +++ b/backends/test/suite/reporting.py @@ -28,35 +28,32 @@ class TestResult(IntEnum): SUCCESS_UNDELEGATED = 1 """ The test succeeded without the backend delegating anything. """ - EAGER_FAIL = 2 - """ The test failed due to the model failing to run in eager mode. """ + SKIPPED = 2 + """ The test was skipped due to a non-backend failure. """ QUANTIZE_FAIL = 3 """ The test failed due to the quantization stage failing. """ - EXPORT_FAIL = 4 - """ The test failed due to the model failing to export. """ - - LOWER_FAIL = 5 + LOWER_FAIL = 4 """ The test failed due to a failure in partitioning or lowering. """ - PTE_LOAD_FAIL = 6 + PTE_LOAD_FAIL = 5 """ The test failed due to the resulting PTE failing to load. """ - PTE_RUN_FAIL = 7 + PTE_RUN_FAIL = 6 """ The test failed due to the resulting PTE failing to run. """ - OUTPUT_MISMATCH_FAIL = 8 + OUTPUT_MISMATCH_FAIL = 7 """ The test failed due to a mismatch between runtime and reference outputs. """ - UNKNOWN_FAIL = 9 + UNKNOWN_FAIL = 8 """ The test failed in an unknown or unexpected manner. """ def is_success(self): return self in {TestResult.SUCCESS, TestResult.SUCCESS_UNDELEGATED} def is_non_backend_failure(self): - return self in {TestResult.EAGER_FAIL, TestResult.EAGER_FAIL} + return self in {TestResult.SKIPPED} def is_backend_failure(self): return not self.is_success() and not self.is_non_backend_failure() @@ -66,12 +63,10 @@ def display_name(self): return "Success (Delegated)" elif self == TestResult.SUCCESS_UNDELEGATED: return "Success (Undelegated)" - elif self == TestResult.EAGER_FAIL: - return "Fail (Eager)" + elif self == TestResult.SKIPPED: + return "Skipped" elif self == TestResult.QUANTIZE_FAIL: return "Fail (Quantize)" - elif self == TestResult.EXPORT_FAIL: - return "Fail (Export)" elif self == TestResult.LOWER_FAIL: return "Fail (Lowering)" elif self == TestResult.PTE_LOAD_FAIL: diff --git a/backends/test/suite/runner.py b/backends/test/suite/runner.py index 5e4f1dcf32a..101e168476b 100644 --- a/backends/test/suite/runner.py +++ b/backends/test/suite/runner.py @@ -9,6 +9,14 @@ import torch +# Set of unsupported ops that should cause tests to be skipped +UNSUPPORTED_PORTABLE_OPS = { + "aten::_embedding_bag", + "aten::median", + "aten::median.dim", + "aten::round.decimals", +} + from executorch.backends.test.harness.error_statistics import ErrorStatistics from executorch.backends.test.harness.stages import StageType from executorch.backends.test.suite.discovery import discover_tests, TestFilter @@ -70,7 +78,7 @@ def build_result( try: model(*inputs) except Exception as e: - return build_result(TestResult.EAGER_FAIL, e) + return build_result(TestResult.SKIPPED, e) try: tester = flow.tester_factory(model, inputs) @@ -96,7 +104,7 @@ def build_result( tester._get_default_stage(StageType.EXPORT, dynamic_shapes=dynamic_shapes), ) except Exception as e: - return build_result(TestResult.EXPORT_FAIL, e) + return build_result(TestResult.SKIPPED, e) lower_start_time = time.perf_counter() try: @@ -125,7 +133,16 @@ def build_result( if n.op == "call_function" ) - # Only run the runtime portion if something was delegated (or the flow doesn't delegate). + # Check if any undelegated ops are in the unsupported ops set. + has_unsupported_ops = any( + op in UNSUPPORTED_PORTABLE_OPS for op in undelegated_op_counts.keys() + ) + + # Skip the test if there are unsupported portable ops remaining. + if has_unsupported_ops: + return build_result(TestResult.SKIPPED) + + # Only run the runtime portion if something was delegated (or the flow doesn't delegate) if is_delegated or not flow.is_delegated: try: tester.to_executorch().serialize() @@ -142,12 +159,15 @@ def build_result( tester.run_method_and_compare_outputs( inputs=None if generate_random_test_inputs else inputs, statistics_callback=lambda stats: error_statistics.append(stats), + atol=1e-1, + rtol=4e-2, ) except AssertionError as e: return build_result(TestResult.OUTPUT_MISMATCH_FAIL, e) except Exception as e: return build_result(TestResult.PTE_RUN_FAIL, e) else: + # Skip the test if nothing is delegated return build_result(TestResult.SUCCESS_UNDELEGATED) return build_result(TestResult.SUCCESS) diff --git a/backends/test/suite/tests/test_reporting.py b/backends/test/suite/tests/test_reporting.py index 3b711e45949..5eab5648335 100644 --- a/backends/test/suite/tests/test_reporting.py +++ b/backends/test/suite/tests/test_reporting.py @@ -54,7 +54,7 @@ flow="flow1", name="test2_backend2_flow1", params={"use_dynamic_shapes": True}, - result=TestResult.EXPORT_FAIL, + result=TestResult.SKIPPED, error=None, tensor_error_statistics=[], ), @@ -108,7 +108,7 @@ def test_csv_report_simple(self): self.assertEqual(records[3]["Test Case"], "test2") self.assertEqual(records[3]["Backend"], "backend2") self.assertEqual(records[3]["Flow"], "flow1") - self.assertEqual(records[3]["Result"], "Fail (Export)") + self.assertEqual(records[3]["Result"], "Skipped") self.assertEqual(records[3]["Dtype"], "") self.assertEqual(records[3]["Use_dynamic_shapes"], "True")