Update

[ghstack-poisoned]

Author: GregoryComer

backends/test/suite/context.py

@@ -1,13 +1,16 @@
 # Test run context management. This is used to determine the test context for reporting
 # purposes.
 class TestContext:
+    subtest_index: int
+
     def __init__(
         self, test_name: str, test_base_name: str, flow_name: str, params: dict | None
     ):
         self.test_name = test_name
         self.test_base_name = test_base_name
         self.flow_name = flow_name
         self.params = params
+        self.subtest_index = 0
 
     def __enter__(self):
         global _active_test_context

backends/test/suite/flow.py

@@ -1,6 +1,6 @@
 import logging
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import Callable
 
 from executorch.backends.test.harness import Tester

backends/test/suite/operators/__init__.py

@@ -152,12 +152,16 @@ def _test_op(
             flow,
             context.test_name,
             context.test_base_name,
+            context.subtest_index,
             context.params,
             generate_random_test_inputs=generate_random_test_inputs,
         )
 
         log_test_summary(run_summary)
 
+        # This is reset when a new test is started - it creates the context per-test.
+        context.subtest_index = context.subtest_index + 1
+
         if not run_summary.result.is_success():
             if run_summary.result.is_backend_failure():
                 raise RuntimeError("Test failure.") from run_summary.error

backends/test/suite/reporting.py

@@ -21,6 +21,7 @@
 CSV_FIELD_NAMES = [
     "Test ID",
     "Test Case",
+    "Subtest",
     "Flow",
     "Params",
     "Result",
@@ -163,6 +164,9 @@ class TestCaseSummary:
     name: str
     """ The full name of test, including flow and parameter suffixes. """
 
+    subtest_index: int
+    """ The subtest number. If a test case runs multiple tests, this field can be used to disambiguate. """
+
     params: dict | None
     """ Test-specific parameters, such as dtype. """
 
@@ -356,6 +360,7 @@ def write_csv_row(record: TestCaseSummary, output: TextIO):
     row = {
         "Test ID": record.name,
         "Test Case": record.base_name,
+        "Subtest": record.subtest_index,
         "Flow": record.flow,
         "Params": _serialize_params(record.params),
         "Result": record.result.to_short_str(),

backends/test/suite/runner.py

@@ -45,6 +45,7 @@ def run_test(  # noqa: C901
     flow: TestFlow,
     test_name: str,
     test_base_name: str,
+    subtest_index: int,
     params: dict | None,
     dynamic_shapes: Any | None = None,
     generate_random_test_inputs: bool = True,
@@ -64,6 +65,7 @@ def build_result(
         return TestCaseSummary(
             backend=flow.backend,
             base_name=test_base_name,
+            subtest_index=subtest_index,
             flow=flow.name,
             name=test_name,
             params=params,