Update

[ghstack-poisoned]

Author: GregoryComer

.ci/scripts/test_backend.sh

@@ -63,7 +63,7 @@ if [[ $IS_MACOS -eq 1 ]]; then
 else
     SETUP_SCRIPT=.ci/scripts/setup-linux.sh
 fi
-${CONDA_PREFIX} CMAKE_ARGS="$EXTRA_BUILD_ARGS" $SETUP_SCRIPT --build-tool cmake --build-mode Release --editable true
+CMAKE_ARGS="$EXTRA_BUILD_ARGS" ${CONDA_PREFIX} $SETUP_SCRIPT --build-tool cmake --build-mode Release --editable true
 
 EXIT_CODE=0
 ${CONDA_PREFIX} pytest -c /dev/nul -n auto backends/test/suite/$SUITE/ -m flow_$FLOW --json-report --json-report-file="$REPORT_FILE" || EXIT_CODE=$?

backends/test/suite/conftest.py

@@ -84,6 +84,7 @@ def test_runner(request):
 
 @pytest.hookimpl(optionalhook=True)
 def pytest_json_runtest_metadata(item, call):
+    # Store detailed results in the test report under the metadata key.
     metadata = {"subtests": []}
 
     if hasattr(item, "funcargs") and "test_runner" in item.funcargs:
@@ -143,5 +144,39 @@ def pytest_json_runtest_metadata(item, call):
             )
 
             metadata["subtests"].append(subtest_metadata)
-
     return metadata
+
+
+@pytest.hookimpl(optionalhook=True)
+def pytest_json_modifyreport(json_report):
+    # Post-process the report, mainly to populate metadata for crashed tests. The runtest_metadata
+    # hook doesn't seem to be called when there's a native crash, but xdist still creates a report
+    # entry.
+
+    for test_data in json_report["tests"]:
+        if "metadata" not in test_data:
+            test_data["metadata"] = {}
+        metadata = test_data["metadata"]
+        if "subtests" not in metadata:
+            metadata["subtests"] = []
+        subtests = metadata["subtests"]
+
+        # Native crashes are recorded differently and won't have the full metadata.
+        # Pytest-xdist records crash info under the "???" key.
+        if "???" in test_data:
+            test_id = test_data["nodeid"].strip("::")  # Remove leading ::
+            test_base_id = test_id.split("[")[
+                0
+            ]  # Strip parameterization to get the base test case
+            params = test_id[len(test_base_id) + 1 : -1].split("-")
+            flow = params[0]
+
+            crashed_test_meta = {
+                "Test ID": test_id,
+                "Test Case": test_base_id,
+                "Flow": flow,
+                "Result": "Fail",
+                "Result Detail": "Process Crash",
+                "Error": test_data["???"].get("longrepr", "Process crashed."),
+            }
+            subtests.append(crashed_test_meta)

backends/test/suite/generate_markdown_summary_json.py

@@ -70,25 +70,24 @@ def aggregate_results(json_path: str) -> AggregatedSummary:
     counts_by_param = {}
 
     for test_data in data["tests"]:
-        result_meta = test_data.get("metadata")
-        if result_meta:
-            for subtest_meta in result_meta["subtests"]:
-                result = subtest_meta["Result"]
-                result_detail = subtest_meta.get("Result Detail") or ""
+        result_meta = test_data["metadata"]
+        for subtest_meta in result_meta["subtests"]:
+            result = subtest_meta["Result"]
+            result_detail = subtest_meta.get("Result Detail") or ""
 
-                counts.add_row(result, result_detail)
+            counts.add_row(result, result_detail)
 
-                test_id = subtest_meta["Test ID"]
-                base_test = subtest_meta["Test Case"]
-                params = test_id[len(base_test) + 1 : -1]
+            test_id = subtest_meta["Test ID"]
+            base_test = subtest_meta["Test Case"]
+            params = test_id[len(base_test) + 1 : -1]
 
-                if params:
-                    if params not in counts_by_param:
-                        counts_by_param[params] = ResultCounts()
-                    counts_by_param[params].add_row(result, result_detail)
+            if params:
+                if params not in counts_by_param:
+                    counts_by_param[params] = ResultCounts()
+                counts_by_param[params].add_row(result, result_detail)
 
-                if result.lower() == "fail":
-                    failed_tests.append(subtest_meta)
+            if result.lower() == "fail":
+                failed_tests.append(subtest_meta)
 
     return AggregatedSummary(
         counts=counts,