separate subtests failures to give better indication on tests analysis

Signed-off-by: Roni Kishner<[email protected]>

Author: RoniKishner

AUTHORS

@@ -395,6 +395,7 @@ Roland Puntaier
 Romain Dorgueil
 Roman Bolshakov
 Ronny Pfannschmidt
+Roni Kishner
 Ross Lawley
 Ruaridh Williamson
 Russel Winder

changelog/13986.bugfix.rst

@@ -0,0 +1,6 @@
+Show subtests failures separate from normal test failures in the final test summary.
+
+Subtest failures are now reported separately as "subtests failed" instead of being counted as regular "failed" tests, providing clearer statistics.
+
+For example, a test with 3 subtests where 1 fails and 2 pass now shows:
+``1 failed, 2 subtests passed, 1 subtests failed`` instead of ``2 failed, 2 subtests passed``.

src/_pytest/pytester.py

@@ -510,8 +510,9 @@ def _config_for_test() -> Generator[Config]:
 
 # Regex to match the session duration string in the summary: "74.34s".
 rex_session_duration = re.compile(r"\d+\.\d\ds")
-# Regex to match all the counts and phrases in the summary line: "34 passed, 111 skipped".
-rex_outcome = re.compile(r"(\d+) (\w+)")
+# Regex to match all the counts and phrases in the summary line:
+# "34 passed, 111 skipped, 3 subtests passed, 1 subtests failed".
+rex_outcome = re.compile(r"(\d+) ([\w\s]+?)(?=,| in|$)")
 
 
 @final
@@ -578,14 +579,17 @@ def parse_summary_nouns(cls, lines) -> dict[str, int]:
         for line in reversed(lines):
             if rex_session_duration.search(line):
                 outcomes = rex_outcome.findall(line)
-                ret = {noun: int(count) for (count, noun) in outcomes}
+                ret = {noun.strip(): int(count) for (count, noun) in outcomes}
                 break
         else:
             raise ValueError("Pytest terminal summary report not found")
 
         to_plural = {
             "warning": "warnings",
             "error": "errors",
+            "subtest failed": "subtests failed",
+            "subtest passed": "subtests passed",
+            "subtest skipped": "subtests skipped",
         }
         return {to_plural.get(k, k): v for k, v in ret.items()}
 

src/_pytest/subtests.py

@@ -387,7 +387,7 @@ def pytest_report_teststatus(
             return category, short, f"{status}{description}"
 
         if report.failed:
-            return outcome, "u", f"SUBFAILED{description}"
+            return "subtests failed", "u", f"SUBFAILED{description}"
         else:
             if report.passed:
                 if quiet:

src/_pytest/terminal.py

@@ -8,6 +8,7 @@
 
 import argparse
 from collections import Counter
+from collections import defaultdict
 from collections.abc import Callable
 from collections.abc import Generator
 from collections.abc import Mapping
@@ -1167,6 +1168,35 @@ def summary_failures(self) -> None:
         style = self.config.option.tbstyle
         self.summary_failures_combined("failed", "FAILURES", style=style)
 
+    def _add_subtests_to_failed_reports(
+        self, failed_reports: list[BaseReport]
+    ) -> list[BaseReport]:
+        """Combine failed reports with subtest failed reports, ordering subtests before main tests.
+
+        For each test nodeid, subtest failures are shown before the main test failure.
+        """
+        subtest_failed_reports = self.getreports("subtests failed")
+        subtest_reports_by_nodeid: dict[str, list[BaseReport]] = defaultdict(list)
+        for rep in subtest_failed_reports:
+            subtest_reports_by_nodeid[rep.nodeid].append(rep)
+
+        ordered_reports: list[BaseReport] = []
+        seen_nodeids: set[str] = set()
+        
+        for rep in failed_reports:
+            nodeid = rep.nodeid
+            if nodeid not in seen_nodeids:
+                seen_nodeids.add(nodeid)
+                if nodeid in subtest_reports_by_nodeid:
+                    ordered_reports.extend(subtest_reports_by_nodeid[nodeid])
+            ordered_reports.append(rep)
+        
+        for nodeid, subtests in subtest_reports_by_nodeid.items():
+            if nodeid not in seen_nodeids:
+                ordered_reports.extend(subtests)
+
+        return ordered_reports
+
     def summary_xfailures(self) -> None:
         show_tb = self.config.option.xfail_tb
         style = self.config.option.tbstyle if show_tb else "no"
@@ -1183,6 +1213,10 @@ def summary_failures_combined(
         if style != "no":
             if not needed_opt or self.hasopt(needed_opt):
                 reports: list[BaseReport] = self.getreports(which_reports)
+
+                if which_reports == "failed":
+                    reports = self._add_subtests_to_failed_reports(reports)
+
                 if not reports:
                     return
                 self.write_sep("=", sep_title)
@@ -1272,6 +1306,9 @@ def show_simple(lines: list[str], *, stat: str) -> None:
             if not failed:
                 return
             config = self.config
+            # For failed reports, also include subtests failed reports
+            if stat == "failed":
+                failed = self._add_subtests_to_failed_reports(failed)
             for rep in failed:
                 color = _color_for_type.get(stat, _color_for_type_default)
                 line = _get_line_with_reprcrash_message(
@@ -1380,7 +1417,7 @@ def _get_main_color(self) -> tuple[str, list[str]]:
 
     def _determine_main_color(self, unknown_type_seen: bool) -> str:
         stats = self.stats
-        if "failed" in stats or "error" in stats:
+        if "failed" in stats or "error" in stats or "subtests failed" in stats:
             main_color = "red"
         elif "warnings" in stats or "xpassed" in stats or unknown_type_seen:
             main_color = "yellow"

testing/test_subtests.py

@@ -55,7 +55,7 @@ def test_zaz(subtests):
         [
             "test_*.py uFuF.    *     [[]100%[]]",
             *summary_lines,
-            "* 4 failed, 1 passed in *",
+            "* 2 failed, 1 passed, 2 subtests failed in *",
         ]
     )
 
@@ -69,7 +69,7 @@ def test_zaz(subtests):
             "test_*.py::test_zaz SUBPASSED[[]zaz subtest[]]    *     [[]100%[]]",
             "test_*.py::test_zaz PASSED                        *     [[]100%[]]",
             *summary_lines,
-            "* 4 failed, 1 passed, 1 subtests passed in *",
+            "* 2 failed, 1 passed, 1 subtests passed, 2 subtests failed in *",
         ]
     )
     pytester.makeini(
@@ -87,7 +87,7 @@ def test_zaz(subtests):
             "test_*.py::test_bar FAILED                        *     [[] 66%[]]",
             "test_*.py::test_zaz PASSED                        *     [[]100%[]]",
             *summary_lines,
-            "* 4 failed, 1 passed in *",
+            "* 2 failed, 2 subtests failed, 1 passed in *",
         ]
     )
     result.stdout.no_fnmatch_line("test_*.py::test_zaz SUBPASSED[[]zaz subtest[]]*")
@@ -307,7 +307,7 @@ def test_foo(subtests, x):
             "*.py::test_foo[[]1[]] SUBFAILED[[]custom[]] (i=1) *[[]100%[]]",
             "*.py::test_foo[[]1[]] FAILED                      *[[]100%[]]",
             "contains 1 failed subtest",
-            "* 4 failed, 4 subtests passed in *",
+            "* 2 failed, 4 subtests passed, 2 subtests failed in *",
         ]
     )
 
@@ -325,7 +325,7 @@ def test_foo(subtests, x):
             "*.py::test_foo[[]1[]] SUBFAILED[[]custom[]] (i=1) *[[]100%[]]",
             "*.py::test_foo[[]1[]] FAILED                      *[[]100%[]]",
             "contains 1 failed subtest",
-            "* 4 failed in *",
+            "* 2 failed, 2 subtests failed in *",
         ]
     )
 
@@ -344,7 +344,7 @@ def test_foo(subtests):
     result = pytester.runpytest("-v")
     result.stdout.fnmatch_lines(
         [
-            "* 2 failed, 2 subtests passed in *",
+            "* 1 failed, 2 subtests passed, 1 subtests failed in *",
         ]
     )
 
@@ -365,7 +365,7 @@ def test_foo(subtests):
     result.stdout.fnmatch_lines(
         [
             "*AssertionError: top-level failure",
-            "* 2 failed, 2 subtests passed in *",
+            "* 1 failed, 2 subtests passed, 1 subtests failed in *",
         ]
     )
 
@@ -386,14 +386,14 @@ def test_foo(subtests):
     result = pytester.runpytest("-v")
     result.stdout.fnmatch_lines(
         [
-            "* 2 failed, 2 subtests passed in *",
+            "* 1 failed, 2 subtests passed, 1 subtests failed in *",
         ]
     )
 
     result = pytester.runpytest("-v", flag)
     result.stdout.fnmatch_lines(
         [
-            "* 2 failed, 2 subtests passed in *",
+            "* 1 failed, 2 subtests passed, 1 subtests failed in *",
         ]
     )
 
@@ -427,7 +427,7 @@ def test_zaz(self):
         result = pytester.runpytest()
         result.stdout.fnmatch_lines(
             [
-                "* 3 failed, 2 passed in *",
+                "* 1 failed, 2 passed, 1 subtests passed, 2 subtests failed in *",
             ]
         )
 
@@ -578,9 +578,7 @@ def test_foo(self):
         result.stdout.fnmatch_lines(
             [
                 "*.py u.                                                           *            [[]100%[]]",
-                "*=== short test summary info ===*",
-                "SUBFAILED[[]subtest 2[]] *.py::T::test_foo - AssertionError: fail subtest 2",
-                "* 1 failed, 1 passed in *",
+                "* 1 passed, 1 subtests failed in *",
             ]
         )
 
@@ -590,9 +588,9 @@ def test_foo(self):
                 "*.py::T::test_foo SUBSKIPPED[[]subtest 1[]] (skip subtest 1)      *            [[]100%[]]",
                 "*.py::T::test_foo SUBFAILED[[]subtest 2[]]                        *            [[]100%[]]",
                 "*.py::T::test_foo PASSED                                          *            [[]100%[]]",
+                "*=== short test summary info ===*",
                 "SUBSKIPPED[[]subtest 1[]] [[]1[]] *.py:*: skip subtest 1",
-                "SUBFAILED[[]subtest 2[]] *.py::T::test_foo - AssertionError: fail subtest 2",
-                "* 1 failed, 1 passed, 1 skipped in *",
+                "* 1 passed, 1 skipped, 1 subtests failed in *",
             ]
         )
 
@@ -607,9 +605,7 @@ def test_foo(self):
             [
                 "*.py::T::test_foo SUBFAILED[[]subtest 2[]]                        *            [[]100%[]]",
                 "*.py::T::test_foo PASSED                                          *            [[]100%[]]",
-                "*=== short test summary info ===*",
-                r"SUBFAILED[[]subtest 2[]] *.py::T::test_foo - AssertionError: fail subtest 2",
-                r"* 1 failed, 1 passed in *",
+                "* 1 passed, 1 subtests failed in *",
             ]
         )
         result.stdout.no_fnmatch_line(
@@ -814,7 +810,7 @@ def test(subtests):
         result = pytester.runpytest("-p no:logging")
         result.stdout.fnmatch_lines(
             [
-                "*2 failed in*",
+                "*1 failed, 1 subtests failed in*",
             ]
         )
         result.stdout.no_fnmatch_line("*root:test_no_logging.py*log line*")
@@ -899,12 +895,15 @@ def test_foo(subtests):
         """
     )
     result = pytester.runpytest("--exitfirst")
-    assert result.parseoutcomes()["failed"] == 2
+    outcomes = result.parseoutcomes()
+    assert outcomes["failed"] == 1
+    assert outcomes["subtests failed"] == 1
     result.stdout.fnmatch_lines(
         [
-            "SUBFAILED*[[]sub1[]] *.py::test_foo - assert False*",
+            "*=== short test summary info ===*",
             "FAILED *.py::test_foo - assert False",
-            "* stopping after 2 failures*",
+            "*stopping after 2 failures*",
+            "*1 failed, 1 subtests failed*",
         ],
         consecutive=True,
     )