Fix the total number of tests in pass_rate.py (#4343)

PyTest JUnit XML report contains incorrect number of total tests if test
is passed but there is an error at teardown.

Fixes #4341.

Signed-off-by: Pavel Chekin <[email protected]>

Author: pbchekin

scripts/pass_rate.py

@@ -84,22 +84,26 @@ def parse_report(report_path: pathlib.Path) -> ReportStats:
     """Parses the specified report."""
     stats = ReportStats(name=report_path.stem)
     root = parse(report_path).getroot()
+    testsuite_fixme_tests = set()
+
     for testsuite in root:
-        testsuite_fixme_tests = set()
-        stats.total += int(testsuite.get('tests'))
-        for skipped in testsuite.iter('skipped'):
+        for testcase in testsuite.iter('testcase'):
+            stats.total += 1
+            if testcase.find('failure') is not None or testcase.find('error') is not None:
+                stats.failed += 1
+                continue
+            skipped = testcase.find('skipped')
+            if skipped is None:
+                continue
             if skipped.get('type') == 'pytest.skip':
                 stats.skipped += 1
             elif skipped.get('type') == 'pytest.xfail':
                 stats.xfailed += 1
-        for _ in testsuite.iter('failure'):
-            stats.failed += 1
-        for _ in testsuite.iter('error'):
-            stats.failed += 1
-        for warning in get_warnings(report_path.parent, report_path.stem):
-            if 'FIXME' in warning.message:
-                testsuite_fixme_tests.add(warning.location)
-        stats.fixme += len(testsuite_fixme_tests)
+
+    for warning in get_warnings(report_path.parent, report_path.stem):
+        if 'FIXME' in warning.message:
+            testsuite_fixme_tests.add(warning.location)
+    stats.fixme += len(testsuite_fixme_tests)
 
     stats.passed = stats.total - stats.failed - stats.skipped - stats.xfailed
     return stats

scripts/test_pass_rate.py

@@ -22,6 +22,20 @@
   - test22
 """
 
+# Real JUnit report with the 3 tests. One test passed, but there is an error at test teardown.
+# Note that pytest incorrectly reports 4 tests instead of 3.
+# https://github.com/intel/intel-xpu-backend-for-triton/issues/4341.
+JUNIT_XML_REPORT = """\
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites>
+  <testsuite name="pytest" errors="1" failures="0" skipped="2" tests="4" time="31.615" timestamp="2025-05-28T15:26:42.685704+00:00" hostname="example">
+    <testcase classname="python.test.unit.test_perf_warning" name="test_remark_swp_op_before_operands" time="17.602"><error message="xxx"></error></testcase>
+    <testcase classname="python.test.unit.test_perf_warning" name="test_mma_remark" time="0.019"><skipped type="pytest.xfail" message="Not designed for XPU" /></testcase>
+    <testcase classname="python.test.unit.test_perf_warning" name="test_remark_vectorization" time="0.030"><skipped type="pytest.xfail" message="Not designed for XPU" /></testcase>
+  </testsuite>
+</testsuites>
+"""
+
 
 def test_get_warnings_empty_file(tmp_path):
     warnings_path = tmp_path / 'suite-warnings.json'
@@ -70,3 +84,14 @@ def test_generate_junit_report(tmp_path):
     assert stats[0].failed == 1
     assert stats[0].skipped == 1
     assert stats[0].total == 3
+
+
+def test_parse_report(tmp_path):
+    report_path = tmp_path / 'suite.xml'
+    report_path.write_text(JUNIT_XML_REPORT)
+    stats = pass_rate.parse_report(report_path)
+    assert stats.passed == 0
+    assert stats.xfailed == 2
+    assert stats.failed == 1
+    assert stats.skipped == 0
+    assert stats.total == 3