Add check for missing tests in skip list (#3199)

A command line option `--select-fail-on-missing` for `pytest-select`
does not work with `pytest-xdist`: the error message is cryptic and does
no contain the list of missing tests, so we removed this option. There
are missing tests in our skip lists so we need to return the check for
missing tests back and fix the skip lists.

We collect test warnings to `{suite}-warnings.json` and `pytest-select`
issues a warning `PytestSelectWarning` with the list of missing tests.
We use this warning to show the list of missing tests and stop a
workflow with error.

Author: pbchekin

scripts/pass_rate.py

@@ -7,7 +7,8 @@
 import os
 import pathlib
 import platform
-from typing import List
+import sys
+from typing import Dict, List
 
 from defusedxml.ElementTree import parse
 
@@ -93,6 +94,30 @@ def get_warnings(reports_path: pathlib.Path, suite: str) -> List[TestWarning]:
     return [TestWarning(location=next(iter(w.keys())), message=next(iter(w.values()))) for w in warnings_data]
 
 
+def get_missing_tests(warnings: List[TestWarning]) -> List[str]:
+    """Searches warnings for PytestSelectWarning and returns a list of missing tests."""
+    tests = set()
+    for warning in warnings:
+        if 'PytestSelectWarning: pytest-select: Not all deselected' not in warning.message:
+            continue
+        for line in warning.message.splitlines():
+            if line.startswith('  - '):
+                tests.add(line.removeprefix('  - '))
+    return sorted(list(tests))
+
+
+def get_all_missing_tests(reports_path: pathlib.Path) -> Dict[str, List[str]]:
+    """Returns missing tests for all suites."""
+    all_missing_tests = {}
+    for report in reports_path.glob('*.xml'):
+        suite = report.stem
+        warnings = get_warnings(reports_path, suite)
+        missing_tests = get_missing_tests(warnings)
+        if missing_tests:
+            all_missing_tests[suite] = missing_tests
+    return all_missing_tests
+
+
 def parse_report(report_path: pathlib.Path, skiplist_dir: pathlib.Path) -> ReportStats:
     """Parses the specified report."""
     stats = ReportStats(name=report_path.stem)
@@ -219,6 +244,15 @@ def main():
     args.report_path = pathlib.Path(args.reports)
     args.skiplist_dir = pathlib.Path(
         args.skip_list if args.skip_list else os.getenv('TRITON_TEST_SKIPLIST_DIR', 'scripts/skiplist/default'))
+
+    missing_tests = get_all_missing_tests(args.report_path)
+    if missing_tests:
+        for suite, tests in missing_tests.items():
+            print(f'# Missing tests in {suite}:')
+            for test in tests:
+                print(test)
+        sys.exit(1)
+
     stats = parse_reports(args)
 
     if args.suite == 'all':

scripts/skiplist/lts/debug.txt

@@ -1,4 +1,4 @@
-https://github.com/intel/intel-xpu-backend-for-triton/issues/2755
+# https://github.com/intel/intel-xpu-backend-for-triton/issues/2755
 test/unit/test_debug.py::test_device_assert[True-True-True-False]
 test/unit/test_debug.py::test_device_assert[True-False-None-False]
 test/unit/test_debug.py::test_device_assert[False-True-True-False]