[lit] Add --report-failures-only option for lit test reports (llvm#115439)

- Add option (--report-failures-only) to generate a reduced report for
lit tests that only includes failing tests
- This is a continuation of proposed patches by @gregbedwell here:
    - https://reviews.llvm.org/D143516
    - https://reviews.llvm.org/D143519

---------

Co-authored-by: Greg Bedwell <[email protected]>
Co-authored-by: James Henderson <[email protected]>

Author: 3 people

llvm/docs/CommandGuide/lit.rst

@@ -191,6 +191,10 @@ EXECUTION OPTIONS
 
  Write XUnit-compatible XML test reports to the specified file.
 
+.. option:: --report-failures-only
+
+ Only include unresolved, timed out, failed and unexpectedly passed tests in the report.
+
 .. option:: --resultdb-output RESULTDB_OUTPUT
 
  Write LuCI ResultDB compatible JSON to the specified file.

llvm/utils/lit/lit/cl_arguments.py

@@ -165,6 +165,12 @@ def parse_args():
         type=lit.reports.XunitReport,
         help="Write XUnit-compatible XML test reports to the specified file",
     )
+    execution_group.add_argument(
+        "--report-failures-only",
+        help="Only include unresolved, timed out, failed"
+        " and unexpectedly passed tests in the report",
+        action="store_true",
+    )
     execution_group.add_argument(
         "--resultdb-output",
         type=lit.reports.ResultDBReport,

llvm/utils/lit/lit/main.py

@@ -137,6 +137,10 @@ def main(builtin_params={}):
     print_results(discovered_tests, elapsed, opts)
 
     tests_for_report = selected_tests if opts.shard else discovered_tests
+    if opts.report_failures_only:
+        # Only report tests that failed.
+        tests_for_report = [t for t in tests_for_report if t.isFailure()]
+
     for report in opts.reports:
         report.write_results(tests_for_report, elapsed)
 

llvm/utils/lit/tests/xunit-output-report-failures-only.py

@@ -0,0 +1,12 @@
+## Check xunit output.
+# RUN: not %{lit} --report-failures-only --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
+# RUN: FileCheck --input-file=%t.xunit.xml %s
+
+# CHECK:      <?xml version="1.0" encoding="UTF-8"?>
+# CHECK-NEXT: <testsuites time="{{[0-9.]+}}">
+# CHECK-NEXT: <testsuite name="test-data" tests="1" failures="1" skipped="0" time="{{[0-9.]+}}">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&amp;name.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT:   <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
+# CHECK-NEXT: </testcase>
+# CHECK-NEXT: </testsuite>
+# CHECK-NEXT: </testsuites>