Add --report-failures-only option for lit test reports

Co-authored-by: Greg Bedwell <[email protected]>

Author: rpatel321

llvm/utils/lit/lit/cl_arguments.py

@@ -165,6 +165,12 @@ def parse_args():
         type=lit.reports.XunitReport,
         help="Write XUnit-compatible XML test reports to the specified file",
     )
+    execution_group.add_argument(
+        "--report-failures-only",
+        help="When writing a test report, do not include results for "
+                 "tests that completed successfully or were not run",
+        action="store_true"
+    )
     execution_group.add_argument(
         "--resultdb-output",
         type=lit.reports.ResultDBReport,

llvm/utils/lit/lit/main.py

@@ -137,6 +137,10 @@ def main(builtin_params={}):
     print_results(discovered_tests, elapsed, opts)
 
     tests_for_report = selected_tests if opts.shard else discovered_tests
+    if opts.report_failures_only:
+        # Only report tests that failed
+        tests_for_report = [t for t in tests_for_report if t.isFailure()]
+
     for report in opts.reports:
         report.write_results(tests_for_report, elapsed)
 

llvm/utils/lit/lit/reports.py

@@ -22,6 +22,8 @@ def __init__(self, output_file):
         self.output_file = output_file
         # Set by the option parser later.
         self.use_unique_output_file_name = False
+        self.skipped_codes = {lit.Test.EXCLUDED,
+                              lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
 
     def write_results(self, tests, elapsed):
         if self.use_unique_output_file_name:
@@ -114,8 +116,6 @@ def remove_invalid_xml_chars(s):
 
 
 class XunitReport(Report):
-    skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
-
     def _write_results_to_file(self, tests, elapsed, file):
         tests.sort(key=by_suite_and_test_path)
         tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
@@ -273,8 +273,6 @@ def _write_results_to_file(self, tests, elapsed, file):
 
 
 class TimeTraceReport(Report):
-    skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
-
     def _write_results_to_file(self, tests, elapsed, file):
         # Find when first test started so we can make start times relative.
         first_start_time = min([t.result.start for t in tests])

llvm/utils/lit/tests/xunit-output-report-failures-only.py

@@ -0,0 +1,17 @@
+# UNSUPPORTED: system-windows
+
+# Check xunit output
+# RUN: rm -rf %t.xunit.xml
+# RUN: not %{lit} --report-failures-only --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
+# If xmllint is installed verify that the generated xml is well-formed
+# RUN: sh -c 'if command -v xmllint 2>/dev/null; then xmllint --noout %t.xunit.xml; fi'
+# RUN: FileCheck < %t.xunit.xml %s
+
+# CHECK:      <?xml version="1.0" encoding="UTF-8"?>
+# CHECK-NEXT: <testsuites time="{{[0-9.]+}}">
+# CHECK-NEXT: <testsuite name="test-data" tests="5" failures="1" skipped="3" time="{{[0-9.]+}}">
+# CHECK-NEXT: <testcase classname="test-data.test-data" name="bad&amp;name.ini" time="{{[0-1]\.[0-9]+}}">
+# CHECK-NEXT:   <failure><![CDATA[& < > ]]]]><![CDATA[> &"]]></failure>
+# CHECK-NEXT: </testcase>
+# CHECK-NEXT: </testsuite>
+# CHECK-NEXT: </testsuites>