[llvm][llvm-lit] Add option to create unique result file names if results already exist

When running a build like:
ninja check-clang check-llvm

Prior to my changes you ended up with one results file, in this specific case
Junit XML:
results.xml

This would only include the last set of tests lit ran, which were for llvm.
To get around this, many CI systems will run one check target, move the file
away, then run another.

for target in targets:
  ninja target
  mv results.xml results-${target}.xml

I want to use something like this Buildkite reporting plugin in CI,
which needs to have all the results available:
https://buildkite.com/docs/agent/v3/cli-annotate#using-annotations-to-report-test-results

Modifying CI's build scripts for Windows and Linux is a lot of work.
So my changes instead make lit detect an existing result file and modify the file name
until it finds a unique file name to write to.

Now you will get:
results.xml results.1.xml

This will work for all result file types since I'm doing it in the base Report
class. Now you've got separate files, it's easy to collect them with `<path>/*.xml`.

The number will increment as many times as needed until a useable name
is found.

Author: DavidSpickett

llvm/utils/lit/lit/cl_arguments.py

@@ -175,6 +175,13 @@ def parse_args():
         type=lit.reports.TimeTraceReport,
         help="Write Chrome tracing compatible JSON to the specified file",
     )
+    execution_group.add_argument(
+        "--use-unique-output-file-name",
+        help="When enabled, lit will not overwrite existing test report files. "
+        "Instead it will modify the file name until it finds a file name "
+        "that does not already exist. [Default: Off]",
+        action="store_true",
+    )
     execution_group.add_argument(
         "--timeout",
         dest="maxIndividualTestTime",
@@ -332,16 +339,21 @@ def parse_args():
     else:
         opts.shard = None
 
-    opts.reports = filter(
-        None,
-        [
-            opts.output,
-            opts.xunit_xml_output,
-            opts.resultdb_output,
-            opts.time_trace_output,
-        ],
+    opts.reports = list(
+        filter(
+            None,
+            [
+                opts.output,
+                opts.xunit_xml_output,
+                opts.resultdb_output,
+                opts.time_trace_output,
+            ],
+        )
     )
 
+    for report in opts.reports:
+        report.use_unique_output_file_name = opts.use_unique_output_file_name
+
     return opts
 
 

llvm/utils/lit/lit/reports.py

@@ -3,6 +3,7 @@
 import datetime
 import itertools
 import json
+import os
 
 from xml.sax.saxutils import quoteattr as quo
 
@@ -18,10 +19,31 @@ def by_suite_and_test_path(test):
 class Report(object):
     def __init__(self, output_file):
         self.output_file = output_file
+        # Set by the option parser later.
+        self.use_unique_output_file_name = False
 
     def write_results(self, tests, elapsed):
-        with open(self.output_file, "w") as file:
-            self._write_results_to_file(tests, elapsed, file)
+        if self.use_unique_output_file_name:
+            file = None
+            filepath = self.output_file
+            attempt = 0
+            while file is None:
+                try:
+                    file = open(filepath, "x")
+                except FileExistsError:
+                    attempt += 1
+                    # If there is an extension insert before that because most
+                    # glob patterns for these will be '*.extension'. Otherwise
+                    # add to the end of the path.
+                    path, ext = os.path.splitext(self.output_file)
+                    filepath = path + f".{attempt}" + ext
+
+            with file:
+                self._write_results_to_file(tests, elapsed, file)
+        else:
+            # Overwrite if the results already exist.
+            with open(self.output_file, "w") as file:
+                self._write_results_to_file(tests, elapsed, file)
 
     @abc.abstractmethod
     def _write_results_to_file(self, tests, elapsed, file):

llvm/utils/lit/tests/unique-output-file.py

@@ -0,0 +1,28 @@
+# Check that lit will not overwrite existing result files when given
+# --use-unique-output-file-name.
+
+# Files are overwritten without the option.
+# RUN: rm -rf %t.xunit*.xml
+# RUN: echo "test" > %t.xunit.xml
+# RUN: not %{lit} --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
+# RUN: FileCheck < %t.xunit.xml %s --check-prefix=NEW
+
+# RUN: rm -rf %t.xunit*.xml
+# RUN: echo "test" > %t.xunit.xml
+# Files should not be overwritten with the option.
+# RUN: not %{lit} --xunit-xml-output %t.xunit.xml --use-unique-output-file-name %{inputs}/xunit-output
+# RUN: FileCheck < %t.xunit.xml %s --check-prefix=EXISTING
+# EXISTING: test
+# Results in a new file with "1" added.
+# RUN: FileCheck < %t.xunit.1.xml %s --check-prefix=NEW
+# NEW:      <?xml version="1.0" encoding="UTF-8"?>
+# NEW-NEXT: <testsuites time="{{[0-9.]+}}">
+# (assuming that other tests check the whole contents of the file)
+
+# The number should increment as many times as needed.
+# RUN: touch %t.xunit.2.xml
+# RUN: touch %t.xunit.3.xml
+# RUN: touch %t.xunit.4.xml
+
+# RUN: not %{lit} --xunit-xml-output %t.xunit.xml --use-unique-output-file-name %{inputs}/xunit-output
+# RUN: FileCheck < %t.xunit.5.xml %s --check-prefix=NEW