[llvm][llvm-lit] Add option to create unique result file names if results already exist

llvm/utils/lit/lit/cl_arguments.py

@@ -175,6 +175,15 @@ def parse_args():
         type=lit.reports.TimeTraceReport,
         help="Write Chrome tracing compatible JSON to the specified file",
     )
+    execution_group.add_argument(
+        "--use-unique-output-file-name",
+        help="When enabled, lit will not overwrite existing test report files. "
+        "Instead it will write to a new file named the same as the output file "
+        "name but with an extra part before the file extension. For example "
+        "if results.xml already exists, results.<something>.xml will be written "
+        "to. The <something> is not ordered in any way. [Default: Off]",
+        action="store_true",
+    )
     execution_group.add_argument(
         "--timeout",
         dest="maxIndividualTestTime",
@@ -332,16 +341,21 @@ def parse_args():
     else:
         opts.shard = None
 
-    opts.reports = filter(
-        None,
-        [
-            opts.output,
-            opts.xunit_xml_output,
-            opts.resultdb_output,
-            opts.time_trace_output,
-        ],
+    opts.reports = list(
+        filter(
+            None,
+            [
+                opts.output,
+                opts.xunit_xml_output,
+                opts.resultdb_output,
+                opts.time_trace_output,
+            ],
+        )
     )
 
+    for report in opts.reports:
+        report.use_unique_output_file_name = opts.use_unique_output_file_name
+
     return opts
 
 

llvm/utils/lit/lit/reports.py

@@ -1,7 +1,10 @@
+import abc
 import base64
 import datetime
 import itertools
 import json
+import os
+import tempfile
 
 from xml.sax.saxutils import quoteattr as quo
 
@@ -14,11 +17,34 @@ def by_suite_and_test_path(test):
     return (test.suite.name, id(test.suite), test.path_in_suite)
 
 
-class JsonReport(object):
+class Report(object):
     def __init__(self, output_file):
         self.output_file = output_file
+        # Set by the option parser later.
+        self.use_unique_output_file_name = False
 
     def write_results(self, tests, elapsed):
+        if self.use_unique_output_file_name:
+            filename, ext = os.path.splitext(os.path.basename(self.output_file))
+            fd, _ = tempfile.mkstemp(
+                suffix=ext, prefix=f"{filename}.", dir=os.path.dirname(self.output_file)
+            )
+            report_file = os.fdopen(fd, "w")
+        else:
+            # Overwrite if the results already exist.
+            report_file = open(self.output_file, "w")
+
+        with report_file:
+            self._write_results_to_file(tests, elapsed, report_file)
+
+    @abc.abstractmethod
+    def _write_results_to_file(self, tests, elapsed, file):
+        """Write test results to the file object "file"."""
+        pass
+
+
+class JsonReport(Report):
+    def _write_results_to_file(self, tests, elapsed, file):
         unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
         tests = [t for t in tests if t.result.code not in unexecuted_codes]
         # Construct the data we will write.
@@ -67,9 +93,8 @@ def write_results(self, tests, elapsed):
 
             tests_data.append(test_data)
 
-        with open(self.output_file, "w") as file:
-            json.dump(data, file, indent=2, sort_keys=True)
-            file.write("\n")
+        json.dump(data, file, indent=2, sort_keys=True)
+        file.write("\n")
 
 
 _invalid_xml_chars_dict = {
@@ -88,21 +113,18 @@ def remove_invalid_xml_chars(s):
     return s.translate(_invalid_xml_chars_dict)
 
 
-class XunitReport(object):
-    def __init__(self, output_file):
-        self.output_file = output_file
-        self.skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
+class XunitReport(Report):
+    skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
 
-    def write_results(self, tests, elapsed):
+    def _write_results_to_file(self, tests, elapsed, file):
         tests.sort(key=by_suite_and_test_path)
         tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
 
-        with open(self.output_file, "w") as file:
-            file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
-            file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed))
-            for suite, test_iter in tests_by_suite:
-                self._write_testsuite(file, suite, list(test_iter))
-            file.write("</testsuites>\n")
+        file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+        file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed))
+        for suite, test_iter in tests_by_suite:
+            self._write_testsuite(file, suite, list(test_iter))
+        file.write("</testsuites>\n")
 
     def _write_testsuite(self, file, suite, tests):
         skipped = 0
@@ -206,11 +228,8 @@ def gen_resultdb_test_entry(
     return test_data
 
 
-class ResultDBReport(object):
-    def __init__(self, output_file):
-        self.output_file = output_file
-
-    def write_results(self, tests, elapsed):
+class ResultDBReport(Report):
+    def _write_results_to_file(self, tests, elapsed, file):
         unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
         tests = [t for t in tests if t.result.code not in unexecuted_codes]
         data = {}
@@ -249,17 +268,14 @@ def write_results(self, tests, elapsed):
                         )
                     )
 
-        with open(self.output_file, "w") as file:
-            json.dump(data, file, indent=2, sort_keys=True)
-            file.write("\n")
+        json.dump(data, file, indent=2, sort_keys=True)
+        file.write("\n")
 
 
-class TimeTraceReport(object):
-    def __init__(self, output_file):
-        self.output_file = output_file
-        self.skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
+class TimeTraceReport(Report):
+    skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
 
-    def write_results(self, tests, elapsed):
+    def _write_results_to_file(self, tests, elapsed, file):
         # Find when first test started so we can make start times relative.
         first_start_time = min([t.result.start for t in tests])
         events = [
@@ -270,8 +286,7 @@ def write_results(self, tests, elapsed):
 
         json_data = {"traceEvents": events}
 
-        with open(self.output_file, "w") as time_trace_file:
-            json.dump(json_data, time_trace_file, indent=2, sort_keys=True)
+        json.dump(json_data, time_trace_file, indent=2, sort_keys=True)
 
     def _get_test_event(self, test, first_start_time):
         test_name = test.getFullName()

llvm/utils/lit/tests/unique-output-file.py

@@ -0,0 +1,22 @@
+## Check that lit will not overwrite existing result files when given
+## --use-unique-output-file-name.
+
+## Files are overwritten without the option.
+# RUN: rm -f %t.xunit*.xml
+# RUN: echo "test" > %t.xunit.xml
+# RUN: not %{lit} --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
+# RUN: FileCheck < %t.xunit.xml %s --check-prefix=NEW
+# NEW:      <?xml version="1.0" encoding="UTF-8"?>
+# NEW-NEXT: <testsuites time="{{[0-9.]+}}">
+## (other tests will check the contents of the whole file)
+
+# RUN: rm -f %t.xunit*.xml
+# RUN: echo "test" > %t.xunit.xml
+## Files should not be overwritten with the option.
+# RUN: not %{lit} --xunit-xml-output %t.xunit.xml --use-unique-output-file-name %{inputs}/xunit-output
+# RUN: FileCheck < %t.xunit.xml %s --check-prefix=EXISTING
+# EXISTING: test
+## Results in a new file with some discriminator added.
+# RUN: ls -l %t.xunit*.xml | wc -l | FileCheck %s --check-prefix=NUMFILES
+# NUMFILES: 2
+# RUN: FileCheck < %t.xunit.*.xml %s --check-prefix=NEW