[Utils] Adds support for diff based tests to lit's --update-tests

This adds an updater to lit's --update-tests flag with support for
`diff`. If a RUN line containing the `diff` command fails, this function
will use heuristics to try to deduce which file is the "reference" file,
and copy the contents of the other file to the reference. If it cannot
deduce which file is the reference file, it does nothing.

The heuristics are currently:
 - does one of the files end in .expected while the other does not? Then
   the .expected file is the reference.
 - does one of the file paths contain the substring ".tmp" while the
   other does not? Then the file not containing ".tmp" is the reference.
   This matches cases where one file path is constructed using the `%t`
   substitution.

Author: hnrklssn

llvm/utils/lit/lit/DiffUpdater.py

@@ -0,0 +1,39 @@
+import shutil
+
+
+def get_source_and_target(a, b):
+    """
+    Try to figure out which file is the test output and which is the reference.
+    """
+    expected_suffix = ".expected"
+    if a.endswith(expected_suffix) and not b.endswith(expected_suffix):
+        return b, a
+    if b.endswith(expected_suffix) and not a.endswith(expected_suffix):
+        return a, b
+
+    tmp_substr = ".tmp"
+    if tmp_substr in a and not tmp_substr in b:
+        return a, b
+    if tmp_substr in b and not tmp_substr in a:
+        return b, a
+
+    return None
+
+
+def filter_flags(args):
+    return [arg for arg in args if not arg.startswith("-")]
+
+
+def diff_test_updater(result, test):
+    args = filter_flags(result.command.args)
+    if len(args) != 3:
+        return None
+    [cmd, a, b] = args
+    if cmd != "diff":
+        return None
+    res = get_source_and_target(a, b)
+    if not res:
+        return f"update-diff-test: could not deduce source and target from {a} and {b}"
+    source, target = res
+    shutil.copy(source, target)
+    return f"update-diff-test: copied {source} to {target}"

llvm/utils/lit/lit/LitConfig.py

@@ -8,6 +8,7 @@
 import lit.formats
 import lit.TestingConfig
 import lit.util
+from lit.DiffUpdater import diff_test_updater
 
 # LitConfig must be a new style class for properties to work
 class LitConfig(object):
@@ -93,7 +94,7 @@ def __init__(
         self.per_test_coverage = per_test_coverage
         self.gtest_sharding = bool(gtest_sharding)
         self.update_tests = update_tests
-        self.test_updaters = []
+        self.test_updaters = [diff_test_updater]
 
     @property
     def maxIndividualTestTime(self):

llvm/utils/lit/tests/Inputs/diff-test-update/.gitignore

@@ -0,0 +1 @@
+*.txt

llvm/utils/lit/tests/Inputs/diff-test-update/1.in

@@ -0,0 +1 @@
+FOO

llvm/utils/lit/tests/Inputs/diff-test-update/2.in

@@ -0,0 +1 @@
+BAR

llvm/utils/lit/tests/Inputs/diff-test-update/diff-bail.test

@@ -0,0 +1,3 @@
+# There is no indication here of which file is the reference file to update
+# RUN: diff %S/1.in %S/2.in
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-bail2.test

@@ -0,0 +1,7 @@
+# RUN: mkdir %t
+# RUN: cp %S/1.in %t/1.txt
+# RUN: cp %S/2.in %t/2.txt
+
+# There is no indication here of which file is the reference file to update
+# RUN: diff %t/1.txt %t/2.txt
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-expected.test

@@ -0,0 +1,5 @@
+# RUN: mkdir %t
+# RUN: cp %S/1.in %t/my-file.expected
+# RUN: cp %S/2.in %t/my-file.txt
+# RUN: diff %t/my-file.expected %t/my-file.txt
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-tmp-dir.test

@@ -0,0 +1,6 @@
+# RUN: mkdir %t
+# RUN: touch %S/empty.txt
+# RUN: cp %S/1.in %t/1.txt
+
+# RUN: diff %t/1.txt %S/empty.txt
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-tmp.test

@@ -0,0 +1,3 @@
+# RUN: cp %S/1.in %t.txt
+# RUN: cp %S/2.in %S/diff-t-out.txt
+# RUN: diff %t.txt %S/diff-t-out.txt