[Utils] Adds support for diff based tests to lit's --update-tests (#154147)

This adds an updater to lit's --update-tests flag with support for
`diff`. If a RUN line containing the `diff` command fails, this function
will use heuristics to try to deduce which file is the "reference" file,
and copy the contents of the other file to the reference. If it cannot
deduce which file is the reference file, it does nothing.

The heuristics are currently:
 - does one of the files end in .expected while the other does not? Then
   the .expected file is the reference.
 - does one of the file paths contain the substring ".tmp" while the
   other does not? Then the file not containing ".tmp" is the reference.
   This matches cases where one file path is constructed using the `%t`
   substitution.

Author: hnrklssn

llvm/utils/lit/lit/DiffUpdater.py

@@ -0,0 +1,55 @@
+import shutil
+
+"""
+This file provides the `diff_test_updater` function, which is invoked on failed RUN lines when lit is executed with --update-tests.
+It checks whether the failed command is `diff` and, if so, uses heuristics to determine which file is the checked-in reference file and which file is output from the test case.
+The heuristics are currently as follows:
+    - if exactly one file ends with ".expected" (common pattern in LLVM), that file is the reference file and the other is the output file
+    - if exactly one file path contains ".tmp" (e.g. because it contains the expansion of "%t"), that file is the reference file and the other is the output file
+If the command matches one of these patterns the output file content is copied to the reference file to make the test pass.
+Otherwise the test is ignored.
+
+Possible improvements:
+    - Support stdin patterns like "my_binary %s | diff expected.txt"
+    - Scan RUN lines to see if a file is the source of output from a previous command.
+      If it is then it is not a reference file that can be copied to, regardless of name, since the test will overwrite it anyways.
+    - Only update the parts that need updating (based on the diff output). Could help avoid noisy updates when e.g. whitespace changes are ignored.
+"""
+
+
+def get_source_and_target(a, b):
+    """
+    Try to figure out which file is the test output and which is the reference.
+    """
+    expected_suffix = ".expected"
+    if a.endswith(expected_suffix) and not b.endswith(expected_suffix):
+        return b, a
+    if b.endswith(expected_suffix) and not a.endswith(expected_suffix):
+        return a, b
+
+    tmp_substr = ".tmp"
+    if tmp_substr in a and not tmp_substr in b:
+        return a, b
+    if tmp_substr in b and not tmp_substr in a:
+        return b, a
+
+    return None
+
+
+def filter_flags(args):
+    return [arg for arg in args if not arg.startswith("-")]
+
+
+def diff_test_updater(result, test):
+    args = filter_flags(result.command.args)
+    if len(args) != 3:
+        return None
+    [cmd, a, b] = args
+    if cmd != "diff":
+        return None
+    res = get_source_and_target(a, b)
+    if not res:
+        return f"update-diff-test: could not deduce source and target from {a} and {b}"
+    source, target = res
+    shutil.copy(source, target)
+    return f"update-diff-test: copied {source} to {target}"

llvm/utils/lit/lit/LitConfig.py

@@ -8,6 +8,7 @@
 import lit.formats
 import lit.TestingConfig
 import lit.util
+from lit.DiffUpdater import diff_test_updater
 
 # LitConfig must be a new style class for properties to work
 class LitConfig(object):
@@ -93,7 +94,7 @@ def __init__(
         self.per_test_coverage = per_test_coverage
         self.gtest_sharding = bool(gtest_sharding)
         self.update_tests = update_tests
-        self.test_updaters = []
+        self.test_updaters = [diff_test_updater]
 
     @property
     def maxIndividualTestTime(self):

llvm/utils/lit/tests/Inputs/diff-test-update/.gitignore

@@ -0,0 +1 @@
+*.txt

llvm/utils/lit/tests/Inputs/diff-test-update/1.in

@@ -0,0 +1 @@
+FOO

llvm/utils/lit/tests/Inputs/diff-test-update/2.in

@@ -0,0 +1 @@
+BAR

llvm/utils/lit/tests/Inputs/diff-test-update/diff-bail.test

@@ -0,0 +1,3 @@
+# There is no indication here of which file is the reference file to update
+# RUN: diff %S/1.in %S/2.in
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-bail2.test

@@ -0,0 +1,7 @@
+# RUN: mkdir %t
+# RUN: cp %S/1.in %t/1.txt
+# RUN: cp %S/2.in %t/2.txt
+
+# There is no indication here of which file is the reference file to update
+# RUN: diff %t/1.txt %t/2.txt
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-expected.test

@@ -0,0 +1,5 @@
+# RUN: mkdir %t
+# RUN: cp %S/1.in %t/my-file.expected
+# RUN: cp %S/2.in %t/my-file.txt
+# RUN: diff %t/my-file.expected %t/my-file.txt
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-tmp-dir.test

@@ -0,0 +1,6 @@
+# RUN: mkdir %t
+# RUN: touch %S/empty.txt
+# RUN: cp %S/1.in %t/1.txt
+
+# RUN: diff %t/1.txt %S/empty.txt
+

llvm/utils/lit/tests/Inputs/diff-test-update/diff-tmp.test

@@ -0,0 +1,3 @@
+# RUN: cp %S/1.in %t.txt
+# RUN: cp %S/2.in %S/diff-t-out.txt
+# RUN: diff %t.txt %S/diff-t-out.txt