[utils] integrate update-verify-tests with lit's --update-tests

This adds a lit plugin wrapping the core of update-verify-tests. When
lit is invoked with --update-tests it will call the plugin, which checks
if the failure is due to a -verify mismatch. If it is, it tries to
repair the test. If the source file was originally created by
split-file, the changes are propagated to the parent file.

No tests are added, because I don't want to run nested llvm-lit tests
in Swift's test suite, but the core functionality is tested through
update-verify-tests.py.

Author: hnrklssn

test/lit.cfg

@@ -3291,3 +3291,8 @@ lit_config.note(f"Target Triple: {config.target_triple}, Variant Triple: {config
 
 lit_config.note("Available features: " + ", ".join(sorted(config.available_features)))
 config.substitutions.append( ('%use_no_opaque_pointers', '-Xcc -Xclang -Xcc -no-opaque-pointers' ) )
+
+if lit_config.update_tests:
+    sys.path.append(config.swift_utils)
+    from update_verify_tests.litplugin import uvt_lit_plugin
+    lit_config.test_updaters.append(uvt_lit_plugin)

utils/update-verify-tests.py

@@ -32,9 +32,14 @@ def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--prefix", default="", help="The prefix passed to -verify")
     args = parser.parse_args()
-    (ret_code, output) = check_expectations(sys.stdin.readlines(), args.prefix)
-    print(output)
-    sys.exit(ret_code)
+    (err, updated_files) = check_expectations(sys.stdin.readlines(), args.prefix)
+    if err:
+        print(err)
+        sys.exit(1)
+
+    if len(updated_files) > 1:
+        print("\n\t".join(["updated files:"] + updated_files))
+    print(f"updated file: {updated_files[0]}")
 
 
 if __name__ == "__main__":

utils/update_verify_tests/core.py

@@ -570,13 +570,13 @@ def update_test_files(errors, prefix):
         try:
             update_test_file(filename, diag_errors, prefix, updated_test_files)
         except KnownException as e:
-            return f"Error in update-verify-tests while updating {filename}: {e}"
+            return (
+                f"Error in update-verify-tests while updating {filename}: {e}",
+                None,
+            )
     updated_files = list(updated_test_files)
     assert updated_files
-    if len(updated_files) == 1:
-        return f"updated file {updated_files[0]}"
-    updated_files_s = "\n\t".join(updated_files)
-    return "updated files:\n\t{updated_files_s}"
+    return (None, updated_files)
 
 
 """
@@ -786,8 +786,8 @@ def check_expectations(tool_output, prefix):
             top_level.extend(curr)
 
     except KnownException as e:
-        return (1, f"Error in update-verify-tests while parsing tool output: {e}")
+        return (f"Error in update-verify-tests while parsing tool output: {e}", None)
     if top_level:
-        return (0, update_test_files(top_level, prefix))
+        return update_test_files(top_level, prefix)
     else:
-        return (1, "no mismatching diagnostics found")
+        return ("no mismatching diagnostics found", None)

utils/update_verify_tests/litplugin.py

@@ -0,0 +1,135 @@
+import os
+import shlex
+import pathlib
+from update_verify_tests.core import check_expectations
+
+"""
+This file provides the `uvt_lit_plugin` function, which is invoked on failed RUN lines when lit is executed with --update-tests.
+It checks whether the failed command is a swift compiler invocation with the `-verify` flag and analyses the output to try to
+repair the failed test. If the updated file was originally created by `split-file` it updates the corresponding slice in the source file.
+"""
+
+
+class SplitFileTarget:
+    def __init__(self, slice_start_idx, test_path, lines, name):
+        self.slice_start_idx = slice_start_idx
+        self.test_path = test_path
+        self.lines = lines
+        self.name = name
+
+    def copyFrom(self, source):
+        lines_before = self.lines[: self.slice_start_idx + 1]
+        self.lines = self.lines[self.slice_start_idx + 1 :]
+        slice_end_idx = None
+        for i, l in enumerate(self.lines):
+            if SplitFileTarget._get_split_line_path(l) != None:
+                slice_end_idx = i
+                break
+        if slice_end_idx is not None:
+            lines_after = self.lines[slice_end_idx:]
+        else:
+            lines_after = []
+        with open(source, "r") as f:
+            new_lines = lines_before + f.readlines() + lines_after
+        with open(self.test_path, "w") as f:
+            for l in new_lines:
+                f.write(l)
+
+    def __str__(self):
+        return f"slice {self.name} in {self.test_path}"
+
+    @staticmethod
+    def get_target_dir(commands, test_path):
+        # posix=True breaks Windows paths because \ is treated as an escaping character
+        for cmd in commands:
+            split = shlex.split(cmd, posix=False)
+            if "split-file" not in split:
+                continue
+            start_idx = split.index("split-file")
+            split = split[start_idx:]
+            if len(split) < 3:
+                continue
+            p = unquote(split[1].strip())
+            if not test_path.samefile(p):
+                continue
+            return unquote(split[2].strip())
+        return None
+
+    @staticmethod
+    def create(path, commands, test_path, target_dir):
+        path = pathlib.Path(path)
+        with open(test_path, "r") as f:
+            lines = f.readlines()
+        for i, l in enumerate(lines):
+            p = SplitFileTarget._get_split_line_path(l)
+            if p and path.samefile(os.path.join(target_dir, p)):
+                idx = i
+                break
+        else:
+            return None
+        return SplitFileTarget(idx, test_path, lines, p)
+
+    @staticmethod
+    def _get_split_line_path(l):
+        if len(l) < 6:
+            return None
+        if l.startswith("//"):
+            l = l[2:]
+        else:
+            l = l[1:]
+        if l.startswith("--- "):
+            l = l[4:]
+        else:
+            return None
+        return l.rstrip()
+
+
+def unquote(s):
+    if len(s) > 1 and s[0] == s[-1] and (s[0] == '"' or s[0] == "'"):
+        return s[1:-1]
+    return s
+
+
+def propagate_split_files(test_path, updated_files, commands):
+    test_path = pathlib.Path(test_path)
+    split_target_dir = SplitFileTarget.get_target_dir(commands, test_path)
+    if not split_target_dir:
+        return updated_files
+
+    new = []
+    for file in updated_files:
+        target = SplitFileTarget.create(file, commands, test_path, split_target_dir)
+        if target:
+            target.copyFrom(file)
+            new.append(target)
+        else:
+            new.append(file)
+    return new
+
+
+def uvt_lit_plugin(result, test, commands):
+    if (
+        not any(e.endswith("swift-frontend") for e in result.command.args)
+        or not "-verify" in result.command.args
+    ):
+        return None
+
+    prefix = ""
+    for i, arg in enumerate(result.command.args):
+        if arg == "-verify-additional-prefix":
+            if i + 1 >= len(result.command.args):
+                return None
+            if prefix:
+                # can only handle at most 1 additional prefix at the moment
+                return None
+            prefix = result.command.args[i + 1]
+
+    (err, updated_files) = check_expectations(result.stderr.split("\n"), prefix)
+    if err:
+        return err
+
+    updated_files = propagate_split_files(test.getFilePath(), updated_files, commands)
+
+    if len(updated_files) > 1:
+        return "\n\t".join(["updated files:"] + updated_files)
+    return f"updated file: {updated_files[0]}"