[Utils] Add --update-tests to lit

hnrklssn · hnrklssn · commit 451a178dbb46 · 2024-09-13T20:10:09.000-07:00
This adds a flag to lit for detecting and updating failing tests when
possible to do so automatically. The flag uses a plugin architecture
where config files can add additional auto-updaters for the types of
tests in the test suite. When a test fails with --update-tests enabled
lit passes the test RUN invocation and output to each registered test
updater until one of them signals that it updated the test. As such it
is the responsibility of the test updater to only update tests where it
is reasonably certain that it will actually fix the test, or come close
to doing so.

Also adds one initial test updater specific to clang's Sema test suite,
which updates the expected-[diag-kind] lines in tests using the -verify
flag.
diff --git a/clang/test/Sema/lit.local.cfg b/clang/test/Sema/lit.local.cfg
@@ -2,3 +2,15 @@ config.substitutions = list(config.substitutions)
 config.substitutions.insert(
     0, (r"%clang\b", """*** Do not use the driver in Sema tests. ***""")
 )
+
+if lit_config.update_tests:
+    import sys
+    import os
+
+    curdir = os.path.dirname(os.path.realpath(__file__))
+    testdir = os.path.dirname(curdir)
+    clangdir = os.path.dirname(testdir)
+    utilspath = os.path.join(clangdir, "utils")
+    sys.path.append(utilspath)
+    from UpdateVerifyTests.litplugin import verify_test_updater
+    lit_config.test_updaters.append(verify_test_updater)
diff --git a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{use of undeclared identifier 'a'}}
     a = 2; a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected b/clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1 2{{use of undeclared identifier 'a'}}
     a = 2; a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
          //     expected-error@+1    2      {{use of undeclared identifier 'a'}}
     a = 2; a = 2; b = 2; b = 2; c = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected b/clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
          //     expected-error@+3          {{use of undeclared identifier 'c'}}
          //     expected-error@+2    2      {{use of undeclared identifier 'b'}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c
@@ -1,9 +1,11 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2;
     // expected-error@-1{{use of undeclared identifier 'a'}}
     b = 2;// expected-error{{use of undeclared identifier 'b'}}
     c = 2;
-    // expected-error@5{{use of undeclared identifier 'c'}}
+    // expected-error@7{{use of undeclared identifier 'c'}}
     d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
 
     e = 2; // error to trigger mismatch
diff --git a/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected
@@ -1,9 +1,11 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2;
     // expected-error@-1{{use of undeclared identifier 'a'}}
     b = 2;// expected-error{{use of undeclared identifier 'b'}}
     c = 2;
-    // expected-error@5{{use of undeclared identifier 'c'}}
+    // expected-error@7{{use of undeclared identifier 'c'}}
     d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
 
     // expected-error@+1{{use of undeclared identifier 'e'}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2;
     b = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected b/clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{use of undeclared identifier 'a'}}
     a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2; b = 2; c = 2;
 }
diff --git a/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected b/clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+3{{use of undeclared identifier 'c'}}
     // expected-error@+2{{use of undeclared identifier 'b'}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-checks.c b/clang/test/utils/update-verify-tests/Inputs/no-checks.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     bar = 2;
 }
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{use of undeclared identifier 'bar'}}
     bar = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-diags.c b/clang/test/utils/update-verify-tests/Inputs/no-diags.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{asdf}}
     int a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected
@@ -1,4 +1,6 @@
 // expected-no-diagnostics
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     int a = 2;
 }
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 // expected-no-diagnostics
 void foo() {
     a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected b/clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{use of undeclared identifier 'a'}}
     a = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify=check %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2; // check-error{{asdf}}
            // expected-error@-1{ignored}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected b/clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify=check %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2; // check-error{{use of undeclared identifier 'a'}}
            // expected-error@-1{ignored}}
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     bar = 2;     //   expected-error       {{asdf}}
 }
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected b/clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     bar = 2;     //   expected-error       {{use of undeclared identifier 'bar'}}
 }
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{asdf}}
     bar = 2;
diff --git a/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected b/clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{use of undeclared identifier 'bar'}}
     bar = 2;
diff --git a/clang/test/utils/update-verify-tests/LitTests/.gitignore b/clang/test/utils/update-verify-tests/LitTests/.gitignore
@@ -0,0 +1,4 @@
+*.c
+*.c.expected
+.lit_test_times.txt
+Output
diff --git a/clang/test/utils/update-verify-tests/LitTests/lit.cfg b/clang/test/utils/update-verify-tests/LitTests/lit.cfg
@@ -0,0 +1,31 @@
+import lit.formats
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+from lit.llvm import llvm_config
+from os.path import dirname
+
+config.name = "Lit Update Verify Tests"
+config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+config.suffixes = [
+    ".c",
+]
+config.test_source_root = dirname(__file__)
+config.target_triple = None
+config.host_triple = None
+
+config.llvm_tools_dir = lit_config.path[0] # --path is explicitly passed by lit-plugin.test
+llvm_config.use_default_substitutions()
+llvm_config.use_clang()
+config.substitutions.append(("%clang_cc1", "%clang -cc1"))
+
+if lit_config.update_tests:
+    import sys
+    import os
+
+    config.clang_src_dir = dirname(dirname(dirname(dirname(config.test_source_root))))
+    utilspath = os.path.join(config.clang_src_dir, "utils")
+    sys.path.append(utilspath)
+    from UpdateVerifyTests.litplugin import verify_test_updater
+    # normally we'd append to the existing list, but when testing
+    # verify_test_updater we don't want diff_test_updater to accidentally interfere
+    lit_config.test_updaters = [verify_test_updater]
diff --git a/clang/test/utils/update-verify-tests/lit-plugin.test b/clang/test/utils/update-verify-tests/lit-plugin.test
@@ -0,0 +1,5 @@
+# RUN: cp %S/Inputs/*.c %S/LitTests
+# RUN: cp %S/Inputs/*.c.expected %S/LitTests
+# RUN: not %{lit} %S/LitTests
+# RUN: not %{lit} %S/LitTests --update-tests
+# RUN: %{lit} %S/LitTests
diff --git a/clang/test/utils/update-verify-tests/lit.local.cfg b/clang/test/utils/update-verify-tests/lit.local.cfg
@@ -12,6 +12,7 @@ if config.standalone_build:
     config.unsupported = True
 else:
     config.suffixes = [".test"]
+    config.excludes.add("LitTests")
 
     script_path = os.path.join(
         config.clang_src_dir, "utils", "update-verify-tests.py"
@@ -23,3 +24,13 @@ else:
             "%s %s" % (python, shell_quote(script_path)),
         )
     )
+    config.substitutions.append(("%{lit}", "%{lit-no-order-opt} --order=lexical"))
+
+    lit_path = os.path.join(config.llvm_tools_dir, "llvm-lit")
+    lit_path = os.path.abspath(lit_path)
+    config.substitutions.append(
+    (
+        "%{lit-no-order-opt}",
+        f"{python} {lit_path} -j1 --path {config.llvm_tools_dir}"
+    )
+)
diff --git a/clang/utils/UpdateVerifyTests/__init__.py b/clang/utils/UpdateVerifyTests/__init__.py
@@ -0,0 +1 @@
+
diff --git a/clang/utils/UpdateVerifyTests/litplugin.py b/clang/utils/UpdateVerifyTests/litplugin.py
@@ -0,0 +1,37 @@
+import sys
+from lit.formats import ShTest
+from UpdateVerifyTests.core import check_expectations
+import re
+
+verify_r = re.compile(r"-verify(?:=(\w+))?")
+
+
+def get_verify_prefixes(command):
+    def get_default(prefix):
+        if prefix:
+            return prefix
+        return "expected"
+
+    prefixes = set()
+    for arg in command.args:
+        m = verify_r.match(arg)
+        if not m:
+            continue
+        prefix = m[1]
+        if not prefix:
+            prefix = "expected"
+        prefixes.add(prefix)
+    return prefixes
+
+    
+def verify_test_updater(result):
+    if not result.stderr:
+        return None
+    prefixes = get_verify_prefixes(result.command)
+    if not prefixes:
+        return None
+    if len(prefixes) > 1:
+        return f"update-verify-test: not updating because of multiple prefixes - {prefixes}"
+    [prefix] = prefixes
+    return check_expectations(result.stderr.splitlines(), prefix)
+
diff --git a/clang/utils/update-verify-tests.py b/clang/utils/update-verify-tests.py
@@ -21,6 +21,9 @@
 Example usage:
   clang -verify [file] | python3 update-verify-tests.py
   clang -verify=check [file] | python3 update-verify-tests.py --prefix check
+
+This can also be invoked automatically by lit for failing '-verify' tests in Sema by running:
+  llvm-lit --update-tests clang/test/Sema
 """
 
 
diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
@@ -38,6 +38,7 @@ def __init__(
         parallelism_groups={},
         per_test_coverage=False,
         gtest_sharding=True,
+        update_tests=False,
     ):
         # The name of the test runner.
         self.progname = progname
@@ -89,6 +90,8 @@ def __init__(
         self.parallelism_groups = parallelism_groups
         self.per_test_coverage = per_test_coverage
         self.gtest_sharding = bool(gtest_sharding)
+        self.update_tests = update_tests
+        self.test_updaters = []
 
     @property
     def maxIndividualTestTime(self):
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
@@ -1190,6 +1190,18 @@ def executeScriptInternal(
                 str(result.timeoutReached),
             )
 
+        if litConfig.update_tests:
+            for test_updater in litConfig.test_updaters:
+                try:
+                    update_output = test_updater(result)
+                except Exception as e:
+                    out += f"Exception occurred in test updater: {e}"
+                    continue
+                if update_output:
+                    for line in update_output.splitlines():
+                        out += f"# {line}\n"
+                    break
+
     return out, err, exitCode, timeoutInfo
 
 
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
@@ -204,6 +204,12 @@ def parse_args():
         action="store_true",
         help="Exit with status zero even if some tests fail",
     )
+    execution_group.add_argument(
+        "--update-tests",
+        dest="update_tests",
+        action="store_true",
+        help="Try to update regression tests to reflect current behavior, if possible",
+    )
     execution_test_time_group = execution_group.add_mutually_exclusive_group()
     execution_test_time_group.add_argument(
         "--skip-test-time-recording",
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
@@ -64,12 +64,17 @@ def __init__(self, lit_config, config):
             self.with_environment("_TAG_REDIR_ERR", "TXT")
             self.with_environment("_CEE_RUNOPTS", "FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)")
 
+        if lit_config.update_tests:
+            self.use_lit_shell = True
+
         # Choose between lit's internal shell pipeline runner and a real shell.
         # If LIT_USE_INTERNAL_SHELL is in the environment, we use that as an
         # override.
         lit_shell_env = os.environ.get("LIT_USE_INTERNAL_SHELL")
         if lit_shell_env:
             self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env)
+            if not self.use_lit_shell and lit_config.update_tests:
+                print("note: --update-tests is not supported when using external shell")
 
         if not self.use_lit_shell:
             features.add("shell")
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
@@ -42,6 +42,7 @@ def main(builtin_params={}):
         config_prefix=opts.configPrefix,
         per_test_coverage=opts.per_test_coverage,
         gtest_sharding=opts.gtest_sharding,
+        update_tests=opts.update_tests,
     )
 
     discovered_tests = lit.discovery.find_tests_for_inputs(

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+// RUN: %clang_cc1 -verify %s`
	`2`	`+// RUN: diff %s %s.expected`
`1`	`3`	`void foo() {`
`2`	`4`	`// expected-error@+1{{use of undeclared identifier 'a'}}`
`3`	`5`	`a = 2; a = 2;`