[llvm][lit] Add option to run only the failed tests

llvm/docs/CommandGuide/lit.rst

@@ -314,6 +314,11 @@ The timing data is stored in the `test_exec_root` in a file named
   place of this option, which is especially useful in environments where the
   call to ``lit`` is issued indirectly.
 
+.. option:: --filter-failed
+
+  Run only those tests that previously failed. Tests that have been newly added
+  but not yet run are not included.
+
 .. option:: --xfail LIST
 
   Treat those tests whose name is in the semicolon separated list ``LIST`` as

llvm/utils/lit/lit/cl_arguments.py

@@ -302,6 +302,12 @@ def parse_args():
         help="Filter out tests with paths matching the given regular expression",
         default=os.environ.get("LIT_FILTER_OUT", "^$"),
     )
+    selection_group.add_argument(
+        "--filter-failed",
+        dest="filterFailed",
+        help="Only run tests which failed in the previous run",
+        action="store_true",
+    )
     selection_group.add_argument(
         "--xfail",
         metavar="LIST",

llvm/utils/lit/lit/main.py

@@ -90,6 +90,9 @@ def main(builtin_params={}):
         and not opts.filter_out.search(t.getFullName())
     ]
 
+    if opts.filterFailed:
+        selected_tests = [t for t in selected_tests if t.previous_failure]
+
     if not selected_tests:
         sys.stderr.write(
             "error: filter did not match any tests "

llvm/utils/lit/tests/Inputs/filter-failed/fail.txt

@@ -0,0 +1 @@
+RUN: false

llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg

@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "filter-failed"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None

llvm/utils/lit/tests/Inputs/filter-failed/pass.txt

@@ -0,0 +1 @@
+RUN: true

llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt

@@ -0,0 +1,2 @@
+RUN: false
+XFAIL: *

llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt

@@ -0,0 +1,2 @@
+RUN: true
+XFAIL: *

llvm/utils/lit/tests/filter-failed-delete.py

@@ -0,0 +1,16 @@
+# Shows behaviour when a previously failed test was deleted
+# before running with --filter-failed.
+
+# RUN: rm -rf %t
+# RUN: cp -r %{inputs}/filter-failed %t
+#
+# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-FIRST
+#
+# RUN: rm %t/fail.txt
+# RUN: not %{lit} --filter-failed %t | FileCheck %s --check-prefix=CHECK-SECOND
+
+# CHECK-FIRST: Testing: 5 tests
+# CHECK-FIRST: FAIL: filter-failed :: fail.txt
+
+# CHECK-SECOND: Testing: 2 of 4 tests
+# CHECK-SECOND-NOT: filter-failed :: fail.txt

llvm/utils/lit/tests/filter-failed-rerun.py

@@ -0,0 +1,18 @@
+# Checks that --filter-failed won't re-run tests that have passed
+# since the last time --filter-failed was run.
+
+# RUN: rm -rf %t
+# RUN: cp -r %{inputs}/filter-failed %t
+#
+# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-FIRST
+#
+# RUN: cp %t/pass.txt %t/fail.txt
+# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-SECOND
+# RUN: not %{lit} --filter-failed %t | FileCheck %s --check-prefix=CHECK-THIRD
+
+# CHECK-FIRST: FAIL: filter-failed :: fail.txt
+
+# CHECK-SECOND: PASS: filter-failed :: fail.txt
+
+# CHECK-THIRD: Testing: 2 of 5 tests
+# CHECK-THIRD-NOT: filter-failed :: fail.txt

llvm/utils/lit/tests/filter-failed.py

@@ -0,0 +1,23 @@
+# Checks that --filter-failed only runs tests that previously failed.
+
+# RUN: rm -rf %t
+# RUN: cp -r %{inputs}/filter-failed %t
+#
+# RUN: not %{lit} %t
+#
+# RUN: echo "RUN: false" > %t/new-fail.txt
+# RUN: echo "RUN: true"  > %t/new-pass.txt
+#
+# RUN: not %{lit} --filter-failed %t | FileCheck %s
+
+# CHECK: Testing: 3 of 7 tests
+# CHECK-DAG: FAIL: filter-failed :: fail.txt
+# CHECK-DAG: UNRESOLVED: filter-failed :: unresolved.txt
+# CHECK-DAG: XPASS: filter-failed :: xpass.txt
+
+# CHECK: Testing Time:
+# CHECK: Total Discovered Tests:
+# CHECK-NEXT:   Excluded : 4 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Failed : 1 {{\([0-9]*\.[0-9]*%\)}}
+# CHECK-NEXT:   Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}