[lit] Optionally exclude xfail tests (#151191)

See the related issue. We want to set up a build bot where `opt` runs with `-enable-profcheck`, which inserts `MD_prof` before running the rest of the pipeline requested from `opt`, and then validates resulting profile information (more info in the RFC linked by the aforementioned issue)

In that setup, we will also ignore `FileCheck`: while the profile info inserted is, currently, equivalent to the profile info a pass would observe via `BranchProbabilityInfo`/`BlockFrequencyInfo`, (1) we may want to change that, and (2) some tests are quite sensitive to the output IR, and break if, for instance, extra metadata is present (which it would be due to `-enable-profcheck`). Since we're just interested in profile consistency on the upcoming bot, ignoring `FileCheck` is simpler and sufficient. However, this has the effect of passing XFAIL tests. Rather than listing them all, the alternative is to just exclude XFAIL tests.

This PR adds support for that by introducing a `--exclude-xfail` option to `llvm-lit`.

Issue #147390

Author: mtrofin

llvm/docs/CommandGuide/lit.rst

@@ -356,6 +356,11 @@ The timing data is stored in the `test_exec_root` in a file named
   primary purpose is to suppress an ``XPASS`` result without modifying a test
   case that uses the ``XFAIL`` directive.
 
+.. option:: --exclude-xfail
+
+  ``XFAIL`` tests won't be run, unless they are listed in the ``--xfail-not``
+  (or ``LIT_XFAIL_NOT``) lists.
+
 .. option:: --num-shards M
 
  Divide the set of selected tests into ``M`` equal-sized subsets or

llvm/utils/lit/lit/Test.py

@@ -247,6 +247,9 @@ def __init__(
         # and will be honored when the test result is supplied.
         self.xfails = []
 
+        # Exclude this test if it's xfail.
+        self.exclude_xfail = False
+
         # If true, ignore all items in self.xfails.
         self.xfail_not = False
 

llvm/utils/lit/lit/TestRunner.py

@@ -2175,6 +2175,8 @@ def parseIntegratedTestScript(test, additional_parsers=[], require_script=True):
     assert parsed["DEFINE:"] == script
     assert parsed["REDEFINE:"] == script
     test.xfails += parsed["XFAIL:"] or []
+    if test.exclude_xfail and test.isExpectedToFail():
+        return lit.Test.Result(Test.EXCLUDED, "excluding XFAIL tests")
     test.requires += parsed["REQUIRES:"] or []
     test.unsupported += parsed["UNSUPPORTED:"] or []
     if parsed["ALLOW_RETRIES:"]:

llvm/utils/lit/lit/cl_arguments.py

@@ -303,6 +303,16 @@ def parse_args():
         help="do not XFAIL tests with paths in the semicolon separated list",
         default=os.environ.get("LIT_XFAIL_NOT", ""),
     )
+    selection_group.add_argument(
+        "--exclude-xfail",
+        help="exclude XFAIL tests (unless they are in the --xfail-not list). "
+        "Note: This option is implemented in "
+        "lit.TestRunner.parseIntegratedTestScript and so will have no effect on "
+        "test formats that do not call that and do not implement the option "
+        "separately.",
+        default=False,
+        action="store_true",
+    )
     selection_group.add_argument(
         "--num-shards",
         dest="numShards",

llvm/utils/lit/lit/main.py

@@ -240,6 +240,8 @@ def mark_xfail(selected_tests, opts):
             t.xfails += "*"
         if test_file in opts.xfail_not or test_full_name in opts.xfail_not:
             t.xfail_not = True
+        if opts.exclude_xfail:
+            t.exclude_xfail = True
 
 
 def mark_excluded(discovered_tests, selected_tests):

llvm/utils/lit/tests/Inputs/xfail-cl/true-xfail-conditionally.txt

@@ -0,0 +1,2 @@
+# XFAIL: this-does-not-exist
+# RUN: true

llvm/utils/lit/tests/xfail-cl.py

@@ -5,6 +5,18 @@
 # RUN:   %{inputs}/xfail-cl \
 # RUN: | FileCheck --check-prefix=CHECK-FILTER %s
 
+# RUN: %{lit} --xfail 'false.txt;false2.txt;top-level-suite :: b :: test.txt' \
+# RUN:   --exclude-xfail \
+# RUN:   %{inputs}/xfail-cl \
+# RUN: | FileCheck --check-prefixes=CHECK-EXCLUDED,CHECK-EXCLUDED-NOOVERRIDE %s
+
+# RUN: %{lit} --xfail 'false.txt;false2.txt;top-level-suite :: b :: test.txt' \
+# RUN:   --xfail-not 'true-xfail.txt' \
+# RUN:   --exclude-xfail \
+# RUN:   %{inputs}/xfail-cl \
+# RUN: | FileCheck --check-prefixes=CHECK-EXCLUDED,CHECK-EXCLUDED-OVERRIDE %s
+
+
 # RUN: env LIT_XFAIL='false.txt;false2.txt;top-level-suite :: b :: test.txt' \
 # RUN:   LIT_XFAIL_NOT='true-xfail.txt;top-level-suite :: a :: test-xfail.txt' \
 # RUN: %{lit} %{inputs}/xfail-cl \
@@ -23,7 +35,7 @@
 
 # END.
 
-# CHECK-FILTER: Testing: 10 tests, {{[0-9]*}} workers
+# CHECK-FILTER: Testing: 11 tests, {{[0-9]*}} workers
 # CHECK-FILTER-DAG: {{^}}PASS: top-level-suite :: a :: test.txt
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: b :: test.txt
 # CHECK-FILTER-DAG: {{^}}XFAIL: top-level-suite :: a :: false.txt
@@ -37,3 +49,17 @@
 
 # CHECK-OVERRIDE: Testing: 1 tests, {{[0-9]*}} workers
 # CHECK-OVERRIDE: {{^}}PASS: top-level-suite :: true-xfail.txt
+
+# CHECK-EXCLUDED: Testing: 11 tests, {{[0-9]*}} workers
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: a :: false.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: a :: test-xfail.txt
+# CHECK-EXCLUDED-DAG: {{^}}PASS: top-level-suite :: a :: test.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: b :: false.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: b :: test-xfail.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: b :: test.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: false.txt
+# CHECK-EXCLUDED-DAG: {{^}}EXCLUDED: top-level-suite :: false2.txt
+# CHECK-EXCLUDED-DAG: {{^}}PASS: top-level-suite :: true-xfail-conditionally.txt
+# CHECK-EXCLUDED-NOOVERRIDE-DAG: {{^}}EXCLUDED: top-level-suite :: true-xfail.txt
+# CHECK-EXCLUDED-OVERRIDE-DAG: {{^}}PASS: top-level-suite :: true-xfail.txt
+# CHECK-EXCLUDED-DAG: {{^}}PASS: top-level-suite :: true.txt