[lit] Add support to print paths relative to CWD in the test summary report (#154317)

This patch adds a -r|--relative-paths option to llvm-lit, which when
enabled will print test case names using paths relative to the current
working directory. The legacy default without that option is that test
cases are identified using a path relative to the test suite.

Only the summary report is impacted. That normally include failed tests,
unless unless options such as --show-pass.

Background to this patch was the discussion here

https://discourse.llvm.org/t/test-case-paths-in-llvm-lit-output-are-lacking-the-location-of-the-test-dir-itself/87973
with a goal to making it easier to copy-n-paste the path to the failing
test cases.

Examples showing difference in "Passed Tests" and "Failed Tests":

 > llvm-lit --show-pass test/Transforms/Foo
 PASS: LLVM :: Transforms/Foo/test1.txt (1 of 2)
 FAIL: LLVM :: Transforms/Foo/test2.txt (2 of 2)
 Passed Tests (1):
   LLVM :: Transforms/Foo/test1.txt
 Failed Tests (1):
   LLVM :: Transforms/Foo/test2.txt

 > llvm-lit --show-pass --relative-paths test/Transforms/Foo
 PASS: LLVM :: Transforms/Foo/test1.txt (1 of 2)
 FAIL: LLVM :: Transforms/Foo/test2.txt (2 of 2)
 Passed Tests (1):
   test/Transforms/Foo/test1.txt
 Failed Tests (1):
   test/Transforms/Foo/test2.txt

Author: bjope

llvm/utils/lit/lit/Test.py

@@ -307,6 +307,11 @@ def isFailure(self):
     def getFullName(self):
         return self.suite.config.name + " :: " + "/".join(self.path_in_suite)
 
+    def getSummaryName(self, printPathRelativeCWD):
+        if printPathRelativeCWD:
+            return os.path.relpath(self.getFilePath())
+        return self.getFullName()
+
     def getFilePath(self):
         if self.file_path:
             return self.file_path

llvm/utils/lit/lit/cl_arguments.py

@@ -91,6 +91,13 @@ def parse_args():
         help="Enable -v, but for all tests not just failed tests.",
         action="store_true",
     )
+    format_group.add_argument(
+        "-r",
+        "--relative-paths",
+        dest="printPathRelativeCWD",
+        help="Print paths relative to CWD",
+        action="store_true",
+    )
     format_group.add_argument(
         "-o",
         "--output",

llvm/utils/lit/lit/display.py

@@ -136,9 +136,7 @@ def print_result(self, test):
         # Show the test failure output, if requested.
         if (test.isFailure() and self.opts.showOutput) or self.opts.showAllOutput:
             if test.isFailure():
-                print(
-                    "%s TEST '%s' FAILED %s" % ("*" * 20, test.getFullName(), "*" * 20)
-                )
+                print("%s TEST '%s' FAILED %s" % ("*" * 20, test_name, "*" * 20))
             out = test.result.output
             # Encode/decode so that, when using Python 3.6.5 in Windows 10,
             # print(out) doesn't raise UnicodeEncodeError if out contains
@@ -159,7 +157,7 @@ def print_result(self, test):
 
         # Report test metrics, if present.
         if test.result.metrics:
-            print("%s TEST '%s' RESULTS %s" % ("*" * 10, test.getFullName(), "*" * 10))
+            print("%s TEST '%s' RESULTS %s" % ("*" * 10, test_name, "*" * 10))
             items = sorted(test.result.metrics.items())
             for metric_name, value in items:
                 print("%s: %s " % (metric_name, value.format()))

llvm/utils/lit/lit/main.py

@@ -329,20 +329,21 @@ def print_results(tests, elapsed, opts):
             sorted(tests_by_code[code], key=lambda t: t.getFullName()),
             code,
             opts.shown_codes,
+            opts.printPathRelativeCWD,
         )
 
     print_summary(total_tests, tests_by_code, opts.quiet, elapsed)
 
 
-def print_group(tests, code, shown_codes):
+def print_group(tests, code, shown_codes, printPathRelativeCWD):
     if not tests:
         return
     if not code.isFailure and code not in shown_codes:
         return
     print("*" * 20)
     print("{} Tests ({}):".format(code.label, len(tests)))
     for test in tests:
-        print("  %s" % test.getFullName())
+        print("  %s" % test.getSummaryName(printPathRelativeCWD))
     sys.stdout.write("\n")
 
 

llvm/utils/lit/tests/Inputs/print-relative-path/lit.cfg

@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "print-relative-path"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None

llvm/utils/lit/tests/Inputs/print-relative-path/test.txt

@@ -0,0 +1 @@
+# RUN: echo %var

llvm/utils/lit/tests/Inputs/print-relative-path/test2.txt

@@ -0,0 +1,2 @@
+# RUN: false
+

llvm/utils/lit/tests/print-relative-path.py

@@ -0,0 +1,18 @@
+# RUN: %{lit} --ignore-fail --show-pass %{inputs}/print-relative-path | FileCheck --check-prefix=CHECK-DEFAULT %s
+# RUN: %{lit} --ignore-fail --show-pass -r %{inputs}/print-relative-path | FileCheck --check-prefix=CHECK-RELATIVE %s
+# RUN: %{lit} --ignore-fail --show-pass --relative-paths %{inputs}/print-relative-path | FileCheck --check-prefix=CHECK-RELATIVE %s
+
+
+# CHECK-DEFAULT: PASS: print-relative-path :: test.txt (1 of 2)
+# CHECK-DEFAULT: FAIL: print-relative-path :: test2.txt (2 of 2)
+# CHECK-DEFAULT: Passed Tests (1):
+# CHECK-DEFAULT:  print-relative-path :: test.txt
+# CHECK-DEFAULT: Failed Tests (1):
+# CHECK-DEFAULT:  print-relative-path :: test2.txt
+
+# CHECK-RELATIVE: PASS: print-relative-path :: test.txt (1 of 2)
+# CHECK-RELATIVE: FAIL: print-relative-path :: test2.txt (2 of 2)
+# CHECK-RELATIVE: Passed Tests (1):
+# CHECK-RELATIVE:  Inputs/print-relative-path/test.txt
+# CHECK-RELATIVE: Failed Tests (1):
+# CHECK-RELATIVE:  Inputs/print-relative-path/test2.txt