[LIT][E2E] Fix LIT hang when executing non-existent files (#17505)

Fixes #16351

In our containers if an exception is not immediately caught lit will
hang on the following call
https://github.com/intel/llvm/blob/4adaef0e444cf9a7e11f433b9995ece6bf9e0aa4/llvm/utils/lit/lit/run.py#L93

This can occur when using the internal lit shell and trying to run a
program that does not exist. In this case `_executeShCmd` will throw an
internal shell error, which will not be caught by the function directly
calling it, `executeShCmd`, rather it is caught one function higher in
the call stack in `executeScriptInternal`. Because that exception is
percolated up the call stack instead of being immediately caught lit
will hang. This patch changes the location where we catch this exception
to `executeShCmd` instead to avoid this.

Previously to avoid this we would use the external lit shell. However
this introduces some differences in how we need to write tests (i.e.,
needing to add `--crash` for certain `not` calls), it slightly changes
how test output is printed (all in one block, rather than separated by
`RUN:` lines), and it messes up the path to executables when running on
Windows (all `\` were interpreted as escapes for the next characters
leading to trying to execute a non-existent file). This pr also changes
the E2E tests to always use the internal lit shell.

For more background on what causes this hang see:

https://stackoverflow.com/questions/15314189/python-multiprocessing-pool-hangs-at-join
https://bugs.python.org/issue9400
python/cpython#53646

Author: ayylol

llvm/utils/lit/lit/TestRunner.py

@@ -201,7 +201,14 @@ def executeShCmd(cmd, shenv, results, timeout=0):
     timeoutHelper = TimeoutHelper(timeout)
     if timeout > 0:
         timeoutHelper.startTimer()
-    finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper)
+    try:
+        finalExitCode = _executeShCmd(cmd, shenv, results, timeoutHelper)
+    except InternalShellError:
+        e = sys.exc_info()[1]
+        finalExitCode = 127
+        results.append(
+            ShellCommandResult(e.command, "", e.message, finalExitCode, False)
+        )
     timeoutHelper.cancel()
     timeoutInfo = None
     if timeoutHelper.timeoutReached():
@@ -1105,15 +1112,10 @@ def executeScriptInternal(
 
     results = []
     timeoutInfo = None
-    try:
-        shenv = ShellEnvironment(cwd, test.config.environment)
-        exitCode, timeoutInfo = executeShCmd(
-            cmd, shenv, results, timeout=litConfig.maxIndividualTestTime
-        )
-    except InternalShellError:
-        e = sys.exc_info()[1]
-        exitCode = 127
-        results.append(ShellCommandResult(e.command, "", e.message, exitCode, False))
+    shenv = ShellEnvironment(cwd, test.config.environment)
+    exitCode, timeoutInfo = executeShCmd(
+        cmd, shenv, results, timeout=litConfig.maxIndividualTestTime
+    )
 
     out = err = ""
     for i, result in enumerate(results):

llvm/utils/lit/tests/Inputs/timeout-hang/lit.cfg

@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "timeout-hang"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None

llvm/utils/lit/tests/Inputs/timeout-hang/run-nonexistent.txt

@@ -0,0 +1 @@
+RUN: nonexistent

llvm/utils/lit/tests/timeout-hang.py

@@ -0,0 +1,27 @@
+# REQUIRES: lit-max-individual-test-time
+
+# Python has some issues dealing with exceptions when multiprocessing,
+# which can cause hangs. Previously this could occur when we encountered
+# an internal shell exception, and had a timeout set.
+
+# This test runs a lit test that tries to launch a non-existent file,
+# throwing an exception. We expect this to fail immediately, rather than
+# timeout.
+
+# DEFINE: %{timeout}=1
+
+# RUN: not %{lit} %{inputs}/timeout-hang/run-nonexistent.txt \
+# RUN: --timeout=%{timeout} --param external=0 | %{python} %s %{timeout}
+
+import sys
+import re
+
+timeout_time = float(sys.argv[1])
+testing_time = float(re.search(r"Testing Time: (.*)s", sys.stdin.read()).group(1))
+
+if testing_time < timeout_time:
+    print("Testing took less than timeout")
+    sys.exit(0)
+else:
+    print("Testing took as long or longer than timeout")
+    sys.exit(1)

sycl/test-e2e/format.py

@@ -374,15 +374,10 @@ def get_extra_env(sycl_devices):
             recursion_limit=test.config.recursiveExpansionLimit,
         )
 
-        # TODO: workaround for lit hanging when executing non-existent binary
-        # inside our containers
         if len(script) == 0:
             return lit.Test.Result(lit.Test.UNSUPPORTED, "Lit script is empty")
-        useExternalSh = test.config.test_mode == "run-only"
 
-        result = lit.TestRunner._runShTest(
-            test, litConfig, useExternalSh, script, tmpBase
-        )
+        result = lit.TestRunner._runShTest(test, litConfig, False, script, tmpBase)
 
         # Single triple/device - might be an XFAIL.
         def map_result(features, code):