[lldb][test] Support remote run of Shell tests

1. This commit adds LLDB_TEST_PLATFORM_URL, LLDB_TEST_SYSROOT,
   LLDB_TEST_PLATFORM_WORKING_DIR, LLDB_SHELL_TESTS_DISABLE_REMOTE cmake flags
   to pass arguments for cross-compilation and remote running of both Shell&API
   tests.
2. To run Shell tests remotely, it adds 'platform select' and 'platform connect' commands to %lldb
   substitution.
3. 'remote-linux' feature added to lit to disable tests failing with remote execution.
4. A separate working directory is assigned to each test to avoid
   conflicts during parallel test execution.
5. Remote Shell testing is run only when LLDB_TEST_SYSROOT is set for
   building test sources. Recommended compiler for that is Clang.

Author: dzhidzhoev

lldb/docs/resources/test.rst

@@ -592,15 +592,17 @@ test suite, but there are two things to have in mind:
    multiple connections. For more information on how to setup remote debugging
    see the Remote debugging page.
 2. You must tell the test-suite how to connect to the remote system. This is
-   achieved using the ``--platform-name``, ``--platform-url`` and
-   ``--platform-working-dir`` parameters to ``dotest.py``. These parameters
-   correspond to the platform select and platform connect LLDB commands. You
-   will usually also need to specify the compiler and architecture for the
-   remote system.
-
-Currently, running the remote test suite is supported only with ``dotest.py`` (or
-dosep.py with a single thread), but we expect this issue to be addressed in the
-near future.
+   achieved using the ``LLDB_TEST_PLATFORM_URL``, ``LLDB_TEST_PLATFORM_WORKING_DIR``
+   flags to cmake, and ``--platform-name`` parameter to ``dotest.py``.
+   These parameters correspond to the platform select and platform connect
+   LLDB commands. You will usually also need to specify the compiler and
+   architecture for the remote system.
+3. Remote Shell tests execution is currently supported only for Linux target
+   platform. It's triggered when ``LLDB_TEST_SYSROOT`` is provided for building
+   test sources. It can be disabled by setting ``LLDB_SHELL_TESTS_DISABLE_REMOTE=On``.
+   Shell tests are not guaranteed to pass against remote target if the compiler
+   being used is other than Clang.
+
 
 Running tests in QEMU System Emulation Environment
 ``````````````````````````````````````````````````

lldb/test/API/lit.cfg.py

@@ -303,6 +303,13 @@ def delete_module_cache(path):
 # In particular, (1) is visited at the top of the file, since the script
 # derives other information from it.
 
+if is_configured("lldb_platform_url"):
+    dotest_cmd += ["--platform-url", config.lldb_platform_url]
+if is_configured("lldb_platform_working_dir"):
+    dotest_cmd += ["--platform-working-dir", config.lldb_platform_working_dir]
+if is_configured("cmake_sysroot"):
+    dotest_cmd += ["--sysroot", config.cmake_sysroot]
+
 if is_configured("dotest_user_args_str"):
     dotest_cmd.extend(config.dotest_user_args_str.split(";"))
 

lldb/test/API/lit.site.cfg.py.in

@@ -24,6 +24,9 @@ config.lua_executable = "@Lua_EXECUTABLE@"
 config.lua_test_entry = "TestLuaAPI.py"
 config.dotest_common_args_str = lit_config.substitute("@LLDB_TEST_COMMON_ARGS@")
 config.dotest_user_args_str = lit_config.substitute("@LLDB_TEST_USER_ARGS@")
+config.lldb_platform_url = lit_config.substitute("@LLDB_TEST_PLATFORM_URL@")
+config.lldb_platform_working_dir = lit_config.substitute("@LLDB_TEST_PLATFORM_WORKING_DIR@")
+config.cmake_sysroot = lit_config.substitute("@LLDB_TEST_SYSROOT@" or "@DEFAULT_SYSROOT@")
 config.lldb_enable_python = @LLDB_ENABLE_PYTHON@
 config.dotest_lit_args_str = None
 config.enabled_plugins = []

lldb/test/CMakeLists.txt

@@ -244,6 +244,7 @@ llvm_canonicalize_cmake_booleans(
   LLVM_ENABLE_ZLIB
   LLVM_ENABLE_SHARED_LIBS
   LLDB_HAS_LIBCXX
+  LLDB_SHELL_TESTS_DISABLE_REMOTE
   LLDB_TOOL_LLDB_SERVER_BUILD
   LLDB_USE_SYSTEM_DEBUGSERVER
   LLDB_IS_64_BITS)

lldb/test/Shell/Settings/TestEchoCommands.test

@@ -2,6 +2,8 @@
 # RUN: %lldb -x -b -o 'settings set interpreter.echo-comment-commands false' -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsNoComments.out
 # RUN: %lldb -x -b -o 'settings set interpreter.echo-commands false'         -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsNone.out
 
+XFAIL: remote{{.*}}
+
 RUN: echo start >%t.file
 RUN: %lldb -x -b --source-quietly -s %S/Inputs/EchoCommandsTest.in >>%t.file
 RUN: echo done >>%t.file

lldb/test/Shell/Target/target-label.test

@@ -4,16 +4,16 @@
 
 target create -l "ls" /bin/ls
 target list
-# CHECK: * target #0 (ls): /bin/ls
+# CHECK: * target #0 (ls): [[LS_PATH:.*]]
 
 script lldb.target.SetLabel("")
 target list
-# CHECK: * target #0: /bin/ls
+# CHECK: * target #0: [[LS_PATH]]
 
 target create -l "cat" /bin/cat
 target list
-# CHECK: target #0: /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0: [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH:.*]]
 
 target create -l "cat" /bin/cat
 # CHECK: Cannot use label 'cat' since it's set in target #1.
@@ -22,12 +22,12 @@ target create -l 42 /bin/cat
 # CHECK: error: Cannot use integer as target label.
 
 target select 0
-# CHECK: * target #0: /bin/ls
-# CHECK-NEXT: target #1 (cat): /bin/cat
+# CHECK: * target #0: [[LS_PATH]]
+# CHECK-NEXT: target #1 (cat): [[CAT_PATH]]
 
 target select cat
-# CHECK: target #0: /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0: [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH]]
 
 script lldb.target.GetLabel()
 # CHECK: 'cat'
@@ -36,5 +36,5 @@ script lldb.debugger.GetTargetAtIndex(0).SetLabel('Not cat')
 # CHECK: success
 
 target list
-# CHECK: target #0 (Not cat): /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0 (Not cat): [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH]]

lldb/test/Shell/helper/toolchain.py

@@ -1,10 +1,12 @@
 import os
 import itertools
 import platform
+import re
 import subprocess
 import sys
 
 import lit.util
+from lit.formats import ShTest
 from lit.llvm import llvm_config
 from lit.llvm.subst import FindTool
 from lit.llvm.subst import ToolSubst
@@ -22,6 +24,55 @@ def _disallow(config, execName):
     config.substitutions.append((" {0} ".format(execName), warning.format(execName)))
 
 
+def get_lldb_args(config, suffix=None):
+    lldb_args = []
+    if "remote-linux" in config.available_features:
+        lldb_args += [
+            "-O",
+            '"platform select remote-linux"',
+            "-O",
+            f'"platform connect {config.lldb_platform_url}"',
+        ]
+        if config.lldb_platform_working_dir:
+            dir = f"{config.lldb_platform_working_dir}/shell"
+            if suffix:
+                dir += f"/{suffix}"
+            lldb_args += [
+                "-O",
+                f'"platform shell mkdir -p {dir}"',
+                "-O",
+                f'"platform settings -w {dir}"',
+            ]
+    lldb_args += ["--no-lldbinit", "-S", _get_lldb_init_path(config)]
+    return lldb_args
+
+
+class ShTestLldb(ShTest):
+    def __init__(
+        self, execute_external=False, extra_substitutions=[], preamble_commands=[]
+    ):
+        super().__init__(execute_external, extra_substitutions, preamble_commands)
+
+    def execute(self, test, litConfig):
+        for i, t in enumerate(test.config.substitutions):
+            try:
+                if re.match(t[0], "%lldb"):
+                    cmd = t[1]
+                    if '-O "platform settings -w ' in cmd:
+                        args_def = " ".join(get_lldb_args(test.config))
+                        args_unique = " ".join(
+                            get_lldb_args(test.config, "/".join(test.path_in_suite))
+                        )
+                        test.config.substitutions[i] = (
+                            t[0],
+                            cmd.replace(args_def, args_unique),
+                        )
+                    break
+            except:
+                pass
+        return super().execute(test, litConfig)
+
+
 def use_lldb_substitutions(config):
     # Set up substitutions for primary tools.  These tools must come from config.lldb_tools_dir
     # which is basically the build output directory.  We do not want to find these in path or
@@ -34,7 +85,9 @@ def use_lldb_substitutions(config):
     build_script = os.path.join(build_script, "build.py")
     build_script_args = [
         build_script,
-        "--compiler=any",  # Default to best compiler
+        (
+            "--compiler=clang" if config.enable_remote else "--compiler=any"
+        ),  # Default to best compiler
         "--arch=" + str(config.lldb_bitness),
     ]
     if config.lldb_lit_tools_dir:
@@ -56,7 +109,7 @@ def use_lldb_substitutions(config):
         ToolSubst(
             "%lldb",
             command=FindTool("lldb"),
-            extra_args=["--no-lldbinit", "-S", lldb_init],
+            extra_args=get_lldb_args(config),
             unresolved="fatal",
         ),
         ToolSubst(
@@ -138,7 +191,7 @@ def use_support_substitutions(config):
     # Set up substitutions for support tools.  These tools can be overridden at the CMake
     # level (by specifying -DLLDB_LIT_TOOLS_DIR), installed, or as a last resort, we can use
     # the just-built version.
-    host_flags = ["--target=" + config.host_triple]
+    host_flags = ["--target=" + config.target_triple]
     if platform.system() in ["Darwin"]:
         try:
             out = subprocess.check_output(["xcrun", "--show-sdk-path"]).strip()
@@ -165,6 +218,14 @@ def use_support_substitutions(config):
     if config.cmake_sysroot:
         host_flags += ["--sysroot={}".format(config.cmake_sysroot)]
 
+    if config.enable_remote and config.has_libcxx:
+        host_flags += [
+            "-L{}".format(config.libcxx_libs_dir),
+            "-Wl,-rpath,{}".format(config.libcxx_libs_dir),
+            "-lc++",
+            "-lc++abi",
+        ]
+
     host_flags = " ".join(host_flags)
     config.substitutions.append(("%clang_host", "%clang " + host_flags))
     config.substitutions.append(("%clangxx_host", "%clangxx " + host_flags))

lldb/test/Shell/lit.cfg.py

@@ -21,7 +21,7 @@
 config.name = "lldb-shell"
 
 # testFormat: The test format to use to interpret tests.
-config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+config.test_format = toolchain.ShTestLldb(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files. This is overriden
 # by individual lit.local.cfg files in the test subdirectories.
@@ -68,6 +68,13 @@
     lit_config.note("Running Shell tests in {} mode.".format(lldb_repro_mode))
     toolchain.use_lldb_repro_substitutions(config, lldb_repro_mode)
 
+if config.lldb_platform_url and cmake.cmake_sysroot and config.enable_remote:
+    if re.match(r".*-linux.*", config.target_triple):
+        config.available_features.add("remote-linux")
+else:
+    # After this, enable_remote == True iff remote testing is going to be used.
+    config.enable_remote = False
+
 llvm_config.use_default_substitutions()
 toolchain.use_lldb_substitutions(config)
 toolchain.use_support_substitutions(config)

lldb/test/Shell/lit.site.cfg.py.in

@@ -10,10 +10,15 @@ config.lldb_src_root = "@LLDB_SOURCE_DIR@"
 config.lldb_obj_root = "@LLDB_BINARY_DIR@"
 config.lldb_libs_dir = lit_config.substitute("@LLDB_LIBS_DIR@")
 config.lldb_tools_dir = lit_config.substitute("@LLDB_TOOLS_DIR@")
+config.lldb_platform_url = lit_config.substitute("@LLDB_TEST_PLATFORM_URL@")
+config.lldb_platform_working_dir = lit_config.substitute("@LLDB_TEST_PLATFORM_WORKING_DIR@")
 # Since it comes from the command line, it may have backslashes which
 # should not need to be escaped.
 config.lldb_lit_tools_dir = lit_config.substitute(r"@LLDB_LIT_TOOLS_DIR@")
-config.cmake_sysroot = lit_config.substitute("@CMAKE_SYSROOT@")
+config.cmake_sysroot = lit_config.substitute("@LLDB_TEST_SYSROOT@" or "@DEFAULT_SYSROOT@")
+config.has_libcxx = @LLDB_HAS_LIBCXX@
+config.enable_remote = not @LLDB_SHELL_TESTS_DISABLE_REMOTE@
+config.libcxx_libs_dir = "@LIBCXX_LIBRARY_DIR@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.python_executable = "@Python3_EXECUTABLE@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@