Adding a skipIfBuildType helper to skip tests that may fail or be unstable in certain build configuraitons.

This revealed a number of tests that were unstable, so I also included fixes for those tests as well.

Author: ashgti

lldb/packages/Python/lldbsuite/test/configuration.py

@@ -137,6 +137,9 @@
 # A plugin whose tests will be enabled, like intel-pt.
 enabled_plugins = []
 
+# the build type of lldb
+# Typical values include Debug, Release, RelWithDebInfo and MinSizeRel
+cmake_build_type = None
 
 def shouldSkipBecauseOfCategories(test_categories):
     if use_categories:

lldb/packages/Python/lldbsuite/test/decorators.py

@@ -1105,26 +1105,13 @@ def is_feature_enabled():
     return skipTestIfFn(is_feature_enabled)
 
 
-def skipIfBinaryToLarge(path: Optional[str], maxSize: int):
-    """Skip the test if a binary is to large.
-
-    We skip this test for debug builds because it takes too long
-    parsing lldb's own debug info. Release builds are fine.
-    Checking the size of the lldb-dap binary seems to be a decent
-    proxy for a quick detection. It should be far less than 1 MB in
-    Release builds.
-    """
-
-    def check_binary_size():
-        if not path or not os.path.exists(path):
-            return "invalid path"
-
-        try:
-            size = os.path.getsize(path)
-            if size <= maxSize:
-                return None
-            return f"binary {path} (size = {size} is to larger than {maxSize}"
-        except:
-            return f"failed to read size of {path}"
+def skipIfBuildType(types: list[str]):
+    """Skip tests if built in a specific CMAKE_BUILD_TYPE.
 
-    return skipTestIfFn(check_binary_size)
+    Supported types include 'Release', 'RelWithDebInfo', 'Debug', 'MinSizeRel'.
+    """
+    types = [name.lower() for name in types]
+    return unittest.skipIf(
+        configuration.cmake_build_type.lower() in types,
+        "skip on {} build type(s)".format(", ".join(types)),
+    )

lldb/packages/Python/lldbsuite/test/dotest.py

@@ -297,6 +297,7 @@ def parseOptionsAndInitTestdirs():
     configuration.libcxx_include_dir = args.libcxx_include_dir
     configuration.libcxx_include_target_dir = args.libcxx_include_target_dir
     configuration.libcxx_library_dir = args.libcxx_library_dir
+    configuration.cmake_build_type = args.cmake_build_type.lower()
 
     if args.channels:
         lldbtest_config.channels = args.channels

lldb/packages/Python/lldbsuite/test/dotest_args.py

@@ -300,6 +300,12 @@ def create_parser():
         metavar="platform-available-ports",
         help="Ports available for connection to a lldb server on the remote platform",
     )
+    group.add_argument(
+        "--cmake-build-type",
+        dest="cmake_build_type",
+        metavar="cmake-build-type",
+        help="Specifies the build type on single-configuration",
+    )
 
     # Test-suite behaviour
     group = parser.add_argument_group("Runtime behaviour options")

lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py

@@ -164,6 +164,7 @@ def __init__(
         self.output: dict[str, list[str]] = {}
         self.configuration_done_sent = False
         self.initialized = False
+        self.terminated = False
         self.frame_scopes = {}
         self.init_commands = init_commands
 
@@ -271,6 +272,10 @@ def _handle_recv_packet(self, packet: Optional[ProtocolMessage]) -> bool:
                 # When a new process is attached or launched, remember the
                 # details that are available in the body of the event
                 self.process_event_body = body
+            elif event == "terminated":
+                # If we get the 'terminated' event then lldb-dap has exited
+                # itself.
+                self.terminated = True
             elif event == "exited":
                 # Process exited, mark the status to indicate the process is not
                 # alive.
@@ -388,7 +393,7 @@ def send_recv(self, command):
                 if response_or_request["command"] == "runInTerminal":
                     subprocess.Popen(
                         response_or_request["arguments"]["args"],
-                        env=response_or_request["arguments"]["env"],
+                        env=response_or_request["arguments"].get("env", {}),
                     )
                     self.send_packet(
                         {
@@ -749,13 +754,11 @@ def request_restart(self, restartArguments=None):
         # Caller must still call wait_for_stopped.
         return response
 
-    def request_disconnect(self, terminateDebuggee=None):
-        args_dict = {}
-        if terminateDebuggee is not None:
-            if terminateDebuggee:
-                args_dict["terminateDebuggee"] = True
-            else:
-                args_dict["terminateDebuggee"] = False
+    def request_disconnect(self, terminateDebuggee=False, suspendDebuggee=False):
+        args_dict = {
+            "terminateDebuggee": terminateDebuggee,
+            "suspendDebuggee": suspendDebuggee,
+        }
         command_dict = {
             "command": "disconnect",
             "type": "request",

lldb/test/API/lit.cfg.py

@@ -271,6 +271,9 @@ def delete_module_cache(path):
 if is_configured("lldb_framework_dir"):
     dotest_cmd += ["--framework", config.lldb_framework_dir]
 
+if is_configured("cmake_build_type"):
+    dotest_cmd += ["--cmake-build-type", config.cmake_build_type]
+
 if "lldb-simulator-ios" in config.available_features:
     dotest_cmd += ["--apple-sdk", "iphonesimulator", "--platform-name", "ios-simulator"]
 elif "lldb-simulator-watchos" in config.available_features:

lldb/test/API/lit.site.cfg.py.in

@@ -40,6 +40,7 @@ config.has_libcxx = @LLDB_HAS_LIBCXX@
 config.libcxx_libs_dir = "@LIBCXX_LIBRARY_DIR@"
 config.libcxx_include_dir = "@LIBCXX_GENERATED_INCLUDE_DIR@"
 config.libcxx_include_target_dir = "@LIBCXX_GENERATED_INCLUDE_TARGET_DIR@"
+config.cmake_build_type = "@CMAKE_BUILD_TYPE@"
 # The API tests use their own module caches.
 config.lldb_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_LLDB@", "lldb-api")
 config.clang_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_CLANG@", "lldb-api")

lldb/test/API/tools/lldb-dap/restart/TestDAP_restart_runInTerminal.py

@@ -2,13 +2,12 @@
 Test lldb-dap restart request.
 """
 
-import os
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import line_number
 import lldbdap_testcase
 
 
-@skipIfBinaryToLarge(os.getenv("LLDBDAP_EXEC"), 1_000_000)
+@skipIfBuildType(["debug"])
 class TestDAP_restart_runInTerminal(lldbdap_testcase.DAPTestCaseBase):
     @skipIfWindows
     @skipIf(oslist=["linux"], archs=["arm"])  # Always times out on buildbot
@@ -56,6 +55,7 @@ def test_stopOnEntry(self):
         program = self.getBuildArtifact("a.out")
         self.build_and_launch(program, runInTerminal=True, stopOnEntry=True)
         [bp_main] = self.set_function_breakpoints(["main"])
+        self.dap_server.request_configurationDone()
 
         # When using stopOnEntry, configurationDone doesn't result in a running
         # process, we should immediately get a stopped event instead.

lldb/test/API/tools/lldb-dap/runInTerminal/TestDAP_runInTerminal.py

@@ -10,7 +10,7 @@
 import json
 
 
-@skipIfBinaryToLarge(os.getenv("LLDBDAP_EXEC"), 1_000_000)
+@skipIfBuildType(["debug"])
 class TestDAP_runInTerminal(lldbdap_testcase.DAPTestCaseBase):
     def readPidMessage(self, fifo_file):
         with open(fifo_file, "r") as file:
@@ -90,6 +90,12 @@ def test_runInTerminalWithObjectEnv(self):
         self.assertIn("FOO", request_envs)
         self.assertEqual("BAR", request_envs["FOO"])
 
+        # Ensure we can continue to a breakpoint.
+        source = "main.c"
+        breakpoint_line = line_number(source, "// breakpoint")
+        self.set_source_breakpoints(source, [breakpoint_line])
+        self.continue_to_next_stop()
+
     @skipIfWindows
     @skipIf(oslist=["linux"], archs=no_match(["x86_64"]))
     def test_runInTerminalInvalidTarget(self):
@@ -103,8 +109,8 @@ def test_runInTerminalInvalidTarget(self):
         )
         self.assertFalse(response["success"])
         self.assertIn(
-            "Could not create a target for a program 'INVALIDPROGRAM': 'INVALIDPROGRAM' does not exist",
-            response["message"],
+            "'INVALIDPROGRAM' does not exist",
+            response["body"]["error"]["format"],
         )
 
     @skipIfWindows

lldb/test/API/tools/lldb-dap/server/TestDAP_server.py

@@ -98,11 +98,12 @@ def test_server_interrupt(self):
         self.set_source_breakpoints(source, [breakpoint_line])
         self.continue_to_next_stop()
 
-        # Interrupt the server which should disconnect all clients.
-        process.send_signal(signal.SIGINT)
-
-        self.dap_server.wait_for_terminated()
         self.assertIsNone(
             self.dap_server.exit_status,
             "Process exited before interrupting lldb-dap server",
         )
+
+        # Interrupt the server which should disconnect all clients.
+        process.send_signal(signal.SIGINT)
+
+        self.dap_server.wait_for_terminated()