[lldb-dap] Support cancel requests and add explicit DAP structures.

This is a refactor of lldb-dap focused on supporting cancel requests and well defined Debug Adapter Protocol (DAP) structures.

Existing request handlers today tend to use unstructured `llvm::json::Object`'s to represent requests and replies. The lack of structure makes it hard for us to apply some unfirom handling around requests to better support cancellation. To address this, this change includes a new `Protocol.h` file with a set of well defined POD structures to represent the types in the DAP and includes `toJSON`/`fromJSON` serialization support.

Building on these types, this includes a new approach to registering request handlers with these new well defined types.

```
template <typename Args, typename Body>
void DAP::RegisterRequest(llvm::StringLiteral command,
                          RequestHandler<Args, Body> handler);
...
llvm::Expected<SourceResponseBody> request_source(DAP &dap, const SourceArguments &args) {
  SourceResponseBody body;
  ...
  if (/*some error*/)
    return llvm::make_error<DAPError>("msg");
  return std::move(body);
}
...
  dap.RegisterRequest("source", request_source);
...
```

The main features of this refactor are:

* Created a queue of pending protocol messages to allow for preemptive cancellations and active cancellations.
* A cleaner separation between the Debug Adapter Protocol types and lldb types that include serialization support toJSON/fromJSON.
* Improved error handling using toJSON/fromJSON.
* Unified handling of responses and errors.
* Unified handling of cancel requests.

This change include minimal support to implement the Debug Adapter Protocol 'source' request, 'evaluate' request and the 'exited' event.

Author: ashgti

lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py

@@ -144,9 +144,17 @@ def encode_content(cls, s):
     @classmethod
     def validate_response(cls, command, response):
         if command["command"] != response["command"]:
-            raise ValueError("command mismatch in response")
+            raise ValueError(
+                "command mismatch in response, got '{}', expected '{}', response: {}".format(
+                    command["command"], response["command"], response
+                )
+            )
         if command["seq"] != response["request_seq"]:
-            raise ValueError("seq mismatch in response")
+            raise ValueError(
+                "seq mismatch in response, got '{}', expected {}, response: {}".format(
+                    command["seq"], response["request_seq"], response
+                )
+            )
 
     def get_modules(self):
         module_list = self.request_modules()["body"]["modules"]

lldb/test/API/tools/lldb-dap/cancel/Makefile

@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules

lldb/test/API/tools/lldb-dap/cancel/TestDAP_cancel.py

@@ -0,0 +1,92 @@
+"""
+Test lldb-dap cancel request
+"""
+
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+import lldbdap_testcase
+
+
+class TestDAP_launch(lldbdap_testcase.DAPTestCaseBase):
+    def send_async_req(self, command: str, arguments={}) -> int:
+        seq = self.dap_server.sequence
+        self.dap_server.send_packet(
+            {
+                "type": "request",
+                "command": command,
+                "arguments": arguments,
+            }
+        )
+        return seq
+
+    def async_blocking_request(self, duration: float) -> int:
+        """
+        Sends an evaluate request that will sleep for the specified duration to
+        block the request handling thread.
+        """
+        return self.send_async_req(
+            command="evaluate",
+            arguments={
+                "expression": "`script import time; time.sleep({})".format(duration),
+                "context": "repl",
+            },
+        )
+
+    def async_cancel(self, requestId: int) -> int:
+        return self.send_async_req(command="cancel", arguments={"requestId": requestId})
+
+    def test_pending_request(self):
+        """
+        Tests cancelling a pending request.
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program, stopOnEntry=True)
+        self.continue_to_next_stop()
+
+        # Use a relatively short timeout since this is only to ensure the
+        # following request is queued.
+        blocking_seq = self.async_blocking_request(duration=1.0)
+        # Use a longer timeout to ensure we catch if the request was interrupted
+        # properly.
+        pending_seq = self.async_blocking_request(duration=self.timeoutval)
+        cancel_seq = self.async_cancel(requestId=pending_seq)
+
+        blocking_resp = self.dap_server.recv_packet(filter_type=["response"])
+        self.assertEqual(blocking_resp["request_seq"], blocking_seq)
+        self.assertEqual(blocking_resp["command"], "evaluate")
+        self.assertEqual(blocking_resp["success"], True)
+
+        pending_resp = self.dap_server.recv_packet(filter_type=["response"])
+        self.assertEqual(pending_resp["request_seq"], pending_seq)
+        self.assertEqual(pending_resp["command"], "evaluate")
+        self.assertEqual(pending_resp["success"], False)
+        self.assertEqual(pending_resp["message"], "cancelled")
+
+        cancel_resp = self.dap_server.recv_packet(filter_type=["response"])
+        self.assertEqual(cancel_resp["request_seq"], cancel_seq)
+        self.assertEqual(cancel_resp["command"], "cancel")
+        self.assertEqual(cancel_resp["success"], True)
+        self.continue_to_exit()
+
+    def test_inflight_request(self):
+        """
+        Tests cancelling an inflight request.
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program, stopOnEntry=True)
+        self.continue_to_next_stop()
+
+        blocking_seq = self.async_blocking_request(duration=self.timeoutval)
+        cancel_seq = self.async_cancel(requestId=blocking_seq)
+
+        blocking_resp = self.dap_server.recv_packet(filter_type=["response"])
+        self.assertEqual(blocking_resp["request_seq"], blocking_seq)
+        self.assertEqual(blocking_resp["command"], "evaluate")
+        self.assertEqual(blocking_resp["success"], False)
+        self.assertEqual(blocking_resp["message"], "cancelled")
+
+        cancel_resp = self.dap_server.recv_packet(filter_type=["response"])
+        self.assertEqual(cancel_resp["request_seq"], cancel_seq)
+        self.assertEqual(cancel_resp["command"], "cancel")
+        self.assertEqual(cancel_resp["success"], True)
+        self.continue_to_exit()

lldb/test/API/tools/lldb-dap/cancel/main.c

@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+int main(int argc, char const *argv[]) {
+  printf("Hello world!\n");
+  return 0;
+}

lldb/test/API/tools/lldb-dap/evaluate/TestDAP_evaluate.py

@@ -3,6 +3,7 @@
 """
 
 import re
+import json
 
 import lldbdap_testcase
 import dap_server
@@ -13,17 +14,21 @@
 
 class TestDAP_evaluate(lldbdap_testcase.DAPTestCaseBase):
     def assertEvaluate(self, expression, regex):
-        self.assertRegex(
-            self.dap_server.request_evaluate(expression, context=self.context)["body"][
-                "result"
-            ],
-            regex,
+        resp = self.dap_server.request_evaluate(expression, context=self.context)
+        self.assertEqual(
+            resp["success"],
+            True,
+            "evaluate with expression = '{}' context = '{}' failed unexpectedly: {}".format(
+                expression, self.context, json.dumps(resp)
+            ),
         )
+        self.assertRegex(resp["body"]["result"], regex)
 
     def assertEvaluateFailure(self, expression):
-        self.assertNotIn(
-            "result",
-            self.dap_server.request_evaluate(expression, context=self.context)["body"],
+        self.assertFalse(
+            self.dap_server.request_evaluate(expression, context=self.context)[
+                "success"
+            ],
         )
 
     def isResultExpandedDescription(self):
@@ -231,5 +236,7 @@ def test_hover_evaluate_expressions(self):
 
     @skipIfWindows
     def test_variable_evaluate_expressions(self):
-        # Tests expression evaluations that are triggered in the variable explorer
-        self.run_test_evaluate_expressions("variable", enableAutoVariableSummaries=True)
+        # Tests expression evaluations that are triggered in the variables explorer
+        self.run_test_evaluate_expressions(
+            "variables", enableAutoVariableSummaries=True
+        )

lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py

@@ -1,13 +1,10 @@
 """
-Test lldb-dap setBreakpoints request
+Test lldb-dap launch request
 """
 
-import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
 import lldbdap_testcase
-import time
 import os
 import re
 
@@ -41,7 +38,9 @@ def test_termination(self):
         self.dap_server.request_disconnect()
 
         # Wait until the underlying lldb-dap process dies.
-        self.dap_server.process.wait(timeout=lldbdap_testcase.DAPTestCaseBase.timeoutval)
+        self.dap_server.process.wait(
+            timeout=lldbdap_testcase.DAPTestCaseBase.timeoutval
+        )
 
         # Check the return code
         self.assertEqual(self.dap_server.process.poll(), 0)

lldb/test/API/tools/lldb-dap/optimized/TestDAP_optimized.py

@@ -24,11 +24,11 @@ def test_stack_frame_name(self):
         )
         self.continue_to_breakpoints(breakpoint_ids)
         leaf_frame = self.dap_server.get_stackFrame(frameIndex=0)
-        self.assertTrue(leaf_frame["name"].endswith(" [opt]"))
+        self.assertRegex(leaf_frame["name"], r"\[.*opt.*\]")
         parent_frame = self.dap_server.get_stackFrame(frameIndex=1)
-        self.assertTrue(parent_frame["name"].endswith(" [opt]"))
+        self.assertRegex(parent_frame["name"], r"\[.*opt.*\]")
 
-    @skipIfAsan # On ASAN builds this test intermittently fails https://github.com/llvm/llvm-project/issues/111061
+    @skipIfAsan  # On ASAN builds this test intermittently fails https://github.com/llvm/llvm-project/issues/111061
     @skipIfWindows
     def test_optimized_variable(self):
         """Test optimized variable value contains error."""

lldb/test/API/tools/lldb-dap/server/TestDAP_server.py

@@ -23,6 +23,7 @@ def start_server(self, connection):
 
         def cleanup():
             process.terminate()
+            dap_server.dump_dap_log(log_file_path)
 
         self.addTearDownHook(cleanup)
 
@@ -47,6 +48,7 @@ def run_debug_session(self, connection, name):
         output = self.get_stdout()
         self.assertEqual(output, f"Hello {name}!\r\n")
         self.dap_server.request_disconnect()
+        self.dap_server.terminate()
 
     def test_server_port(self):
         """

lldb/tools/lldb-dap/CMakeLists.txt

@@ -32,6 +32,7 @@ add_lldb_tool(lldb-dap
   LLDBUtils.cpp
   OutputRedirector.cpp
   ProgressEvent.cpp
+  Protocol.cpp
   RunInTerminal.cpp
   SourceBreakpoint.cpp
   Watchpoint.cpp