gh-91048: Refactor and optimize remote debugging module

Include/cpython/pystate.h

@@ -61,6 +61,8 @@ typedef struct _stack_chunk {
     PyObject * data[1]; /* Variable sized */
 } _PyStackChunk;
 
+/* Minimum size of data stack chunk */
+#define _PY_DATA_STACK_CHUNK_SIZE (16*1024)
 struct _ts {
     /* See Python/ceval.c for comments explaining most fields */
 

Include/internal/pycore_debug_offsets.h

@@ -54,11 +54,13 @@ extern "C" {
 # define _Py_Debug_Free_Threaded 1
 # define _Py_Debug_code_object_co_tlbc offsetof(PyCodeObject, co_tlbc)
 # define _Py_Debug_interpreter_frame_tlbc_index offsetof(_PyInterpreterFrame, tlbc_index)
+# define _Py_Debug_interpreter_state_tlbc_generation offsetof(PyInterpreterState, tlbc_indices.tlbc_generation)
 #else
 # define _Py_Debug_gilruntimestate_enabled 0
 # define _Py_Debug_Free_Threaded 0
 # define _Py_Debug_code_object_co_tlbc 0
 # define _Py_Debug_interpreter_frame_tlbc_index 0
+# define _Py_Debug_interpreter_state_tlbc_generation 0
 #endif
 
 
@@ -89,6 +91,8 @@ typedef struct _Py_DebugOffsets {
         uint64_t gil_runtime_state_enabled;
         uint64_t gil_runtime_state_locked;
         uint64_t gil_runtime_state_holder;
+        uint64_t code_object_generation;
+        uint64_t tlbc_generation;
     } interpreter_state;
 
     // Thread state offset;
@@ -216,6 +220,11 @@ typedef struct _Py_DebugOffsets {
         uint64_t gi_frame_state;
     } gen_object;
 
+    struct _llist_node {
+        uint64_t next;
+        uint64_t prev;
+    } llist_node;
+
     struct _debugger_support {
         uint64_t eval_breaker;
         uint64_t remote_debugger_support;
@@ -251,6 +260,8 @@ typedef struct _Py_DebugOffsets {
         .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \
         .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \
         .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \
+        .code_object_generation = offsetof(PyInterpreterState, _code_object_generation), \
+        .tlbc_generation = _Py_Debug_interpreter_state_tlbc_generation, \
     }, \
     .thread_state = { \
         .size = sizeof(PyThreadState), \
@@ -347,6 +358,10 @@ typedef struct _Py_DebugOffsets {
         .gi_iframe = offsetof(PyGenObject, gi_iframe), \
         .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \
     }, \
+    .llist_node = { \
+        .next = offsetof(struct llist_node, next), \
+        .prev = offsetof(struct llist_node, prev), \
+    }, \
     .debugger_support = { \
         .eval_breaker = offsetof(PyThreadState, eval_breaker), \
         .remote_debugger_support = offsetof(PyThreadState, remote_debugger_support),  \

Include/internal/pycore_interp_structs.h

@@ -726,6 +726,10 @@ typedef struct _PyIndexPool {
 
     // Next index to allocate if no free indices are available
     int32_t next_index;
+
+    // Generation counter incremented on thread creation/destruction
+    // Used for TLBC cache invalidation in remote debugging
+    uint32_t tlbc_generation;
 } _PyIndexPool;
 
 typedef union _Py_unique_id_entry {
@@ -843,6 +847,8 @@ struct _is {
     /* The per-interpreter GIL, which might not be used. */
     struct _gil_runtime_state _gil;
 
+    uint64_t _code_object_generation;
+
      /* ---------- IMPORTANT ---------------------------
      The fields above this line are declared as early as
      possible to facilitate out-of-process observability

Lib/asyncio/tools.py

@@ -5,8 +5,11 @@
 from itertools import count
 from enum import Enum
 import sys
-from _remote_debugging import get_all_awaited_by
+from _remote_debugging import RemoteUnwinder
 
+def get_all_awaited_by(pid):
+    unwinder = RemoteUnwinder(pid)
+    return unwinder.get_all_awaited_by()
 
 class NodeType(Enum):
     COROUTINE = 1

Lib/test/test_external_inspection.py

@@ -4,6 +4,7 @@
 import importlib
 import sys
 import socket
+import threading
 from asyncio import staggered, taskgroups
 from unittest.mock import ANY
 from test.support import os_helper, SHORT_TIMEOUT, busy_retry
@@ -16,9 +17,7 @@
 
 try:
     from _remote_debugging import PROCESS_VM_READV_SUPPORTED
-    from _remote_debugging import get_stack_trace
-    from _remote_debugging import get_async_stack_trace
-    from _remote_debugging import get_all_awaited_by
+    from _remote_debugging import RemoteUnwinder
 except ImportError:
     raise unittest.SkipTest("Test only runs when _remote_debugging is available")
 
@@ -33,6 +32,17 @@ def _make_test_script(script_dir, script_basename, source):
     "Test only runs on Linux, Windows and MacOS",
 )
 
+def get_stack_trace(pid):
+    unwinder = RemoteUnwinder(pid, all_threads=True)
+    return unwinder.get_stack_trace()
+
+def get_async_stack_trace(pid):
+    unwinder = RemoteUnwinder(pid)
+    return unwinder.get_async_stack_trace()
+
+def get_all_awaited_by(pid):
+    unwinder = RemoteUnwinder(pid)
+    return unwinder.get_all_awaited_by()
 
 class TestGetStackTrace(unittest.TestCase):
 
@@ -46,7 +56,7 @@ def test_remote_stack_trace(self):
         port = find_unused_port()
         script = textwrap.dedent(
             f"""\
-            import time, sys, socket
+            import time, sys, socket, threading
             # Connect to the test process
             sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             sock.connect(('localhost', {port}))
@@ -61,7 +71,7 @@ def baz():
             def foo():
                 sock.sendall(b"ready"); time.sleep(10_000)  # same line number
 
-            bar()
+            t = threading.Thread(target=bar); t.start(); t.join()
             """
         )
         stack_trace = None
@@ -94,13 +104,20 @@ def foo():
                 p.terminate()
                 p.wait(timeout=SHORT_TIMEOUT)
 
-            expected_stack_trace = [
+            thread_expected_stack_trace = [
                 ("foo", script_name, 14),
                 ("baz", script_name, 11),
                 ("bar", script_name, 9),
+                ('Thread.run', threading.__file__, ANY)
+            ]
+            main_thread_stack_trace = [
+                (ANY, threading.__file__, ANY),
                 ("<module>", script_name, 16),
             ]
-            self.assertEqual(stack_trace, expected_stack_trace)
+            self.assertEqual(stack_trace, [
+                (ANY, thread_expected_stack_trace),
+                (ANY, main_thread_stack_trace),
+            ])
 
     @skip_if_not_supported
     @unittest.skipIf(
@@ -700,13 +717,21 @@ async def main():
     )
     def test_self_trace(self):
         stack_trace = get_stack_trace(os.getpid())
+        self.assertEqual(stack_trace[0][0], threading.get_native_id())
         self.assertEqual(
-            stack_trace[0],
-            (
-                "TestGetStackTrace.test_self_trace",
-                __file__,
-                self.test_self_trace.__code__.co_firstlineno + 6,
-            ),
+            stack_trace[0][1][:2],
+            [
+                (
+                    "get_stack_trace",
+                    __file__,
+                    get_stack_trace.__code__.co_firstlineno + 2,
+                ),
+                (
+                    "TestGetStackTrace.test_self_trace",
+                    __file__,
+                    self.test_self_trace.__code__.co_firstlineno + 6,
+                ),
+            ]
         )
 
 

Makefile.pre.in

@@ -1206,6 +1206,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/unicodeobject.h \
 		$(srcdir)/Include/warnings.h \
 		$(srcdir)/Include/weakrefobject.h \
+		$(srcdir)/Python/remote_debug.h \
 		\
 		pyconfig.h \
 		$(PARSER_HEADERS) \