From 3fef8c7a2d9cf56c280fefb33936bf14f73cff8a Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 12 May 2025 16:36:59 -0400 Subject: [PATCH 01/12] Add attach command to pdb --- Lib/pdb.py | 65 +++++++++++++++++ Lib/test/test_remote_pdb.py | 141 ++++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+) diff --git a/Lib/pdb.py b/Lib/pdb.py index f89d104fcddb9a..d394fc00781d43 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1961,6 +1961,39 @@ def do_debug(self, arg): complete_debug = _complete_expression + def do_attach(self, process): + """attach process + + Attach to process, which can be a subprocess.Popen, + multiprocessing.Process or a pid. + """ + import multiprocessing + import subprocess + + try: + process = self._getval(process) + except: + # Error message is already displayed + return + + if isinstance(process, subprocess.Popen): + pid = process.pid + elif isinstance(process, multiprocessing.Process): + pid = process.pid + elif isinstance(process, int): + pid = process + else: + self.error("Invalid process: %s" % process) + return + + self.message(f"Attaching to process {pid}") + try: + attach(pid) + except Exception as e: + self._error_exc() + return + self.message(f"Detached from process {pid}") + def do_quit(self, arg): """q(uit) | exit @@ -2741,6 +2774,8 @@ def _ensure_valid_message(self, msg): # Due to aliases this list is not static, but the client # needs to know it for multi-line editing. pass + case {"attach": int()}: + pass case _: raise AssertionError( f"PDB message doesn't follow the schema! {msg}" @@ -2925,6 +2960,28 @@ def detach(self): # close() can fail if the connection was broken unexpectedly. pass + def do_attach(self, process): + import multiprocessing + import subprocess + + try: + process = self._getval(process) + except: + # Error message is already displayed + return + + if isinstance(process, subprocess.Popen): + pid = process.pid + elif isinstance(process, multiprocessing.Process): + pid = process.pid + elif isinstance(process, int): + pid = process + else: + self.error("Invalid process: %s" % process) + return + + self._send(attach=pid) + def do_debug(self, arg): # Clear our cached list of valid commands; the recursive debugger might # send its own differing list, and so ours needs to be re-sent. @@ -3277,6 +3334,14 @@ def process_payload(self, payload): state = "dumb" self.state = state self.prompt_for_reply(prompt) + case {"attach": int(pid)}: + print(f"Attaching to process {pid}") + try: + attach(pid) + print(f"Detached from process {pid}") + except Exception as exc: + msg = traceback.format_exception_only(exc)[-1].strip() + print("***", msg, flush=True) case _: raise RuntimeError(f"Unrecognized payload {payload}") diff --git a/Lib/test/test_remote_pdb.py b/Lib/test/test_remote_pdb.py index aef8a6b0129092..2dfccbd9d1fadc 100644 --- a/Lib/test/test_remote_pdb.py +++ b/Lib/test/test_remote_pdb.py @@ -819,6 +819,18 @@ def test_reading_empty_json_during_completion(self): expected_state={"state": "interact"}, ) + def test_client_attach(self): + with unittest.mock.patch("pdb.attach") as mock_attach: + incoming = [ + ("server", {"attach": 1234}), + ] + self.do_test( + incoming=incoming, + expected_outgoing=[], + expected_stdout_substring="Attaching to process 1234", + ) + mock_attach.assert_called_once_with(1234) + class RemotePdbTestCase(unittest.TestCase): """Tests for the _PdbServer class.""" @@ -957,6 +969,15 @@ def test_registering_commands(self): ["_pdbcmd_silence_frame_status", "print('hi')"], ) + def test_server_attach(self): + self.sockfile.add_input({"reply": "attach 1234"}) + self.sockfile.add_input({"signal": "EOF"}) + + self.pdb.cmdloop() + + outputs = self.sockfile.get_output() + self.assertEqual(outputs[2], {"attach": 1234}) + def test_detach(self): """Test the detach method.""" with unittest.mock.patch.object(self.sockfile, 'close') as mock_close: @@ -1579,5 +1600,125 @@ def test_attach_to_process_with_colors(self): self.assertNotIn("while x == 1", output["client"]["stdout"]) self.assertIn("while x == 1", re.sub("\x1b[^m]*m", "", output["client"]["stdout"])) + +@unittest.skipIf(not sys.is_remote_debug_enabled(), "Remote debugging is not enabled") +@unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux" and sys.platform != "win32", + "Test only runs on Linux, Windows and MacOS") +@cpython_only +@requires_subprocess() +class PdbAttachCommand(unittest.TestCase): + def do_test(self, target, commands): + with tempfile.TemporaryDirectory() as tmpdir: + target = textwrap.dedent(target) + target_path = os.path.join(tmpdir, "target.py") + with open(target_path, "wt") as f: + f.write(target) + + script = textwrap.dedent( + f""" + import subprocess + import sys + process = subprocess.Popen([sys.executable, {target_path!r}], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + breakpoint() + """) + script_path = os.path.join(tmpdir, "script.py") + + with open(script_path, "wt") as f: + f.write(script) + + process = subprocess.Popen( + [sys.executable, script_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + text=True + ) + + self.addCleanup(process.terminate) + + self.addCleanup(process.stdout.close) + self.addCleanup(process.stderr.close) + + stdout, stderr = process.communicate(textwrap.dedent(commands), + timeout=SHORT_TIMEOUT) + + return stdout, stderr + + def test_attach_simple(self): + """Test basic attach command""" + target = """ + block = True + import time + while block: + time.sleep(0.2) + def test_function(): + x = 42 + return x + test_function() + """ + + commands = """ + attach process + block = False + b test_function + c + n + p x + 42 + quit + continue + """ + stdout, _ = self.do_test(target, commands) + self.assertIn("84", stdout) + + def test_attach_multiprocessing(self): + """Spawn a process with multiprocessing and attach to it.""" + target = """ + block = True + import time + import multiprocessing + + def worker(queue): + block = True + queue.put(42) + while block: + time.sleep(0.2) + + def test_function(queue): + data = queue.get() + return data + + if __name__ == '__main__': + while block: + time.sleep(0.2) + + queue = multiprocessing.Queue() + p = multiprocessing.Process(target=worker, args=(queue,)) + p.start() + test_function(queue) + p.join() + """ + + commands = """ + attach process + block = False + b test_function + c + attach p + block = False + q + n + p data + 42 + quit + continue + """ + stdout, _ = self.do_test(target, commands) + self.assertIn("84", stdout) + + + if __name__ == "__main__": unittest.main() From 9ddd1001421f71e9612c3f8c8d58b942b2635004 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 20:58:16 +0000 Subject: [PATCH 02/12] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2025-05-12-20-58-11.gh-issue-133953.1dswu9.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-05-12-20-58-11.gh-issue-133953.1dswu9.rst diff --git a/Misc/NEWS.d/next/Library/2025-05-12-20-58-11.gh-issue-133953.1dswu9.rst b/Misc/NEWS.d/next/Library/2025-05-12-20-58-11.gh-issue-133953.1dswu9.rst new file mode 100644 index 00000000000000..7332826df9a9da --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-12-20-58-11.gh-issue-133953.1dswu9.rst @@ -0,0 +1 @@ +``attach`` command is added to :mod:`pdb` to attach to a running process. From 2a9887f1a48058801caed08276c0dc40b5d163c0 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 12 May 2025 16:52:47 -0400 Subject: [PATCH 03/12] Unify process to pid --- Lib/pdb.py | 78 +++++++++++++++++++++++------------------------------- 1 file changed, 33 insertions(+), 45 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index d394fc00781d43..363ecfaed469b1 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -709,6 +709,27 @@ def _get_asyncio_task(self): task = None return task + def _get_pid_from_process(self, process): + """process could be a subprocess.Popen, multiprocessing.Process or a pid + """ + # They are not used elsewhere so do a lazy import + from multiprocessing import Process + from subprocess import Popen + + try: + process = self._getval(process) + except: + # Error message is already displayed + return None + + if isinstance(process, (Process, Popen)): + return process.pid + elif isinstance(process, int): + return process + + self.error(f"Invalid process {process}") + return None + def interaction(self, frame, tb_or_exc): # Restore the previous signal handler at the Pdb prompt. if Pdb._previous_sigint_handler: @@ -1967,32 +1988,16 @@ def do_attach(self, process): Attach to process, which can be a subprocess.Popen, multiprocessing.Process or a pid. """ - import multiprocessing - import subprocess - - try: - process = self._getval(process) - except: - # Error message is already displayed - return + pid = self._get_pid_from_process(process) - if isinstance(process, subprocess.Popen): - pid = process.pid - elif isinstance(process, multiprocessing.Process): - pid = process.pid - elif isinstance(process, int): - pid = process - else: - self.error("Invalid process: %s" % process) - return - - self.message(f"Attaching to process {pid}") - try: - attach(pid) - except Exception as e: - self._error_exc() - return - self.message(f"Detached from process {pid}") + if pid is not None: + self.message(f"Attaching to process {pid}") + try: + attach(pid) + except Exception as e: + self._error_exc() + return + self.message(f"Detached from process {pid}") def do_quit(self, arg): """q(uit) | exit @@ -2961,26 +2966,9 @@ def detach(self): pass def do_attach(self, process): - import multiprocessing - import subprocess - - try: - process = self._getval(process) - except: - # Error message is already displayed - return - - if isinstance(process, subprocess.Popen): - pid = process.pid - elif isinstance(process, multiprocessing.Process): - pid = process.pid - elif isinstance(process, int): - pid = process - else: - self.error("Invalid process: %s" % process) - return - - self._send(attach=pid) + pid = self._get_pid_from_process(process) + if pid is not None: + self._send(attach=pid) def do_debug(self, arg): # Clear our cached list of valid commands; the recursive debugger might From f48a1c609adae824e30b27a9e8ac5d3bc667f88a Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 12 May 2025 16:57:22 -0400 Subject: [PATCH 04/12] Add docs --- Doc/library/pdb.rst | 7 +++++++ Doc/whatsnew/3.15.rst | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index a0304edddf6478..d6dd7128e09cb1 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -699,6 +699,13 @@ can be overridden by the local file. :pdbcmd:`interact` directs its output to the debugger's output channel rather than :data:`sys.stderr`. +.. pdbcommand:: attach process + + Attach to a running process. The *process* argument could be either a + :class:`subprocess.Popen`, :class:`multiprocessing.Process` or a process ID. + + .. versionadded:: 3.15 + .. _debugger-aliases: .. pdbcommand:: alias [name [command]] diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 6ce7f964020fb9..5b04debf6d0477 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -89,6 +89,12 @@ New modules Improved modules ================ +pdb +--- + +* :pdbcommand:`attach` command has been added to attach to a running process. + (Contributed by Tian Gao in :gh:`133954`.) + ssl --- From 93e5f5803eb7274da5ec50580ef8d9bd7f7f98b1 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 12 May 2025 16:59:20 -0400 Subject: [PATCH 05/12] Polish the whatsnew entry a bit --- Doc/whatsnew/3.15.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 5b04debf6d0477..a19f31c95671bf 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -92,7 +92,8 @@ Improved modules pdb --- -* :pdbcommand:`attach` command has been added to attach to a running process. +* :pdbcommand:`attach` command is added to attach to a running process + from :mod:`pdb`. (Contributed by Tian Gao in :gh:`133954`.) ssl From 52976669c83fe3cee3c1609d4622f3cb6b733b02 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 12 May 2025 17:00:51 -0400 Subject: [PATCH 06/12] Fix lint --- Doc/library/pdb.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index d6dd7128e09cb1..45c619d108a379 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -700,7 +700,7 @@ can be overridden by the local file. output channel rather than :data:`sys.stderr`. .. pdbcommand:: attach process - + Attach to a running process. The *process* argument could be either a :class:`subprocess.Popen`, :class:`multiprocessing.Process` or a process ID. From 1e774a0e12e299e82a922b29b3286f365146c92b Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 12 May 2025 17:05:58 -0400 Subject: [PATCH 07/12] Fix docs --- Doc/whatsnew/3.15.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 99f679acb23d64..b0b30f84c6faa6 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -92,8 +92,7 @@ Improved modules pdb --- -* :pdbcommand:`attach` command is added to attach to a running process - from :mod:`pdb`. +* ``attach`` command is added to attach to a running process from :mod:`pdb`. (Contributed by Tian Gao in :gh:`133954`.) ssl From b3ffe8c30d7ec5335f6db6bf4c6c8d5cfecff1c9 Mon Sep 17 00:00:00 2001 From: gaogaotiantian Date: Mon, 12 May 2025 18:12:26 -0400 Subject: [PATCH 08/12] Add a sync mechanism --- Lib/test/test_remote_pdb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_remote_pdb.py b/Lib/test/test_remote_pdb.py index 2dfccbd9d1fadc..5c3e8d01f33d47 100644 --- a/Lib/test/test_remote_pdb.py +++ b/Lib/test/test_remote_pdb.py @@ -1683,9 +1683,10 @@ def test_attach_multiprocessing(self): def worker(queue): block = True - queue.put(42) + queue.put(0) while block: time.sleep(0.2) + queue.put(42) def test_function(queue): data = queue.get() @@ -1698,6 +1699,7 @@ def test_function(queue): queue = multiprocessing.Queue() p = multiprocessing.Process(target=worker, args=(queue,)) p.start() + queue.get() test_function(queue) p.join() """ From 13d77ad782ffbe7fd82c10883308f66c0a133f10 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 12 May 2025 18:58:03 -0400 Subject: [PATCH 09/12] Skip the test if env does not support it --- Lib/test/test_remote_pdb.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Lib/test/test_remote_pdb.py b/Lib/test/test_remote_pdb.py index 5c3e8d01f33d47..32bc72eb477272 100644 --- a/Lib/test/test_remote_pdb.py +++ b/Lib/test/test_remote_pdb.py @@ -1607,6 +1607,45 @@ def test_attach_to_process_with_colors(self): @cpython_only @requires_subprocess() class PdbAttachCommand(unittest.TestCase): + + @classmethod + def setUpClass(cls): + # We need to do a quick test to see if we have the permission to remote + # execute the code. If not, just skip the whole test. + script_path = TESTFN + "script.py" + remote_path = TESTFN + "remote.py" + script = textwrap.dedent(""" + import time + print("ready", flush=True) + while True: + print('hello') + time.sleep(0.1) + """) + + with open(script_path, "w") as f: + f.write(script) + + with open(remote_path, "w") as f: + f.write("pass\n") + + with subprocess.Popen( + [sys.executable, script_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) as proc: + try: + proc.stdout.readline() + sys.remote_exec(proc.pid, remote_path) + except PermissionError: + print("raise") + # Skip the test if we don't have permission to execute remote code + raise unittest.SkipTest("We don't have permission to execute remote code") + finally: + os.unlink(script_path) + os.unlink(remote_path) + proc.terminate() + def do_test(self, target, commands): with tempfile.TemporaryDirectory() as tmpdir: target = textwrap.dedent(target) From 9c8302635f22a7d3ab45aada88dcceb5266bd505 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 24 May 2025 11:35:51 -0400 Subject: [PATCH 10/12] Address comments and add a test --- Lib/pdb.py | 32 ++++++++++++++++++-------------- Lib/test/test_remote_pdb.py | 11 +++++++++++ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 1e492195b74e4c..5299cd9a46625d 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -710,11 +710,8 @@ def _get_asyncio_task(self): return task def _get_pid_from_process(self, process): - """process could be a subprocess.Popen, multiprocessing.Process or a pid + """process could evaluate to any object with a `process` attribute or an integer """ - # They are not used elsewhere so do a lazy import - from multiprocessing import Process - from subprocess import Popen try: process = self._getval(process) @@ -722,13 +719,13 @@ def _get_pid_from_process(self, process): # Error message is already displayed return None - if isinstance(process, (Process, Popen)): - return process.pid - elif isinstance(process, int): - return process + pid = getattr(process, "pid", process) - self.error(f"Invalid process {process}") - return None + if not isinstance(pid, int): + self.error(f"Invalid process {process!r}") + return None + + return pid def interaction(self, frame, tb_or_exc): # Restore the previous signal handler at the Pdb prompt. @@ -1982,13 +1979,13 @@ def do_debug(self, arg): complete_debug = _complete_expression - def do_attach(self, process): + def do_attach(self, arg): """attach process - Attach to process, which can be a subprocess.Popen, - multiprocessing.Process or a pid. + Attach to process, which can be any object that has a pid + attribute or a process ID. """ - pid = self._get_pid_from_process(process) + pid = self._get_pid_from_process(arg) if pid is not None: self.message(f"Attaching to process {pid}") @@ -2780,6 +2777,7 @@ def _ensure_valid_message(self, msg): # needs to know it for multi-line editing. pass case {"attach": int()}: + # Ask the client to attach to the given process ID. pass case _: raise AssertionError( @@ -3441,6 +3439,12 @@ def _connect( def attach(pid, commands=()): """Attach to a running process with the given PID.""" + + if threading.current_thread() is not threading.main_thread(): + raise RuntimeError( + "pdb.attach() must be called from the main thread" + ) + with ExitStack() as stack: server = stack.enter_context( closing(socket.create_server(("localhost", 0))) diff --git a/Lib/test/test_remote_pdb.py b/Lib/test/test_remote_pdb.py index 32bc72eb477272..ed9c82a323ed97 100644 --- a/Lib/test/test_remote_pdb.py +++ b/Lib/test/test_remote_pdb.py @@ -1600,6 +1600,17 @@ def test_attach_to_process_with_colors(self): self.assertNotIn("while x == 1", output["client"]["stdout"]) self.assertIn("while x == 1", re.sub("\x1b[^m]*m", "", output["client"]["stdout"])) + def test_attach_from_worker_thread(self): + # Test attaching from a worker thread + def worker(): + with self.assertRaises(RuntimeError): + # We are not allowed to attach from a thread that's not main + pdb.attach(1234) + + thread = threading.Thread(target=worker) + thread.start() + thread.join() + @unittest.skipIf(not sys.is_remote_debug_enabled(), "Remote debugging is not enabled") @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux" and sys.platform != "win32", From 455d9bbe771aae5caba47f2a45a7788e0a92eabe Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 24 May 2025 22:01:54 -0400 Subject: [PATCH 11/12] Remove space --- Lib/pdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index 5299cd9a46625d..87670a7009347d 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -725,7 +725,7 @@ def _get_pid_from_process(self, process): self.error(f"Invalid process {process!r}") return None - return pid + return pid def interaction(self, frame, tb_or_exc): # Restore the previous signal handler at the Pdb prompt. From 77fb481ad49db14ce898b172ac587199d192b883 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 25 May 2025 11:51:59 -0400 Subject: [PATCH 12/12] Update pdb.rst --- Doc/library/pdb.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 45c619d108a379..2e8260a8adb9d7 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -702,7 +702,8 @@ can be overridden by the local file. .. pdbcommand:: attach process Attach to a running process. The *process* argument could be either a - :class:`subprocess.Popen`, :class:`multiprocessing.Process` or a process ID. + process ID, or any object that has a ``pid`` attribute like + :class:`subprocess.Popen` or :class:`multiprocessing.Process`. .. versionadded:: 3.15