From cf73b09941657ebb9541e7d8450f522a1ac72408 Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Wed, 2 Mar 2022 11:59:45 +0000 Subject: [PATCH 01/13] Prevent hanging test on Windows Not sure why this is not needed on CI, but I seem to need it locally. --- ipykernel/tests/test_debugger.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 43a96ef22..5862d8a52 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -32,7 +32,10 @@ def wait_for_debug_request(kernel, command, arguments=None, full_reply=False): @pytest.fixture -def kernel(): +def kernel(request): + if sys.platform == "win32": + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) with new_kernel() as kc: yield kc From 4ef5f6d7b12aea98c4c6b9a717a01ecf0b165f07 Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Fri, 25 Feb 2022 16:05:06 +0000 Subject: [PATCH 02/13] Make test util wait_for_debug_event --- ipykernel/tests/test_debugger.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 5862d8a52..59169978e 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -31,6 +31,13 @@ def wait_for_debug_request(kernel, command, arguments=None, full_reply=False): return reply if full_reply else reply["content"] +def wait_for_debug_event(kernel, event): + msg = {"msg_type": "", "content": {}} + while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != event: + msg = kernel.get_iopub_msg(timeout=TIMEOUT) + return msg + + @pytest.fixture def kernel(request): if sys.platform == "win32": @@ -156,12 +163,9 @@ def test_stop_on_breakpoint(kernel_with_debug): wait_for_debug_request(kernel_with_debug, "configurationDone", full_reply=True) kernel_with_debug.execute(code) - - # Wait for stop on breakpoint - msg = {"msg_type": "", "content": {}} - while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != "stopped": - msg = kernel_with_debug.get_iopub_msg(timeout=TIMEOUT) + # Wait for stop on breakpoint + msg = wait_for_debug_event(kernel_with_debug, "stopped") assert msg["content"]["body"]["reason"] == "breakpoint" @@ -192,12 +196,9 @@ def f(a, b): wait_for_debug_request(kernel_with_debug, "configurationDone", full_reply=True) kernel_with_debug.execute(code) - - # Wait for stop on breakpoint - msg = {"msg_type": "", "content": {}} - while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != "stopped": - msg = kernel_with_debug.get_iopub_msg(timeout=TIMEOUT) + # Wait for stop on breakpoint + msg = wait_for_debug_event(kernel_with_debug, "stopped") assert msg["content"]["body"]["reason"] == "breakpoint" @@ -251,9 +252,7 @@ def test_rich_inspect_at_breakpoint(kernel_with_debug): kernel_with_debug.execute(code) # Wait for stop on breakpoint - msg = {"msg_type": "", "content": {}} - while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != "stopped": - msg = kernel_with_debug.get_iopub_msg(timeout=TIMEOUT) + wait_for_debug_event(kernel_with_debug, "stopped") stacks = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": 1})[ "body" From 18969078f82036b7e430656d3cd0adebf35903e4 Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Fri, 25 Feb 2022 16:06:38 +0000 Subject: [PATCH 03/13] Add test for stepping into a lib --- ipykernel/tests/test_debugger.py | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 59169978e..980608162 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -224,6 +224,46 @@ def test_rich_inspect_not_at_breakpoint(kernel_with_debug): assert reply["body"]["data"] == {"text/plain": f"'{value}'"} +def test_step_into_lib(kernel_with_debug): + code = """import traitlets +traitlets.validate('foo', 'bar') +""" + + r = wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code}) + source = r["body"]["sourcePath"] + + wait_for_debug_request( + kernel_with_debug, + "setBreakpoints", + { + "breakpoints": [{"line": 1}], + "source": {"path": source}, + "sourceModified": False, + }, + ) + + wait_for_debug_request(kernel_with_debug, "debugInfo") + + r = wait_for_debug_request(kernel_with_debug, "configurationDone") + kernel_with_debug.execute(code) + + # Wait for stop on breakpoint + wait_for_debug_event(kernel_with_debug, "stopped") + + # Setp over the import statement + wait_for_debug_request(kernel_with_debug, "next", {"threadId": 1}) + wait_for_debug_event(kernel_with_debug, "stopped") + # Attempt to step into the function call + wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": 1}) + wait_for_debug_event(kernel_with_debug, "stopped") + + reply = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": 1}) + + names = [f.get("name") for f in reply["body"]["stackFrames"]] + # "" will be the name of the cell + assert names == ["validate", ""] + + def test_rich_inspect_at_breakpoint(kernel_with_debug): code = """def f(a, b): c = a + b From ed384dcab24b0f16545a82795ed2e7842872da8d Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Wed, 2 Mar 2022 12:01:23 +0000 Subject: [PATCH 04/13] Run test with "just_my_code" --- ipykernel/tests/test_debugger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 980608162..24f535979 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -43,7 +43,7 @@ def kernel(request): if sys.platform == "win32": import asyncio asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - with new_kernel() as kc: + with new_kernel(getattr(request, "param", None)) as kc: yield kc @@ -224,6 +224,7 @@ def test_rich_inspect_not_at_breakpoint(kernel_with_debug): assert reply["body"]["data"] == {"text/plain": f"'{value}'"} +@pytest.mark.parametrize("kernel", [["--Kernel.debug_just_my_code=False"]], indirect=True) def test_step_into_lib(kernel_with_debug): code = """import traitlets traitlets.validate('foo', 'bar') From 6359437ba1f8ddb74384c6c37e8ea741caabbc56 Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Wed, 2 Mar 2022 16:22:29 +0000 Subject: [PATCH 05/13] Add step past end test Test to cover what happens if we step past the end of a cell's code. Expected behavior is to continue until next execute arrives, and then stop on the first statement in the new code. --- ipykernel/tests/test_debugger.py | 47 ++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 24f535979..24a4575ae 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -1,3 +1,4 @@ +from queue import Empty import sys import pytest @@ -31,10 +32,14 @@ def wait_for_debug_request(kernel, command, arguments=None, full_reply=False): return reply if full_reply else reply["content"] -def wait_for_debug_event(kernel, event): +def wait_for_debug_event(kernel, event, timeout=TIMEOUT, verbose=False): msg = {"msg_type": "", "content": {}} while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != event: - msg = kernel.get_iopub_msg(timeout=TIMEOUT) + msg = kernel.get_iopub_msg(timeout=timeout) + if verbose: + print(msg.get("msg_type")) + if (msg.get("msg_type") == "debug_event"): + print(f' {msg["content"].get("event")}') return msg @@ -265,6 +270,44 @@ def test_step_into_lib(kernel_with_debug): assert names == ["validate", ""] +def test_step_into_end(kernel_with_debug): + code = 'print("foo")\n' + + r = wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code}) + source = r["body"]["sourcePath"] + + wait_for_debug_request( + kernel_with_debug, + "setBreakpoints", + { + "breakpoints": [{"line": 1}], + "source": {"path": source}, + "sourceModified": False, + }, + ) + + wait_for_debug_request(kernel_with_debug, "debugInfo") + + r = wait_for_debug_request(kernel_with_debug, "configurationDone") + kernel_with_debug.execute(code) + + # Wait for stop on breakpoint + wait_for_debug_event(kernel_with_debug, "stopped") + + # Attempt to step into the print statement (will continue execution, but + # should stop on first line of next execute request) + wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": 1}) + # assert no stop statement is given + with pytest.raises(Empty): + wait_for_debug_event(kernel_with_debug, "stopped", timeout=3) + + # execute some new code without breakpoints, assert it stops + code = 'print("bar")\nprint("alice")\n' + wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code}) + kernel_with_debug.execute(code) + wait_for_debug_event(kernel_with_debug, "stopped") + + def test_rich_inspect_at_breakpoint(kernel_with_debug): code = """def f(a, b): c = a + b From d634e5a902cd315ec7ebde2b203d79c202c867c4 Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Thu, 3 Mar 2022 11:20:22 +0000 Subject: [PATCH 06/13] Improve step past edge test --- ipykernel/tests/test_debugger.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 24a4575ae..63fa49340 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -271,7 +271,7 @@ def test_step_into_lib(kernel_with_debug): def test_step_into_end(kernel_with_debug): - code = 'print("foo")\n' + code = '5 + 5;\n' r = wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code}) source = r["body"]["sourcePath"] @@ -298,8 +298,19 @@ def test_step_into_end(kernel_with_debug): # should stop on first line of next execute request) wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": 1}) # assert no stop statement is given - with pytest.raises(Empty): + try: wait_for_debug_event(kernel_with_debug, "stopped", timeout=3) + except Empty: + pass + else: + # we're stopped somewhere. Fail with trace + reply = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": 1}) + entries = [] + for f in reversed(reply["body"]["stackFrames"]): + source = f.get("source", {}).get("path") or "" + loc = f'{source} ({f.get("line")},{f.get("column")})' + entries.append(f'{loc}: {f.get("name")}') + raise AssertionError('Unexpectedly stopped. Debugger stack:\n {0}'.format("\n ".join(entries))) # execute some new code without breakpoints, assert it stops code = 'print("bar")\nprint("alice")\n' From 654bb5da6062b206dee9fed81b22ab5053d1eace Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Mon, 7 Mar 2022 19:08:34 +0000 Subject: [PATCH 07/13] Make sure we get the thread rigth Probably redundant, but more robust --- ipykernel/tests/test_debugger.py | 43 +++++++++++++++++--------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 63fa49340..8a5c163c9 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -32,7 +32,7 @@ def wait_for_debug_request(kernel, command, arguments=None, full_reply=False): return reply if full_reply else reply["content"] -def wait_for_debug_event(kernel, event, timeout=TIMEOUT, verbose=False): +def wait_for_debug_event(kernel, event, timeout=TIMEOUT, verbose=False, full_reply=False): msg = {"msg_type": "", "content": {}} while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != event: msg = kernel.get_iopub_msg(timeout=timeout) @@ -40,7 +40,7 @@ def wait_for_debug_event(kernel, event, timeout=TIMEOUT, verbose=False): print(msg.get("msg_type")) if (msg.get("msg_type") == "debug_event"): print(f' {msg["content"].get("event")}') - return msg + return msg if full_reply else msg["content"] @pytest.fixture @@ -171,7 +171,7 @@ def test_stop_on_breakpoint(kernel_with_debug): # Wait for stop on breakpoint msg = wait_for_debug_event(kernel_with_debug, "stopped") - assert msg["content"]["body"]["reason"] == "breakpoint" + assert msg["body"]["reason"] == "breakpoint" @pytest.mark.skipif(sys.version_info >= (3, 10), reason="TODO Does not work on Python 3.10") @@ -204,7 +204,7 @@ def f(a, b): # Wait for stop on breakpoint msg = wait_for_debug_event(kernel_with_debug, "stopped") - assert msg["content"]["body"]["reason"] == "breakpoint" + assert msg["body"]["reason"] == "breakpoint" def test_rich_inspect_not_at_breakpoint(kernel_with_debug): @@ -250,20 +250,23 @@ def test_step_into_lib(kernel_with_debug): wait_for_debug_request(kernel_with_debug, "debugInfo") - r = wait_for_debug_request(kernel_with_debug, "configurationDone") + wait_for_debug_request(kernel_with_debug, "configurationDone") kernel_with_debug.execute(code) # Wait for stop on breakpoint - wait_for_debug_event(kernel_with_debug, "stopped") + r = wait_for_debug_event(kernel_with_debug, "stopped") + assert r["body"]["reason"] == "breakpoint" - # Setp over the import statement - wait_for_debug_request(kernel_with_debug, "next", {"threadId": 1}) - wait_for_debug_event(kernel_with_debug, "stopped") - # Attempt to step into the function call - wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": 1}) - wait_for_debug_event(kernel_with_debug, "stopped") + reply = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": r["body"].get("threadId", 1)}) - reply = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": 1}) + # Step over the import statement + wait_for_debug_request(kernel_with_debug, "next", {"threadId": r["body"].get("threadId", 1)}) + r = wait_for_debug_event(kernel_with_debug, "stopped") + assert r["body"]["reason"] == "step" + # Attempt to step into the function call + wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": r["body"].get("threadId", 1)}) + r = wait_for_debug_event(kernel_with_debug, "stopped") + assert r["body"]["reason"] == "step" names = [f.get("name") for f in reply["body"]["stackFrames"]] # "" will be the name of the cell @@ -288,23 +291,23 @@ def test_step_into_end(kernel_with_debug): wait_for_debug_request(kernel_with_debug, "debugInfo") - r = wait_for_debug_request(kernel_with_debug, "configurationDone") + wait_for_debug_request(kernel_with_debug, "configurationDone") kernel_with_debug.execute(code) # Wait for stop on breakpoint - wait_for_debug_event(kernel_with_debug, "stopped") + r = wait_for_debug_event(kernel_with_debug, "stopped") # Attempt to step into the print statement (will continue execution, but # should stop on first line of next execute request) - wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": 1}) + wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": r["body"].get("threadId", 1)}) # assert no stop statement is given try: - wait_for_debug_event(kernel_with_debug, "stopped", timeout=3) + r = wait_for_debug_event(kernel_with_debug, "stopped", timeout=3) except Empty: pass else: # we're stopped somewhere. Fail with trace - reply = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": 1}) + reply = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": r["body"].get("threadId", 1)}) entries = [] for f in reversed(reply["body"]["stackFrames"]): source = f.get("source", {}).get("path") or "" @@ -347,9 +350,9 @@ def test_rich_inspect_at_breakpoint(kernel_with_debug): kernel_with_debug.execute(code) # Wait for stop on breakpoint - wait_for_debug_event(kernel_with_debug, "stopped") + r = wait_for_debug_event(kernel_with_debug, "stopped") - stacks = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": 1})[ + stacks = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": r["body"].get("threadId", 1)})[ "body" ]["stackFrames"] From ddd4f15886c7f1129d8c82c2bf7e5b847fee29c2 Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Mon, 7 Mar 2022 19:11:56 +0000 Subject: [PATCH 08/13] Ensure we correctly find the ipykernel root frame When launching the kernel in-process, there might be frames before the ipykernel root, making the assumption that it is the first frame invalid. --- ipykernel/debugger.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/ipykernel/debugger.py b/ipykernel/debugger.py index dfe5d0a9b..b973e5459 100644 --- a/ipykernel/debugger.py +++ b/ipykernel/debugger.py @@ -459,18 +459,21 @@ async def stackTrace(self, message): # {'id': yyy, 'name': '', ... } <= this is the first frame of ipykernel code # or only the frames from the notebook. # We want to remove all the frames from ipykernel when they are present. - try: - sf_list = reply["body"]["stackFrames"] - module_idx = len(sf_list) - next( - i - for i, v in enumerate(reversed(sf_list), 1) - if v["name"] == "" and i != 1 - ) - reply["body"]["stackFrames"] = reply["body"]["stackFrames"][ - : module_idx + 1 - ] - except StopIteration: - pass + sf_list = reply["body"]["stackFrames"] + kernel_root_found = False + for i, v in enumerate(reversed(sf_list), 1): + if v["name"] == "": + if kernel_root_found: + module_idx = len(sf_list) - i + break + else: + kernel_root_found = True + else: + return reply + + reply["body"]["stackFrames"] = reply["body"]["stackFrames"][ + : module_idx + 1 + ] return reply def accept_variable(self, variable_name): @@ -567,7 +570,7 @@ async def debugInfo(self, message): async def inspectVariables(self, message): self.variable_explorer.untrack_all() # looks like the implementation of untrack_all in ptvsd - # destroys objects we nee din track. We have no choice but + # destroys objects we need in track. We have no choice but # reinstantiate the object self.variable_explorer = VariableExplorer() self.variable_explorer.track() From 240eea517e0db34135bfa2583a435802f4c4a83d Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Tue, 8 Mar 2022 13:37:16 +0000 Subject: [PATCH 09/13] Test that we can break in cell --- ipykernel/tests/test_debugger.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 8a5c163c9..15ec50807 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -159,7 +159,7 @@ def test_stop_on_breakpoint(kernel_with_debug): kernel_with_debug, "setBreakpoints", { - "breakpoints": [{"line": 2}], + "breakpoints": [{"line": 2}, {"line": 5}], "source": {"path": source}, "sourceModified": False, }, @@ -172,6 +172,28 @@ def test_stop_on_breakpoint(kernel_with_debug): # Wait for stop on breakpoint msg = wait_for_debug_event(kernel_with_debug, "stopped") assert msg["body"]["reason"] == "breakpoint" + stacks = wait_for_debug_request( + kernel_with_debug, + "stackTrace", + {"threadId": r["body"].get("threadId", 1)} + )["body"]["stackFrames"] + names = [f.get("name") for f in stacks] + assert stacks[0]["line"] == 5 + assert names == [""] + + wait_for_debug_request(kernel_with_debug, "continue", {"threadId": msg["body"].get("threadId", 1)}) + + # Wait for stop on breakpoint + msg = wait_for_debug_event(kernel_with_debug, "stopped") + assert msg["body"]["reason"] == "breakpoint" + stacks = wait_for_debug_request( + kernel_with_debug, + "stackTrace", + {"threadId": r["body"].get("threadId", 1)} + )["body"]["stackFrames"] + names = [f.get("name") for f in stacks] + assert names == ["f", ""] + assert stacks[0]["line"] == 2 @pytest.mark.skipif(sys.version_info >= (3, 10), reason="TODO Does not work on Python 3.10") From 715a625cab3163f14cf518c703fdeddea65eb31b Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske <510760+vidartf@users.noreply.github.com> Date: Thu, 10 Mar 2022 20:02:11 +0000 Subject: [PATCH 10/13] Initial draft proposal for filtering frames Monkey patches pydevd's global debugger instance so that frames from `__tracebackhide__ == "__ipython_bottom__"` and below are excluded from both stack traces and tracing (aka stepping). This should prevent users stepping out into the IPython/ipykernel internals between statements, and after cells. Points of note and/or discussion: - Doesn't yet handle display hooks I think. - Path/module filters are not sufficient, as the same file/frames can appear on both sides of the IPython stack "bottom". - For the same reason, caching of the filter can not be used. - Monkey patching is not ideal, especially since we also access some internals that are probably not considered part of the public API. - Are threads / forking important here? - Note that using `traitlets.validate()` for the test is useful, since it also shows up below the "bottom" of the IPython stack (at the time of writing). Maybe using a more central IPython function would be more reliable? --- ipykernel/debugger.py | 36 +++------------ ipykernel/filtered_pydb.py | 76 ++++++++++++++++++++++++++++++++ ipykernel/tests/test_debugger.py | 21 ++++++--- 3 files changed, 96 insertions(+), 37 deletions(-) create mode 100644 ipykernel/filtered_pydb.py diff --git a/ipykernel/debugger.py b/ipykernel/debugger.py index b973e5459..394ffdeac 100644 --- a/ipykernel/debugger.py +++ b/ipykernel/debugger.py @@ -1,3 +1,4 @@ +from pathlib import Path import sys import os import re @@ -375,8 +376,9 @@ def start(self): if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) host, port = self.debugpy_client.get_host_port() - code = 'import debugpy;' - code += 'debugpy.listen(("' + host + '",' + port + '))' + code = "import debugpy\n" + code += 'debugpy.listen(("' + host + '",' + port + '))\n' + code += (Path(__file__).parent / "filtered_pydb.py").read_text("utf8") content = { 'code': code, 'silent': True @@ -449,32 +451,7 @@ async def source(self, message): return reply async def stackTrace(self, message): - reply = await self._forward_message(message) - # The stackFrames array can have the following content: - # { frames from the notebook} - # ... - # { 'id': xxx, 'name': '', ... } <= this is the first frame of the code from the notebook - # { frames from ipykernel } - # ... - # {'id': yyy, 'name': '', ... } <= this is the first frame of ipykernel code - # or only the frames from the notebook. - # We want to remove all the frames from ipykernel when they are present. - sf_list = reply["body"]["stackFrames"] - kernel_root_found = False - for i, v in enumerate(reversed(sf_list), 1): - if v["name"] == "": - if kernel_root_found: - module_idx = len(sf_list) - i - break - else: - kernel_root_found = True - else: - return reply - - reply["body"]["stackFrames"] = reply["body"]["stackFrames"][ - : module_idx + 1 - ] - return reply + return await self._forward_message(message) def accept_variable(self, variable_name): forbid_list = [ @@ -527,8 +504,7 @@ async def attach(self, message): # The ipykernel source is in the call stack, so the user # has to manipulate the step-over and step-into in a wize way. # Set debugOptions for breakpoints in python standard library source. - if not self.just_my_code: - message['arguments']['debugOptions'] = [ 'DebugStdLib' ] + message['arguments']['options'] = f'DEBUG_STDLIB={not self.just_my_code}' return await self._forward_message(message) async def configurationDone(self, message): diff --git a/ipykernel/filtered_pydb.py b/ipykernel/filtered_pydb.py new file mode 100644 index 000000000..d29508d52 --- /dev/null +++ b/ipykernel/filtered_pydb.py @@ -0,0 +1,76 @@ +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import pydevd + +__db = pydevd.get_global_debugger() +if __db: + __original = __db.get_file_type + __initial = True + def get_file_type(self, frame, abs_real_path_and_basename=None, _cache_file_type=pydevd._CACHE_FILE_TYPE): + ''' + :param abs_real_path_and_basename: + The result from get_abs_path_real_path_and_base_from_file or + get_abs_path_real_path_and_base_from_frame. + + :return + _pydevd_bundle.pydevd_dont_trace_files.PYDEV_FILE: + If it's a file internal to the debugger which shouldn't be + traced nor shown to the user. + + _pydevd_bundle.pydevd_dont_trace_files.LIB_FILE: + If it's a file in a library which shouldn't be traced. + + None: + If it's a regular user file which should be traced. + ''' + global __initial + if __initial: + __initial = False + _cache_file_type.clear() + # Copied normalization: + if abs_real_path_and_basename is None: + try: + # Make fast path faster! + abs_real_path_and_basename = pydevd.NORM_PATHS_AND_BASE_CONTAINER[frame.f_code.co_filename] + except: + abs_real_path_and_basename = pydevd.get_abs_path_real_path_and_base_from_frame(frame) + + cache_key = (frame.f_code.co_firstlineno, abs_real_path_and_basename[0], frame.f_code) + try: + return _cache_file_type[cache_key] + except KeyError: + pass + + ret = __original(frame, abs_real_path_and_basename, _cache_file_type) + if ret is self.PYDEV_FILE: + return ret + if not hasattr(frame, "f_locals"): + return ret + + # if either user or lib, check with our logic + # (we check "user" code in case any of the libs we use are in edit install) + # logic outline: + # - check if current frame is IPython bottom frame (if so filter it) + # - if not, check all ancestor for ipython bottom. Filter if not present. + # - if debugging / developing, do some sanity check of ignored frames, and log any unexecpted frames + + # do not cache, these frames might show up on different sides of the bottom frame! + del _cache_file_type[cache_key] + if frame.f_locals.get("__tracebackhide__") == "__ipython_bottom__": + # Current frame is bottom frame, hide it! + pydevd.pydev_log.debug("Ignoring IPython bottom frame: %s - %s", frame.f_code.co_filename, frame.f_code.co_name) + ret = _cache_file_type[cache_key] = self.PYDEV_FILE + else: + f = frame + while f is not None: + if f.f_locals.get("__tracebackhide__") == "__ipython_bottom__": + # we found ipython bottom in stack, do not change type + return ret + f = f.f_back + pydevd.pydev_log.debug("Ignoring ipykernel frame: %s - %s", frame.f_code.co_filename, frame.f_code.co_name) + + ret = self.PYDEV_FILE + return ret + __db.get_file_type = get_file_type.__get__(__db, pydevd.PyDB) + __db.is_files_filter_enabled = True diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 15ec50807..4d43f339e 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -43,6 +43,13 @@ def wait_for_debug_event(kernel, event, timeout=TIMEOUT, verbose=False, full_rep return msg if full_reply else msg["content"] +def assert_stack_names(kernel, expected_names, thread_id=1): + reply = wait_for_debug_request(kernel, "stackTrace", {"threadId": thread_id}) + names = [f.get("name") for f in reply["body"]["stackFrames"]] + # "" will be the name of the cell + assert names == expected_names + + @pytest.fixture def kernel(request): if sys.platform == "win32": @@ -278,25 +285,25 @@ def test_step_into_lib(kernel_with_debug): # Wait for stop on breakpoint r = wait_for_debug_event(kernel_with_debug, "stopped") assert r["body"]["reason"] == "breakpoint" - - reply = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": r["body"].get("threadId", 1)}) + assert_stack_names(kernel_with_debug, [""], r["body"].get("threadId", 1)) # Step over the import statement wait_for_debug_request(kernel_with_debug, "next", {"threadId": r["body"].get("threadId", 1)}) r = wait_for_debug_event(kernel_with_debug, "stopped") assert r["body"]["reason"] == "step" + assert_stack_names(kernel_with_debug, [""], r["body"].get("threadId", 1)) + # Attempt to step into the function call wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": r["body"].get("threadId", 1)}) r = wait_for_debug_event(kernel_with_debug, "stopped") assert r["body"]["reason"] == "step" - - names = [f.get("name") for f in reply["body"]["stackFrames"]] - # "" will be the name of the cell - assert names == ["validate", ""] + assert_stack_names(kernel_with_debug, ["validate", ""], r["body"].get("threadId", 1)) +# Test with both lib code and only "my code" +@pytest.mark.parametrize("kernel", [[], ["--Kernel.debug_just_my_code=False"]], indirect=True) def test_step_into_end(kernel_with_debug): - code = '5 + 5;\n' + code = 'a = 5 + 5\n' r = wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code}) source = r["body"]["sourcePath"] From 15b1ac1917d312d1c08a655fbb5a320567f99c03 Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske <510760+vidartf@users.noreply.github.com> Date: Thu, 10 Mar 2022 20:05:03 +0000 Subject: [PATCH 11/13] Some nice to haves for testing Some of these might be nice to keep around, especially during the review phase. We can remove (parts of) this before merging if we want to keep things tidy! --- ipykernel/debugger.py | 12 ++++++++++-- ipykernel/kernelbase.py | 2 +- ipykernel/tests/test_debugger.py | 4 +++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ipykernel/debugger.py b/ipykernel/debugger.py index 394ffdeac..daf0eca3c 100644 --- a/ipykernel/debugger.py +++ b/ipykernel/debugger.py @@ -2,7 +2,6 @@ import sys import os import re -import threading import zmq from zmq.utils import jsonapi @@ -350,7 +349,7 @@ def _accept_stopped_thread(self, thread_name): 'Thread-4' ] return thread_name not in forbid_list - + async def handle_stopped_event(self): # Wait for a stopped event message in the stopped queue # This message is used for triggering the 'threads' request @@ -377,8 +376,13 @@ def start(self): os.makedirs(tmp_dir) host, port = self.debugpy_client.get_host_port() code = "import debugpy\n" + # Write debugpy logs? + #code += f'import debugpy; debugpy.log_to({str(Path(__file__).parent)!r});' code += 'debugpy.listen(("' + host + '",' + port + '))\n' code += (Path(__file__).parent / "filtered_pydb.py").read_text("utf8") + # Write pydevd logs? + # code += f'\npydevd.DebugInfoHolder.PYDEVD_DEBUG_FILE = {str(Path(__file__).parent / "debugpy.pydev.log")!r}\n' + # code += "pydevd.DebugInfoHolder.DEBUG_TRACE_LEVEL = 2\n" content = { 'code': code, 'silent': True @@ -505,6 +509,10 @@ async def attach(self, message): # has to manipulate the step-over and step-into in a wize way. # Set debugOptions for breakpoints in python standard library source. message['arguments']['options'] = f'DEBUG_STDLIB={not self.just_my_code}' + # Explicitly ignore IPython implicit hooks ? + message['arguments']['rules'] = [ + # { "module": "IPython.core.displayhook", "include": False }, + ] return await self._forward_message(message) async def configurationDone(self, message): diff --git a/ipykernel/kernelbase.py b/ipykernel/kernelbase.py index c775398fc..270df5774 100644 --- a/ipykernel/kernelbase.py +++ b/ipykernel/kernelbase.py @@ -133,7 +133,7 @@ def _default_ident(self): # Experimental option to break in non-user code. # The ipykernel source is in the call stack, so the user # has to manipulate the step-over and step-into in a wize way. - debug_just_my_code = Bool(True, + debug_just_my_code = Bool(os.environ.get("IPYKERNEL_DEBUG_JUST_MY_CODE", "True").lower() == "true", help="""Set to False if you want to debug python standard and dependent libraries. """ ).tag(config=True) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 4d43f339e..4a2f79b84 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -55,7 +55,9 @@ def kernel(request): if sys.platform == "win32": import asyncio asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - with new_kernel(getattr(request, "param", None)) as kc: + argv = getattr(request, "param", []) + #argv.append("--log-level=DEBUG") + with new_kernel(argv) as kc: yield kc From 01cb840c61d983a5481aff994e0ff3753c4d0047 Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske <510760+vidartf@users.noreply.github.com> Date: Thu, 10 Mar 2022 20:05:13 +0000 Subject: [PATCH 12/13] This seems to work now? --- ipykernel/tests/test_debugger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 4a2f79b84..a2ceac87f 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -205,7 +205,6 @@ def test_stop_on_breakpoint(kernel_with_debug): assert stacks[0]["line"] == 2 -@pytest.mark.skipif(sys.version_info >= (3, 10), reason="TODO Does not work on Python 3.10") def test_breakpoint_in_cell_with_leading_empty_lines(kernel_with_debug): code = """ def f(a, b): From 34dbfac58fd45785d2b3418056a16409b7de5aac Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske <510760+vidartf@users.noreply.github.com> Date: Thu, 10 Mar 2022 20:30:16 +0000 Subject: [PATCH 13/13] Update tests to work with latest IPython --- ipykernel/tests/test_debugger.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index a2ceac87f..8751f5eb3 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -46,7 +46,7 @@ def wait_for_debug_event(kernel, event, timeout=TIMEOUT, verbose=False, full_rep def assert_stack_names(kernel, expected_names, thread_id=1): reply = wait_for_debug_request(kernel, "stackTrace", {"threadId": thread_id}) names = [f.get("name") for f in reply["body"]["stackFrames"]] - # "" will be the name of the cell + # "" will be the name of the cell assert names == expected_names @@ -181,14 +181,7 @@ def test_stop_on_breakpoint(kernel_with_debug): # Wait for stop on breakpoint msg = wait_for_debug_event(kernel_with_debug, "stopped") assert msg["body"]["reason"] == "breakpoint" - stacks = wait_for_debug_request( - kernel_with_debug, - "stackTrace", - {"threadId": r["body"].get("threadId", 1)} - )["body"]["stackFrames"] - names = [f.get("name") for f in stacks] - assert stacks[0]["line"] == 5 - assert names == [""] + assert_stack_names(kernel_with_debug, [""], r["body"].get("threadId", 1)) wait_for_debug_request(kernel_with_debug, "continue", {"threadId": msg["body"].get("threadId", 1)}) @@ -201,8 +194,8 @@ def test_stop_on_breakpoint(kernel_with_debug): {"threadId": r["body"].get("threadId", 1)} )["body"]["stackFrames"] names = [f.get("name") for f in stacks] - assert names == ["f", ""] assert stacks[0]["line"] == 2 + assert names == ["f", ""] def test_breakpoint_in_cell_with_leading_empty_lines(kernel_with_debug): @@ -286,25 +279,25 @@ def test_step_into_lib(kernel_with_debug): # Wait for stop on breakpoint r = wait_for_debug_event(kernel_with_debug, "stopped") assert r["body"]["reason"] == "breakpoint" - assert_stack_names(kernel_with_debug, [""], r["body"].get("threadId", 1)) + assert_stack_names(kernel_with_debug, [""], r["body"].get("threadId", 1)) # Step over the import statement wait_for_debug_request(kernel_with_debug, "next", {"threadId": r["body"].get("threadId", 1)}) r = wait_for_debug_event(kernel_with_debug, "stopped") assert r["body"]["reason"] == "step" - assert_stack_names(kernel_with_debug, [""], r["body"].get("threadId", 1)) + assert_stack_names(kernel_with_debug, [""], r["body"].get("threadId", 1)) # Attempt to step into the function call wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": r["body"].get("threadId", 1)}) r = wait_for_debug_event(kernel_with_debug, "stopped") assert r["body"]["reason"] == "step" - assert_stack_names(kernel_with_debug, ["validate", ""], r["body"].get("threadId", 1)) + assert_stack_names(kernel_with_debug, ["validate", ""], r["body"].get("threadId", 1)) # Test with both lib code and only "my code" @pytest.mark.parametrize("kernel", [[], ["--Kernel.debug_just_my_code=False"]], indirect=True) def test_step_into_end(kernel_with_debug): - code = 'a = 5 + 5\n' + code = 'a = 5 + 5' r = wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code}) source = r["body"]["sourcePath"] @@ -327,7 +320,7 @@ def test_step_into_end(kernel_with_debug): # Wait for stop on breakpoint r = wait_for_debug_event(kernel_with_debug, "stopped") - # Attempt to step into the print statement (will continue execution, but + # Attempt to step into the statement (will continue execution, but # should stop on first line of next execute request) wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": r["body"].get("threadId", 1)}) # assert no stop statement is given