Tests for debugging imports

ipykernel/debugger.py

@@ -1,7 +1,7 @@
+from pathlib import Path
 import sys
 import os
 import re
-import threading
 
 import zmq
 from zmq.utils import jsonapi
@@ -349,7 +349,7 @@ def _accept_stopped_thread(self, thread_name):
             'Thread-4'
         ]
         return thread_name not in forbid_list
-        
+
     async def handle_stopped_event(self):
         # Wait for a stopped event message in the stopped queue
         # This message is used for triggering the 'threads' request
@@ -375,8 +375,14 @@ def start(self):
             if not os.path.exists(tmp_dir):
                 os.makedirs(tmp_dir)
             host, port = self.debugpy_client.get_host_port()
-            code = 'import debugpy;'
-            code += 'debugpy.listen(("' + host + '",' + port + '))'
+            code = "import debugpy\n"
+            # Write debugpy logs?
+            #code += f'import debugpy; debugpy.log_to({str(Path(__file__).parent)!r});'
+            code += 'debugpy.listen(("' + host + '",' + port + '))\n'
+            code += (Path(__file__).parent / "filtered_pydb.py").read_text("utf8")
+            # Write pydevd logs?
+            # code += f'\npydevd.DebugInfoHolder.PYDEVD_DEBUG_FILE = {str(Path(__file__).parent / "debugpy.pydev.log")!r}\n'
+            # code += "pydevd.DebugInfoHolder.DEBUG_TRACE_LEVEL = 2\n"
             content = {
                 'code': code,
                 'silent': True
@@ -449,29 +455,7 @@ async def source(self, message):
         return reply
 
     async def stackTrace(self, message):
-        reply = await self._forward_message(message)
-        # The stackFrames array can have the following content:
-        # { frames from the notebook}
-        # ...
-        # { 'id': xxx, 'name': '<module>', ... } <= this is the first frame of the code from the notebook
-        # { frames from ipykernel }
-        # ...
-        # {'id': yyy, 'name': '<module>', ... } <= this is the first frame of ipykernel code
-        # or only the frames from the notebook.
-        # We want to remove all the frames from ipykernel when they are present.
-        try:
-            sf_list = reply["body"]["stackFrames"]
-            module_idx = len(sf_list) - next(
-                i
-                for i, v in enumerate(reversed(sf_list), 1)
-                if v["name"] == "<module>" and i != 1
-            )
-            reply["body"]["stackFrames"] = reply["body"]["stackFrames"][
-                : module_idx + 1
-            ]
-        except StopIteration:
-            pass
-        return reply
+        return await self._forward_message(message)
 
     def accept_variable(self, variable_name):
         forbid_list = [
@@ -524,8 +508,11 @@ async def attach(self, message):
         # The ipykernel source is in the call stack, so the user
         # has to manipulate the step-over and step-into in a wize way.
         # Set debugOptions for breakpoints in python standard library source.
-        if not self.just_my_code:
-            message['arguments']['debugOptions'] = [ 'DebugStdLib' ]
+        message['arguments']['options'] = f'DEBUG_STDLIB={not self.just_my_code}'
+        # Explicitly ignore IPython implicit hooks ?
+        message['arguments']['rules'] = [
+            # { "module": "IPython.core.displayhook", "include": False },
+        ]
         return await self._forward_message(message)
 
     async def configurationDone(self, message):
@@ -567,7 +554,7 @@ async def debugInfo(self, message):
     async def inspectVariables(self, message):
         self.variable_explorer.untrack_all()
         # looks like the implementation of untrack_all in ptvsd
-        # destroys objects we nee din track. We have no choice but
+        # destroys objects we need in track. We have no choice but
         # reinstantiate the object
         self.variable_explorer = VariableExplorer()
         self.variable_explorer.track()

ipykernel/filtered_pydb.py

@@ -0,0 +1,76 @@
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import pydevd
+
+__db = pydevd.get_global_debugger()
+if __db:
+    __original = __db.get_file_type
+    __initial = True
+    def get_file_type(self, frame, abs_real_path_and_basename=None, _cache_file_type=pydevd._CACHE_FILE_TYPE):
+        '''
+        :param abs_real_path_and_basename:
+            The result from get_abs_path_real_path_and_base_from_file or
+            get_abs_path_real_path_and_base_from_frame.
+
+        :return
+            _pydevd_bundle.pydevd_dont_trace_files.PYDEV_FILE:
+                If it's a file internal to the debugger which shouldn't be
+                traced nor shown to the user.
+
+            _pydevd_bundle.pydevd_dont_trace_files.LIB_FILE:
+                If it's a file in a library which shouldn't be traced.
+
+            None:
+                If it's a regular user file which should be traced.
+        '''
+        global __initial
+        if __initial:
+            __initial = False
+            _cache_file_type.clear()
+        # Copied normalization:
+        if abs_real_path_and_basename is None:
+            try:
+                # Make fast path faster!
+                abs_real_path_and_basename = pydevd.NORM_PATHS_AND_BASE_CONTAINER[frame.f_code.co_filename]
+            except:
+                abs_real_path_and_basename = pydevd.get_abs_path_real_path_and_base_from_frame(frame)
+
+        cache_key = (frame.f_code.co_firstlineno, abs_real_path_and_basename[0], frame.f_code)
+        try:
+            return _cache_file_type[cache_key]
+        except KeyError:
+            pass
+
+        ret = __original(frame, abs_real_path_and_basename, _cache_file_type)
+        if ret is self.PYDEV_FILE:
+            return ret
+        if not hasattr(frame, "f_locals"):
+            return ret
+
+        # if either user or lib, check with our logic
+        # (we check "user" code in case any of the libs we use are in edit install)
+        # logic outline:
+        # - check if current frame is IPython bottom frame (if so filter it)
+        # - if not, check all ancestor for ipython bottom. Filter if not present.
+        # - if debugging / developing, do some sanity check of ignored frames, and log any unexecpted frames
+
+        # do not cache, these frames might show up on different sides of the bottom frame!
+        del _cache_file_type[cache_key]
+        if frame.f_locals.get("__tracebackhide__") == "__ipython_bottom__":
+            # Current frame is bottom frame, hide it!
+            pydevd.pydev_log.debug("Ignoring IPython bottom frame: %s - %s", frame.f_code.co_filename, frame.f_code.co_name)
+            ret = _cache_file_type[cache_key] = self.PYDEV_FILE
+        else:
+            f = frame
+            while f is not None:
+                if f.f_locals.get("__tracebackhide__") == "__ipython_bottom__":
+                    # we found ipython bottom in stack, do not change type
+                    return ret
+                f = f.f_back
+            pydevd.pydev_log.debug("Ignoring ipykernel frame: %s - %s", frame.f_code.co_filename, frame.f_code.co_name)
+
+            ret = self.PYDEV_FILE
+        return ret
+    __db.get_file_type = get_file_type.__get__(__db, pydevd.PyDB)
+    __db.is_files_filter_enabled = True

ipykernel/kernelbase.py

@@ -133,7 +133,7 @@ def _default_ident(self):
     # Experimental option to break in non-user code.
     # The ipykernel source is in the call stack, so the user
     # has to manipulate the step-over and step-into in a wize way.
-    debug_just_my_code = Bool(True,
+    debug_just_my_code = Bool(os.environ.get("IPYKERNEL_DEBUG_JUST_MY_CODE", "True").lower() == "true",
         help="""Set to False if you want to debug python standard and dependent libraries.
         """
     ).tag(config=True)

ipykernel/tests/test_debugger.py

@@ -1,3 +1,4 @@
+from queue import Empty
 import sys
 import pytest
 
@@ -31,9 +32,32 @@ def wait_for_debug_request(kernel, command, arguments=None, full_reply=False):
     return reply if full_reply else reply["content"]
 
 
+def wait_for_debug_event(kernel, event, timeout=TIMEOUT, verbose=False, full_reply=False):
+    msg = {"msg_type": "", "content": {}}
+    while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != event:
+        msg = kernel.get_iopub_msg(timeout=timeout)
+        if verbose:
+            print(msg.get("msg_type"))
+            if (msg.get("msg_type") == "debug_event"):
+                print(f'  {msg["content"].get("event")}')
+    return msg if full_reply else msg["content"]
+
+
+def assert_stack_names(kernel, expected_names, thread_id=1):
+    reply = wait_for_debug_request(kernel, "stackTrace", {"threadId": thread_id})
+    names = [f.get("name") for f in reply["body"]["stackFrames"]]
+    # "<cell line: 1>" will be the name of the cell
+    assert names == expected_names
+
+
 @pytest.fixture
-def kernel():
-    with new_kernel() as kc:
+def kernel(request):
+    if sys.platform == "win32":
+        import asyncio
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+    argv = getattr(request, "param", [])
+    #argv.append("--log-level=DEBUG")
+    with new_kernel(argv) as kc:
         yield kc
 
 
@@ -144,7 +168,7 @@ def test_stop_on_breakpoint(kernel_with_debug):
         kernel_with_debug,
         "setBreakpoints",
         {
-            "breakpoints": [{"line": 2}],
+            "breakpoints": [{"line": 2}, {"line": 5}],
             "source": {"path": source},
             "sourceModified": False,
         },
@@ -153,16 +177,27 @@ def test_stop_on_breakpoint(kernel_with_debug):
     wait_for_debug_request(kernel_with_debug, "configurationDone", full_reply=True)
 
     kernel_with_debug.execute(code)
-    
+
     # Wait for stop on breakpoint
-    msg = {"msg_type": "", "content": {}}
-    while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != "stopped":
-        msg = kernel_with_debug.get_iopub_msg(timeout=TIMEOUT)
+    msg = wait_for_debug_event(kernel_with_debug, "stopped")
+    assert msg["body"]["reason"] == "breakpoint"
+    assert_stack_names(kernel_with_debug, ["<cell line: 5>"], r["body"].get("threadId", 1))
 
-    assert msg["content"]["body"]["reason"] == "breakpoint"
+    wait_for_debug_request(kernel_with_debug, "continue", {"threadId": msg["body"].get("threadId", 1)})
+
+    # Wait for stop on breakpoint
+    msg = wait_for_debug_event(kernel_with_debug, "stopped")
+    assert msg["body"]["reason"] == "breakpoint"
+    stacks = wait_for_debug_request(
+        kernel_with_debug,
+        "stackTrace",
+        {"threadId": r["body"].get("threadId", 1)}
+    )["body"]["stackFrames"]
+    names = [f.get("name") for f in stacks]
+    assert stacks[0]["line"] == 2
+    assert names == ["f", "<cell line: 5>"]
 
 
-@pytest.mark.skipif(sys.version_info >= (3, 10), reason="TODO Does not work on Python 3.10")
 def test_breakpoint_in_cell_with_leading_empty_lines(kernel_with_debug):
     code = """
 def f(a, b):
@@ -189,13 +224,10 @@ def f(a, b):
     wait_for_debug_request(kernel_with_debug, "configurationDone", full_reply=True)
 
     kernel_with_debug.execute(code)
-
-    # Wait for stop on breakpoint
-    msg = {"msg_type": "", "content": {}}
-    while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != "stopped":
-        msg = kernel_with_debug.get_iopub_msg(timeout=TIMEOUT)
 
-    assert msg["content"]["body"]["reason"] == "breakpoint"
+    # Wait for stop on breakpoint
+    msg = wait_for_debug_event(kernel_with_debug, "stopped")
+    assert msg["body"]["reason"] == "breakpoint"
 
 
 def test_rich_inspect_not_at_breakpoint(kernel_with_debug):
@@ -220,6 +252,99 @@ def test_rich_inspect_not_at_breakpoint(kernel_with_debug):
     assert reply["body"]["data"] == {"text/plain": f"'{value}'"}
 
 
+@pytest.mark.parametrize("kernel", [["--Kernel.debug_just_my_code=False"]], indirect=True)
+def test_step_into_lib(kernel_with_debug):
+    code = """import traitlets
+traitlets.validate('foo', 'bar')
+"""
+
+    r = wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code})
+    source = r["body"]["sourcePath"]
+
+    wait_for_debug_request(
+        kernel_with_debug,
+        "setBreakpoints",
+        {
+            "breakpoints": [{"line": 1}],
+            "source": {"path": source},
+            "sourceModified": False,
+        },
+    )
+
+    wait_for_debug_request(kernel_with_debug, "debugInfo")
+
+    wait_for_debug_request(kernel_with_debug, "configurationDone")
+    kernel_with_debug.execute(code)
+
+    # Wait for stop on breakpoint
+    r = wait_for_debug_event(kernel_with_debug, "stopped")
+    assert r["body"]["reason"] == "breakpoint"
+    assert_stack_names(kernel_with_debug, ["<cell line: 1>"], r["body"].get("threadId", 1))
+
+    # Step over the import statement
+    wait_for_debug_request(kernel_with_debug, "next", {"threadId": r["body"].get("threadId", 1)})
+    r = wait_for_debug_event(kernel_with_debug, "stopped")
+    assert r["body"]["reason"] == "step"
+    assert_stack_names(kernel_with_debug, ["<cell line: 2>"], r["body"].get("threadId", 1))
+
+    # Attempt to step into the function call
+    wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": r["body"].get("threadId", 1)})
+    r = wait_for_debug_event(kernel_with_debug, "stopped")
+    assert r["body"]["reason"] == "step"
+    assert_stack_names(kernel_with_debug, ["validate", "<cell line: 2>"], r["body"].get("threadId", 1))
+
+
+# Test with both lib code and only "my code"
+@pytest.mark.parametrize("kernel", [[], ["--Kernel.debug_just_my_code=False"]], indirect=True)
+def test_step_into_end(kernel_with_debug):
+    code = 'a = 5 + 5'
+
+    r = wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code})
+    source = r["body"]["sourcePath"]
+
+    wait_for_debug_request(
+        kernel_with_debug,
+        "setBreakpoints",
+        {
+            "breakpoints": [{"line": 1}],
+            "source": {"path": source},
+            "sourceModified": False,
+        },
+    )
+
+    wait_for_debug_request(kernel_with_debug, "debugInfo")
+
+    wait_for_debug_request(kernel_with_debug, "configurationDone")
+    kernel_with_debug.execute(code)
+
+    # Wait for stop on breakpoint
+    r = wait_for_debug_event(kernel_with_debug, "stopped")
+
+    # Attempt to step into the statement (will continue execution, but
+    # should stop on first line of next execute request)
+    wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": r["body"].get("threadId", 1)})
+    # assert no stop statement is given
+    try:
+        r = wait_for_debug_event(kernel_with_debug, "stopped", timeout=3)
+    except Empty:
+        pass
+    else:
+        # we're stopped somewhere. Fail with trace
+        reply = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": r["body"].get("threadId", 1)})
+        entries = []
+        for f in reversed(reply["body"]["stackFrames"]):
+            source = f.get("source", {}).get("path") or "<unknown>"
+            loc = f'{source} ({f.get("line")},{f.get("column")})'
+            entries.append(f'{loc}:  {f.get("name")}')
+        raise AssertionError('Unexpectedly stopped. Debugger stack:\n    {0}'.format("\n    ".join(entries)))
+
+    # execute some new code without breakpoints, assert it stops
+    code = 'print("bar")\nprint("alice")\n'
+    wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code})
+    kernel_with_debug.execute(code)
+    wait_for_debug_event(kernel_with_debug, "stopped")
+
+
 def test_rich_inspect_at_breakpoint(kernel_with_debug):
     code = """def f(a, b):
     c = a + b
@@ -248,11 +373,9 @@ def test_rich_inspect_at_breakpoint(kernel_with_debug):
     kernel_with_debug.execute(code)
 
     # Wait for stop on breakpoint
-    msg = {"msg_type": "", "content": {}}
-    while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != "stopped":
-        msg = kernel_with_debug.get_iopub_msg(timeout=TIMEOUT)
+    r = wait_for_debug_event(kernel_with_debug, "stopped")
 
-    stacks = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": 1})[
+    stacks = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": r["body"].get("threadId", 1)})[
         "body"
     ]["stackFrames"]