Merge pull request #126 from JuliaDebug/teh/debugging

Implement core debug API here instead of Debugger.jl

Author: timholy

docs/src/dev_reference.md

@@ -43,7 +43,12 @@ JuliaInterpreter.evaluate_call!
 JuliaInterpreter.evaluate_foreigncall
 JuliaInterpreter.maybe_evaluate_builtin
 JuliaInterpreter.maybe_next_call!
+JuliaInterpreter.next_line!
+JuliaInterpreter.next_call!
+JuliaInterpreter.maybe_reset_frame!
+JuliaInterpreter.maybe_step_through_wrapper!
 JuliaInterpreter.handle_err
+JuliaInterpreter.debug_command
 ```
 
 ## Breakpoints

src/JuliaInterpreter.jl

@@ -13,7 +13,8 @@ using InteractiveUtils
 using CodeTracking
 
 export @interpret, Compiled, Frame, root, leaf,
-       BreakpointRef, breakpoint, @breakpoint, breakpoints, enable, disable, remove
+       BreakpointRef, breakpoint, @breakpoint, breakpoints, enable, disable, remove,
+       debug_command
 
 module CompiledCalls
 # This module is for handling intrinsics that must be compiled (llvmcall)
@@ -48,6 +49,8 @@ function set_compiled_methods()
     push!(compiled_methods, which(flush, Tuple{IOStream}))
     push!(compiled_methods, which(disable_sigint, Tuple{Function}))
     push!(compiled_methods, which(reenable_sigint, Tuple{Function}))
+    # Signal-handling in the `print` dispatch hierarchy
+    push!(compiled_methods, which(Base.unsafe_write, Tuple{Base.LibuvStream, Ptr{UInt8}, UInt}))
 end
 
 function __init__()

src/breakpoints.jl

@@ -217,7 +217,7 @@ function breakpoint(filename::AbstractString, line::Integer, args...)
         offset = line1 - method.line
         src = JuliaInterpreter.get_source(method)
         lastline = src.linetable[end]
-        if lastline.line + offset >= line
+        if getline(lastline) + offset >= line
             return breakpoint(method, line, args...)
         end
     end

src/commands.jl

@@ -1,8 +1,8 @@
 """
-    ret = finish!(recurse, frame, istoplevel=false)
-    ret = finish!(frame, istoplevel=false)
+    pc = finish!(recurse, frame, istoplevel=false)
+    pc = finish!(frame, istoplevel=false)
 
-Run `frame` until execution terminates. `ret` is either `nothing` (if execution terminates
+Run `frame` until execution terminates. `pc` is either `nothing` (if execution terminates
 when it hits a `return` statement) or a reference to a breakpoint.
 In the latter case, `leaf(frame)` returns the frame in which it hit the breakpoint.
 
@@ -22,8 +22,8 @@ finish!(frame::Frame, istoplevel::Bool=false) = finish!(finish_and_return!, fram
     ret = finish_and_return!(recurse, frame, istoplevel::Bool=false)
     ret = finish_and_return!(frame, istoplevel::Bool=false)
 
-Call [`JuliaInterpreter.finish!`](@ref) and pass back the return value. If execution
-pauses at a breakpoint, the reference to the breakpoint is returned.
+Call [`JuliaInterpreter.finish!`](@ref) and pass back the return value `ret`. If execution
+pauses at a breakpoint, `ret` is the reference to the breakpoint.
 """
 function finish_and_return!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false)
     pc = finish!(recurse, frame, istoplevel)
@@ -33,26 +33,29 @@ end
 finish_and_return!(frame::Frame, istoplevel::Bool=false) = finish_and_return!(finish_and_return!, frame, istoplevel)
 
 """
-    bpref = dummy_breakpoint(recurse, frame::Frame)
+    bpref = dummy_breakpoint(recurse, frame::Frame, istoplevel)
 
-Return a fake breakpoint. This can be useful as the `recurse` argument to `evaluate_call!`
-(or any of the higher-order commands) to ensure that you return immediately after stepping
-into a call.
+Return a fake breakpoint. `dummy_breakpoint` can be useful as the `recurse` argument to
+`evaluate_call!` (or any of the higher-order commands) to ensure that you return immediately
+after stepping into a call.
 """
-dummy_breakpoint(@nospecialize(recurse), frame::Frame) = BreakpointRef(frame.framecode, 0)
+dummy_breakpoint(@nospecialize(recurse), frame::Frame, istoplevel) = BreakpointRef(frame.framecode, 0)
 
 """
-    ret = finish_stack!(recurse, frame, istoplevel=false)
-    ret = finish_stack!(frame, istoplevel=false)
+    ret = finish_stack!(recurse, frame, rootistoplevel=false)
+    ret = finish_stack!(frame, rootistoplevel=false)
 
 Unwind the callees of `frame`, finishing each before returning to the caller.
-`frame` itself is also finished
-If execution hits a breakpoint, `ret` will be a reference to the breakpoint.
+`frame` itself is also finished. `rootistoplevel` should be true if the root frame is top-level.
+
+`ret` is typically the returned value. If execution hits a breakpoint, `ret` will be a
+reference to the breakpoint.
 """
-function finish_stack!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false)
+function finish_stack!(@nospecialize(recurse), frame::Frame, rootistoplevel::Bool=false)
     frame0 = frame
     frame = leaf(frame)
     while true
+        istoplevel = rootistoplevel && frame.caller === nothing
         ret = finish_and_return!(recurse, frame, istoplevel)
         isa(ret, BreakpointRef) && return ret
         frame === frame0 && return ret
@@ -93,14 +96,22 @@ end
 next_until!(predicate, frame::Frame, istoplevel::Bool=false) =
     next_until!(predicate, finish_and_return!, frame, istoplevel)
 
+"""
+    pc = next_call!(recurse, frame, istoplevel=false)
+    pc = next_call!(frame, istoplevel=false)
+
+Execute the current statement. Continue stepping through `frame` until the next
+`:return` or `:call` expression.
+"""
 next_call!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false) =
     next_until!(is_call_or_return, recurse, frame, istoplevel)
 next_call!(frame::Frame, istoplevel::Bool=false) = next_call!(finish_and_return!, frame, istoplevel)
 
 """
-    maybe_next_call!(predicate, frame, istoplevel=false)
+    pc = maybe_next_call!(recurse, frame, istoplevel=false)
+    pc = maybe_next_call!(frame, istoplevel=false)
 
-Return the current statement of `frame` if it is a `:return` or `:call` expression.
+Return the current program counter of `frame` if it is a `:return` or `:call` expression.
 Otherwise, step through the statements of `frame` until the next `:return` or `:call` expression.
 """
 function maybe_next_call!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false)
@@ -112,8 +123,8 @@ maybe_next_call!(frame::Frame, istoplevel::Bool=false) =
     maybe_next_call!(finish_and_return!, frame, istoplevel)
 
 """
-    through_methoddef_or_done!(recurse, frame)
-    through_methoddef_or_done!(frame)
+    pc = through_methoddef_or_done!(recurse, frame)
+    pc = through_methoddef_or_done!(frame)
 
 Runs `frame` at top level until it either finishes (e.g., hits a `return` statement)
 or defines a new method.
@@ -144,11 +155,20 @@ function changed_line!(expr, line, fls)
     end
 end
 
+"""
+    pc = next_line!(recurse, frame, istoplevel=false)
+    pc = next_line!(frame, istoplevel=false)
+
+Execute until reaching the first call of the next line of the source code.
+Upon return, `pc` is either the new program counter, `nothing` if a `return` is reached,
+or a `BreakpointRef` if it encountered a wrapper call. In the latter case, call `leaf(frame)`
+to obtain the new execution frame.
+"""
 function next_line!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false)
-    initial = linenumber(frame)
-    first = true
     pc = frame.pc
-    while linenumber(frame, pc) == initial
+    initialline, initialfile = linenumber(frame, pc), getfile(frame, pc)
+    first = true
+    while linenumber(frame, pc) == initialline && getfile(frame, pc) == initialfile
         # If this is a return node, interrupt execution
         expr = pc_expr(frame, pc)
         (!first && isexpr(expr, :return)) && return pc
@@ -177,13 +197,170 @@ function next_line!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false
                 expr = pc_expr(frame)
             end
             # Signal that we've switched frames
-            switched && return BreakpointRef(frame.framecode, frame.pc)
+            if switched
+                pc = next_line!(recurse, frame, false)
+                pc === nothing && error("confusing next_line!")
+                lframe = leaf(frame)
+                return isa(pc, BreakpointRef) ? pc : BreakpointRef(lframe.framecode, lframe.pc)
+            end
         else
             pc = step_expr!(recurse, frame, istoplevel)
             (pc === nothing || isa(pc, BreakpointRef)) && return pc
         end
         shouldbreak(frame, pc) && return BreakpointRef(frame.framecode, pc)
     end
-    maybe_next_call!(recurse, frame, pc)
+    maybe_next_call!(recurse, frame, istoplevel)
 end
 next_line!(frame::Frame, istoplevel::Bool=false) = next_line!(finish_and_return!, frame, istoplevel)
+
+"""
+    cframe = maybe_step_through_wrapper!(recurse, frame)
+    cframe = maybe_step_through_wrapper!(frame)
+
+Return the new frame of execution, potentially stepping through "wrapper" methods like those
+that supply default positional arguments or handle keywords. `cframe` is the leaf frame from
+which execution should start.
+"""
+function maybe_step_through_wrapper!(@nospecialize(recurse), frame::Frame)
+    code = frame.framecode
+    stmts, scope = code.src.code, code.scope::Method
+    length(stmts) < 2 && return frame
+    last = stmts[end-1]
+    isexpr(last, :(=)) && (last = last.args[2])
+    is_kw = isa(scope, Method) && startswith(String(Base.unwrap_unionall(scope.sig).parameters[1].name.name), "#kw")
+    if is_kw || isexpr(last, :call) && any(x->x==Core.SlotNumber(1), last.args)
+        # If the last expr calls #self# or passes it to an implementation method,
+        # this is a wrapper function that we might want to step through
+        while frame.pc != length(stmts)-1
+            pc = next_call!(recurse, frame, false)  # since we're in a Method we're not at toplevel
+        end
+        ret = evaluate_call!(dummy_breakpoint, frame, last)
+        @assert isa(ret, BreakpointRef)
+        return maybe_step_through_wrapper!(recurse, callee(frame))
+    end
+    return frame
+end
+maybe_step_through_wrapper!(frame::Frame) = maybe_step_through_wrapper!(finish_and_return!, frame)
+
+"""
+    ret = maybe_reset_frame!(recurse, frame, pc, rootistoplevel)
+
+Perform a return to the caller, or descend to the level of a breakpoint.
+`pc` is the return state from the previous command (e.g., `next_call!` or similar).
+`rootistoplevel` should be true if the root frame is top-level.
+
+`ret` will be `nothing` if we have just completed a top-level frame. Otherwise,
+
+    cframe, cpc = ret
+
+where `cframe` is the frame from which execution should continue and `cpc` is the state
+of `cframe` (the program counter, a `BreakpointRef`, or `nothing`).
+"""
+function maybe_reset_frame!(@nospecialize(recurse), frame::Frame, @nospecialize(pc), rootistoplevel::Bool)
+    isa(pc, BreakpointRef) && return leaf(frame), pc
+    if pc === nothing
+        val = get_return(frame)
+        recycle(frame)
+        frame = caller(frame)
+        frame === nothing && return nothing
+        frame.callee = nothing
+        maybe_assign!(frame, val)
+        frame.pc += 1
+        pc = maybe_next_call!(recurse, frame, rootistoplevel && frame.caller===nothing)
+        return maybe_reset_frame!(recurse, frame, pc, rootistoplevel)
+    end
+    return frame, pc
+end
+
+# Unwind the stack until an exc is eventually caught, thereby
+# returning the frame that caught the exception at the pc of the catch
+# or rethrow the error
+function unwind_exception(frame::Frame, exc)
+    while frame !== nothing
+        if !isempty(frame.framedata.exception_frames)
+            # Exception caught
+            frame.pc = frame.framedata.exception_frames[end]
+            frame.framedata.last_exception[] = exc
+            return frame
+        end
+        recycle(frame)
+        frame = caller(frame)
+        frame === nothing || (frame.callee = nothing)
+    end
+    rethrow(exc)
+end
+
+"""
+    ret = debug_command(recurse, frame, cmd, rootistoplevel=false)
+    ret = debug_command(frame, cmd, rootistoplevel=false)
+
+Perform one "debugger" command. `cmd` should be one of:
+
+- "n": advance to the next line
+- "s": step into the next call
+- "c": continue execution until termination or reaching a breakpoint
+- "finish": finish the current frame and return to the parent
+
+or one of the 'advanced' commands
+
+- "nc": step forward to the next call
+- "se": execute a single statement
+- "si": execute a single statement, stepping in if it's a call
+- "sg": step into the generator of a generated function
+
+`rootistoplevel` and `ret` are as described for [`JuliaInterpreter.maybe_reset_frame!`](@ref).
+"""
+function debug_command(@nospecialize(recurse), frame::Frame, cmd::AbstractString, rootistoplevel::Bool=false)
+    istoplevel = rootistoplevel && frame.caller === nothing
+    if cmd == "si"
+        stmt = pc_expr(frame)
+        cmd = is_call(stmt) ? "s" : "se"
+    end
+    try
+        cmd == "nc" && return maybe_reset_frame!(recurse, frame, next_call!(recurse, frame, istoplevel), rootistoplevel)
+        cmd == "n" && return maybe_reset_frame!(recurse, frame, next_line!(recurse, frame, istoplevel), rootistoplevel)
+        cmd == "se" && return maybe_reset_frame!(recurse, frame, step_expr!(recurse, frame, istoplevel), rootistoplevel)
+
+        enter_generated = false
+        if cmd == "sg"
+            enter_generated = true
+            cmd = "s"
+        end
+        if cmd == "s"
+            pc = maybe_next_call!(recurse, frame, istoplevel)
+            (isa(pc, BreakpointRef) || pc === nothing) && return maybe_reset_frame!(recurse, frame, pc, rootistoplevel)
+            stmt0 = stmt = pc_expr(frame, pc)
+            isexpr(stmt0, :return) && return maybe_reset_frame!(recurse, frame, nothing, rootistoplevel)
+            if isexpr(stmt, :(=))
+                stmt = stmt.args[2]
+            end
+            local ret
+            try
+                ret = evaluate_call!(dummy_breakpoint, frame, stmt; enter_generated=enter_generated)
+            catch err
+                ret = handle_err(recurse, frame, err)
+                return isa(ret, BreakpointRef) ? (leaf(frame), ret) : ret
+            end
+            isa(ret, BreakpointRef) && return maybe_reset_frame!(recurse, frame, ret, rootistoplevel)
+            maybe_assign!(frame, stmt0, ret)
+            frame.pc += 1
+            return frame, frame.pc
+        end
+        if cmd == "c"
+            r = root(frame)
+            ret = finish_stack!(recurse, r, rootistoplevel)
+            return isa(ret, BreakpointRef) ? (leaf(r), ret) : nothing
+        end
+        cmd == "finish" && return maybe_reset_frame!(recurse, frame, finish!(recurse, frame, istoplevel), rootistoplevel)
+    catch err
+        frame = unwind_exception(frame, err)
+        if cmd == "c"
+            return debug_command(recurse, frame, "c", istoplevel)
+        else
+            return debug_command(recurse, frame, "nc", istoplevel)
+        end
+    end
+    throw(ArgumentError("command $cmd not recognized"))
+end
+debug_command(frame::Frame, cmd::AbstractString, rootistoplevel::Bool=false) =
+    debug_command(finish_and_return!, frame, cmd, rootistoplevel)

src/construct.jl

@@ -20,8 +20,8 @@ rather than recursed into via the interpreter.
 """
 const compiled_methods = Set{Method}()
 
-const junk = FrameData[]      # to allow re-use of allocated memory (this is otherwise a bottleneck)
-recycle(frame) = push!(junk, frame.framedata)
+const junk = Base.IdSet{FrameData}()      # to allow re-use of allocated memory (this is otherwise a bottleneck)
+recycle(frame) = push!(junk, frame.framedata)  # using an IdSet ensures that a frame can't be added twice
 
 const empty_svec = Core.svec()
 
@@ -238,7 +238,8 @@ function prepare_framedata(framecode, argvals::Vector{Any})
         ng = isa(ssavt, Int) ? ssavt : length(ssavt::Vector{Any})
         nargs = length(argvals)
         if !isempty(junk)
-            olddata = pop!(junk)
+            olddata = first(junk)
+            delete!(junk, olddata)
             locals, ssavalues, sparams = olddata.locals, olddata.ssavalues, olddata.sparams
             exception_frames, last_reference = olddata.exception_frames, olddata.last_reference
             last_exception = olddata.last_exception