update to JuliaInterpreter 0.10

This commit implements the migration to the new JuliaInterpreter
interface proposed in JuliaDebug/JuliaInterpreter.jl#683.

It purely performs the migration to the new interface and does not
include any refactoring based on it.

In practice, `methods_by_execution!` is quite complex, and using the
`Interpreter` interface may not allow us to simplify its implementation.

That said, this commit seems to achieve a modest latency improvement by
changing the argument type declaration from `@nospecialize(recurse)` to
`interp::Interpreter` (with easier code specialization).

Author: aviatesk

Project.toml

@@ -23,8 +23,8 @@ DistributedExt = "Distributed"
 [compat]
 CodeTracking = "1.2"
 Distributed = "1"
-JuliaInterpreter = "0.9.46"
-LoweredCodeUtils = "3.2"
+JuliaInterpreter = "0.10"
+LoweredCodeUtils = "3.3"
 OrderedCollections = "1"
 # Exclude Requires-1.1.0 - see https://github.com/JuliaPackaging/Requires.jl/issues/94
 Requires = "~1.0, ^1.1.1"

src/Revise.jl

@@ -36,10 +36,10 @@ module Revise
 using OrderedCollections, CodeTracking, JuliaInterpreter, LoweredCodeUtils
 
 using CodeTracking: PkgFiles, basedir, srcfiles, basepath
-using JuliaInterpreter: codelocs, finish_and_return!, get_return, is_doc_expr,
-                        isassign, isidentical, is_quotenode_egal, linetable,
-                        LineTypes, lookup, moduleof, pc_expr, scopeof,
-                        step_expr!, whichtt
+using JuliaInterpreter: Compiled, Frame, Interpreter, LineTypes, RecursiveInterpreter
+using JuliaInterpreter: codelocs, finish_and_return!, get_return, is_doc_expr, isassign,
+                        isidentical, is_quotenode_egal, linetable, lookup, moduleof,
+                        pc_expr, scopeof, step_expr!
 using LoweredCodeUtils: next_or_nothing!, callee_matches
 
 include("packagedef.jl")

src/lowered.jl

@@ -13,15 +13,17 @@ function assign_this!(frame, value)
     frame.framedata.ssavalues[frame.pc] = value
 end
 
+# TODO Define abstract type MethodInfo end
+
 # This defines the API needed to store signatures using methods_by_execution!
 # This default version is simple and only used for testing purposes.
 # The "real" one is CodeTrackingMethodInfo in Revise.jl.
 const MethodInfo = IdDict{Type,LineNumberNode}
-add_signature!(methodinfo::MethodInfo, @nospecialize(sig), ln) = push!(methodinfo, sig=>ln)
-push_expr!(methodinfo::MethodInfo, mod::Module, ex::Expr) = methodinfo
-pop_expr!(methodinfo::MethodInfo) = methodinfo
-add_dependencies!(methodinfo::MethodInfo, be::CodeEdges, src, isrequired) = methodinfo
-add_includes!(methodinfo::MethodInfo, mod::Module, filename) = methodinfo
+add_signature!(methodinfo, @nospecialize(sig), ln) = push!(methodinfo, sig=>ln)
+push_expr!(methodinfo, mod::Module, ex::Expr) = methodinfo
+pop_expr!(methodinfo) = methodinfo
+add_dependencies!(methodinfo, be::CodeEdges, src, isrequired) = methodinfo
+add_includes!(methodinfo, mod::Module, filename) = methodinfo
 
 function is_some_include(@nospecialize(f))
     @assert !isa(f, Core.SSAValue) && !isa(f, JuliaInterpreter.SSAValue)
@@ -187,11 +189,11 @@ function minimal_evaluation!(@nospecialize(predicate), methodinfo, mod::Module,
     add_dependencies!(methodinfo, edges, src, isrequired)
     return isrequired, evalassign
 end
-@noinline minimal_evaluation!(@nospecialize(predicate), methodinfo, frame::JuliaInterpreter.Frame, mode::Symbol) =
+@noinline minimal_evaluation!(@nospecialize(predicate), methodinfo, frame::Frame, mode::Symbol) =
     minimal_evaluation!(predicate, methodinfo, moduleof(frame), frame.framecode.src, mode)
 
-function minimal_evaluation!(methodinfo, frame::JuliaInterpreter.Frame, mode::Symbol)
-    minimal_evaluation!(methodinfo, frame, mode) do @nospecialize(stmt), code
+function minimal_evaluation!(methodinfo, frame::Frame, mode::Symbol)
+    minimal_evaluation!(methodinfo, frame, mode) do @nospecialize(stmt), code::Vector{Any}
         ismeth, haseval, isinclude, isnamespace, istoplevel = categorize_stmt(stmt, code)
         isreq = ismeth | isinclude | istoplevel
         return mode === :sigs ? (isreq, haseval) : (isreq | isnamespace, haseval)
@@ -206,14 +208,14 @@ function methods_by_execution(mod::Module, ex::Expr; kwargs...)
 end
 
 """
-    methods_by_execution!(recurse=JuliaInterpreter.Compiled(), methodinfo, docexprs, mod::Module, ex::Expr;
+    methods_by_execution!([interp::Interpreter=JuliaInterpreter.Compiled(),] methodinfo, docexprs::DocExprs, mod::Module, ex::Expr;
                           mode=:eval, disablebp=true, skip_include=mode!==:eval, always_rethrow=false)
 
 Evaluate or analyze `ex` in the context of `mod`.
 Depending on the setting of `mode` (see the Extended help), it supports full evaluation or just the minimal
 evaluation needed to extract method signatures.
-`recurse` controls JuliaInterpreter's evaluation of any non-intercepted statement;
-likely choices are `JuliaInterpreter.Compiled()` or `JuliaInterpreter.finish_and_return!`.
+`interp` controls JuliaInterpreter's evaluation of any non-intercepted statement;
+likely choices are `JuliaInterpreter.Compiled()` or `JuliaInterpreter.RecursiveInterpreter()`.
 `methodinfo` is a cache for storing information about any method definitions (see [`CodeTrackingMethodInfo`](@ref)).
 `docexprs` is a cache for storing documentation expressions; obtain an empty one with `Revise.DocExprs()`.
 
@@ -249,7 +251,7 @@ The other keyword arguments are more straightforward:
   If false, the error is logged with `@error`. `InterruptException`s are always rethrown.
   This is primarily useful for debugging.
 """
-function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, mod::Module, ex::Expr;
+function methods_by_execution!(interp::Interpreter, methodinfo, docexprs::DocExprs, mod::Module, ex::Expr;
                                mode::Symbol=:eval, disablebp::Bool=true, always_rethrow::Bool=false, kwargs...)
     mode ∈ (:sigs, :eval, :evalmeth, :evalassign) || error("unsupported mode ", mode)
     lwr = Meta.lower(mod, ex)
@@ -261,8 +263,8 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, mod
         mode === :sigs && return nothing, nothing
         return Core.eval(mod, lwr), nothing
     end
-    frame = JuliaInterpreter.Frame(mod, lwr.args[1]::CodeInfo)
-    mode === :eval || LoweredCodeUtils.rename_framemethods!(recurse, frame)
+    frame = Frame(mod, lwr.args[1]::CodeInfo)
+    mode === :eval || LoweredCodeUtils.rename_framemethods!(interp, frame)
     # Determine whether we need interpreted mode
     isrequired, evalassign = minimal_evaluation!(methodinfo, frame, mode)
     # LoweredCodeUtils.print_with_code(stdout, frame.framecode.src, isrequired)
@@ -293,7 +295,7 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, mod
             foreach(disable, active_bp_refs)
         end
         ret = try
-            methods_by_execution!(recurse, methodinfo, docexprs, frame, isrequired; mode=mode, kwargs...)
+            methods_by_execution!(interp, methodinfo, docexprs, frame, isrequired; mode, kwargs...)
         catch err
             (always_rethrow || isa(err, InterruptException)) && (disablebp && foreach(enable, active_bp_refs); rethrow(err))
             loc = location_string(whereis(frame))
@@ -311,10 +313,11 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, mod
     end
     return ret, lwr
 end
-methods_by_execution!(methodinfo, docexprs, mod::Module, ex::Expr; kwargs...) =
-    methods_by_execution!(JuliaInterpreter.Compiled(), methodinfo, docexprs, mod, ex; kwargs...)
+methods_by_execution!(methodinfo, docexprs::DocExprs, mod::Module, ex::Expr; kwargs...) =
+    methods_by_execution!(Compiled(), methodinfo, docexprs, mod, ex; kwargs...)
 
-function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, frame::Frame, isrequired::AbstractVector{Bool}; mode::Symbol=:eval, skip_include::Bool=true)
+function methods_by_execution!(interp::Interpreter, methodinfo, docexprs::DocExprs, frame::Frame, isrequired::AbstractVector{Bool};
+                               mode::Symbol=:eval, skip_include::Bool=true)
     isok(lnn::LineTypes) = !iszero(lnn.line) || lnn.file !== :none   # might fail either one, but accept anything
 
     mod = moduleof(frame)
@@ -336,7 +339,7 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, fra
                 local value
                 for ex in stmt.args
                     ex isa Expr || continue
-                    value = methods_by_execution!(recurse, methodinfo, docexprs, mod, ex; mode=mode, disablebp=false, skip_include=skip_include)
+                    value = methods_by_execution!(interp, methodinfo, docexprs, mod, ex; mode, disablebp=false, skip_include)
                 end
                 isassign(frame, pc) && assign_this!(frame, value)
                 pc = next_or_nothing!(frame)
@@ -348,10 +351,10 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, fra
             #     # way to find them within the already-defined module.
             #     # They may be needed to define later signatures.
             #     # Note that named inner methods don't require special treatment.
-            #     pc = step_expr!(recurse, frame, stmt, true)
+            #     pc = step_expr!(interp, frame, stmt, true)
             elseif head === :method
                 empty!(signatures)
-                ret = methoddef!(recurse, signatures, frame, stmt, pc; define=mode!==:sigs)
+                ret = methoddef!(interp, signatures, frame, stmt, pc; define=mode!==:sigs)
                 if ret === nothing
                     # This was just `function foo end` or similar.
                     # However, it might have been followed by a thunk that defined a
@@ -367,22 +370,22 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, fra
                         end
                     end
                     @assert length(stmt.args) == 1
-                    pc = mode !== :sigs ? step_expr!(recurse, frame, stmt, true) :
+                    pc = mode !== :sigs ? step_expr!(interp, frame, stmt, true) :
                         next_or_nothing!(frame)
                 else
                     pc, pc3 = ret
                     # Get the line number from the body
                     stmt3 = pc_expr(frame, pc3)::Expr
                     lnn = nothing
-                    sigcode = lookup(frame, stmt3.args[2])::Core.SimpleVector
+                    sigcode = lookup(interp, frame, stmt3.args[2])::Core.SimpleVector
                     lnn = sigcode[end]
                     if !isa(lnn, LineNumberNode)
                         lnn = nothing
                     end
                     if lnn === nothing
                         bodycode = stmt3.args[end]
                         if !isa(bodycode, CodeInfo)
-                            bodycode = lookup(frame, bodycode)
+                            bodycode = lookup(interp, frame, bodycode)
                         end
                         if isa(bodycode, CodeInfo)
                             lnn = linetable(bodycode, 1)
@@ -456,13 +459,13 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, fra
                     # avoid redefining types unless we have to
                     pc = next_or_nothing!(frame)
                 else
-                    pc = step_expr!(recurse, frame, stmt, true)
+                    pc = step_expr!(interp, frame, stmt, true)
                 end
             elseif head === :call
-                f = lookup(frame, stmt.args[1])
+                f = lookup(interp, frame, stmt.args[1])
                 if isdefined(Core, :_defaultctors) && f === Core._defaultctors && length(stmt.args) == 3
-                    T = lookup(frame, stmt.args[2])
-                    lnn = lookup(frame, stmt.args[3])
+                    T = lookup(interp, frame, stmt.args[2])
+                    lnn = lookup(interp, frame, stmt.args[3])
                     if T isa Type && lnn isa LineNumberNode
                         empty!(signatures)
                         uT = Base.unwrap_unionall(T)::DataType
@@ -482,12 +485,12 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, fra
                     if mode===:sigs
                         pc = next_or_nothing!(frame)
                     else # also execute this call
-                        pc = step_expr!(recurse, frame, stmt, true)
+                        pc = step_expr!(interp, frame, stmt, true)
                     end
                 elseif f === Core.eval
                     # an @eval or eval block: this may contain method definitions, so intercept it.
-                    evalmod = lookup(frame, stmt.args[2])::Module
-                    evalex = lookup(frame, stmt.args[3])
+                    evalmod = lookup(interp, frame, stmt.args[2])::Module
+                    evalex = lookup(interp, frame, stmt.args[3])
                     value = nothing
                     for (newmod, newex) in ExprSplitter(evalmod, evalex)
                         if is_doc_expr(newex)
@@ -496,7 +499,7 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, fra
                         end
                         newex = unwrap(newex)
                         push_expr!(methodinfo, newmod, newex)
-                        value = methods_by_execution!(recurse, methodinfo, docexprs, newmod, newex; mode=mode, skip_include=skip_include, disablebp=false)
+                        value = methods_by_execution!(interp, methodinfo, docexprs, newmod, newex; mode, skip_include, disablebp=false)
                         pop_expr!(methodinfo)
                     end
                     assign_this!(frame, value)
@@ -505,29 +508,29 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, fra
                     # include calls need to be managed carefully from several standpoints, including
                     # path management and parsing new expressions
                     if length(stmt.args) == 2
-                        add_includes!(methodinfo, mod, lookup(frame, stmt.args[2]))
+                        add_includes!(methodinfo, mod, lookup(interp, frame, stmt.args[2]))
                     elseif length(stmt.args) == 3
-                        add_includes!(methodinfo, lookup(frame, stmt.args[2]), lookup(frame, stmt.args[3]))
+                        add_includes!(methodinfo, lookup(interp, frame, stmt.args[2]), lookup(interp, frame, stmt.args[3]))
                     else
                         error("Bad call to Core.include")
                     end
                     assign_this!(frame, nothing)  # FIXME: the file might return something different from `nothing`
                     pc = next_or_nothing!(frame)
                 elseif skip_include && f === Base.include
                     if length(stmt.args) == 2
-                        add_includes!(methodinfo, mod, lookup(frame, stmt.args[2]))
+                        add_includes!(methodinfo, mod, lookup(interp, frame, stmt.args[2]))
                     else # either include(module, path) or include(mapexpr, path)
-                        mod_or_mapexpr = lookup(frame, stmt.args[2])
+                        mod_or_mapexpr = lookup(interp, frame, stmt.args[2])
                         if isa(mod_or_mapexpr, Module)
-                            add_includes!(methodinfo, mod_or_mapexpr, lookup(frame, stmt.args[3]))
+                            add_includes!(methodinfo, mod_or_mapexpr, lookup(interp, frame, stmt.args[3]))
                         else
                             error("include(mapexpr, path) is not supported") # TODO (issue #634)
                         end
                     end
                     assign_this!(frame, nothing)  # FIXME: the file might return something different from `nothing`
                     pc = next_or_nothing!(frame)
                 elseif f === Base.Docs.doc! # && mode !== :eval
-                    fargs = JuliaInterpreter.collect_args(recurse, frame, stmt)
+                    fargs = JuliaInterpreter.collect_args(interp, frame, stmt)
                     popfirst!(fargs)
                     length(fargs) == 3 && push!(fargs, Union{})  # add the default sig
                     dmod::Module, b::Base.Docs.Binding, str::Base.Docs.DocStr, sig = fargs
@@ -559,16 +562,16 @@ function methods_by_execution!(@nospecialize(recurse), methodinfo, docexprs, fra
                     pc = next_or_nothing!(frame)
                 else
                     # A :call Expr we don't want to intercept
-                    pc = step_expr!(recurse, frame, stmt, true)
+                    pc = step_expr!(interp, frame, stmt, true)
                 end
             else
                 # An Expr we don't want to intercept
                 frame.pc = pc
-                pc = step_expr!(recurse, frame, stmt, true)
+                pc = step_expr!(interp, frame, stmt, true)
             end
         else
             # A statement we don't want to intercept
-            pc = step_expr!(recurse, frame, stmt, true)
+            pc = step_expr!(interp, frame, stmt, true)
         end
         pc === nothing && break
     end

src/packagedef.jl

@@ -515,7 +515,7 @@ end
 function eval_with_signatures(mod, ex::Expr; mode=:eval, kwargs...)
     methodinfo = CodeTrackingMethodInfo(ex)
     docexprs = DocExprs()
-    frame = methods_by_execution!(finish_and_return!, methodinfo, docexprs, mod, ex; mode=mode, kwargs...)[2]
+    frame = methods_by_execution!(methodinfo, docexprs, mod, ex; mode=mode, kwargs...)[2]
     return methodinfo.allsigs, methodinfo.deps, methodinfo.includes, frame
 end