change the recurse interface to AbstractInterpreter-like interface (#683)

Align the `recurse` argument to something like the base Compiler's
`AbstractInterpreter` and make JuliaInterpreter routines overloadable
properly.

This change is quite breaking (thus bumping the minor version of this
package), but necessary to enhance the customizability of JI.
For example, it will make it easier to add changes like
#682 in a nicer way, but also should
enable better designs in packages such as Revise and JET.

- allow specifying interpreter with `@interpret`
- better error handling for `@interpret`
- better `native_call` implementation
- make `lookup` overloadable
- define `NonRecursiveInterpreter` and make `Compiled` alias to it
`Compiled` is preserved for backward compatibility.

---------

Co-authored-by: Cédric Belmant <[email protected]>

Author: aviatesk

Project.toml

@@ -1,6 +1,6 @@
 name = "JuliaInterpreter"
 uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a"
-version = "0.9.46"
+version = "0.10.0"
 
 [deps]
 CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"

bin/generate_builtins.jl

@@ -41,7 +41,7 @@ const REQUIRES_WORLD = Core.Builtin[
     replaceglobal!,
     setglobalonce!,
 ]
-const CALL_LATEST = """args = getargs(args, frame)
+const CALL_LATEST = """args = getargs(interp, args, frame)
         if !expand
             return Some{Any}(Core._call_latest(args...))
         end
@@ -50,7 +50,7 @@ const CALL_LATEST = """args = getargs(args, frame)
         for x in args
             push!(new_expr.args, QuoteNode(x))
         end
-        return maybe_recurse_expanded_builtin(frame, new_expr)
+        return maybe_recurse_expanded_builtin(interp, frame, new_expr)
 """
 
 function scopedname(f)
@@ -90,7 +90,7 @@ function generate_fcall_nargs(fname, minarg, maxarg; requires_world::Bool=false)
         wrapper *= "$nargs\n            "
         argcall = ""
         for i = 1:nargs
-            argcall *= "lookup(frame, args[$(i+1)])"
+            argcall *= "lookup(interp, frame, args[$(i+1)])"
             if i < nargs
                 argcall *= ", "
             end
@@ -104,9 +104,9 @@ function generate_fcall_nargs(fname, minarg, maxarg; requires_world::Bool=false)
     wrapper *= "\n        else"
     # To throw the correct error
     if requires_world
-        wrapper *= "\n            return Some{Any}(Base.invoke_in_world(frame.world, $fname, getargs(args, frame)...)$annotation)"
+        wrapper *= "\n            return Some{Any}(Base.invoke_in_world(frame.world, $fname, getargs(interp, args, frame)...)$annotation)"
     else
-        wrapper *= "\n            return Some{Any}($fname(getargs(args, frame)...)$annotation)"
+        wrapper *= "\n            return Some{Any}($fname(getargs(interp, args, frame)...)$annotation)"
     end
     wrapper *= "\n        end"
     return wrapper
@@ -122,9 +122,9 @@ function generate_fcall(f, table, id)
     # A built-in with arbitrary or unknown number of arguments.
     # This will (unfortunately) use dynamic dispatch.
     if requires_world
-        return "return Some{Any}(Base.invoke_in_world(frame.world, $fname, getargs(args, frame)...))"
+        return "return Some{Any}(Base.invoke_in_world(frame.world, $fname, getargs(interp, args, frame)...))"
     end
-    return "return Some{Any}($fname(getargs(args, frame)...))"
+    return "return Some{Any}($fname(getargs(interp, args, frame)...))"
 end
 
 # `io` is for the generated source file
@@ -140,42 +140,42 @@ function generate_builtins(io::IO)
 """
 # This file is generated by `generate_builtins.jl`. Do not edit by hand.
 
-function getargs(args, frame)
+function getargs(interp::Interpreter, args::Vector{Any}, frame::Frame)
     nargs = length(args)-1  # skip f
     callargs = resize!(frame.framedata.callargs, nargs)
     for i = 1:nargs
-        callargs[i] = lookup(frame, args[i+1])
+        callargs[i] = lookup(interp, frame, args[i+1])
     end
     return callargs
 end
 
 const kwinvoke = Core.kwfunc(Core.invoke)
 
-function maybe_recurse_expanded_builtin(frame, new_expr)
+function maybe_recurse_expanded_builtin(interp::Interpreter, frame::Frame, new_expr::Expr)
     f = new_expr.args[1]
     if isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction)
-        return maybe_evaluate_builtin(frame, new_expr, true)
+        return maybe_evaluate_builtin(interp, frame, new_expr, true)
     else
         return new_expr
     end
 end
 
 \"\"\"
-    ret = maybe_evaluate_builtin(frame, call_expr, expand::Bool)
+    ret = maybe_evaluate_builtin(interp::Interpreter, frame::Frame, call_expr::Expr, expand::Bool)
 
 If `call_expr` is to a builtin function, evaluate it, returning the result inside a `Some` wrapper.
 Otherwise, return `call_expr`.
 
 If `expand` is true, `Core._apply_iterate` calls will be resolved as a call to the applied function.
 \"\"\"
-function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
+function maybe_evaluate_builtin(interp::Interpreter, frame::Frame, call_expr::Expr, expand::Bool)
     args = call_expr.args
     nargs = length(args) - 1
     fex = args[1]
     if isa(fex, QuoteNode)
         f = fex.value
     else
-        f = lookup(frame, fex)
+        f = lookup(interp, frame, fex)
     end
 
     if f isa Core.OpaqueClosure
@@ -208,15 +208,15 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
             print(io,
 """
     $head f === tuple
-        return Some{Any}(ntupleany(i::Int->lookup(frame, args[i+1]), length(args)-1))
+        return Some{Any}(ntupleany(i::Int->lookup(interp, frame, args[i+1]), length(args)-1))
 """)
             continue
         elseif f === Core._apply_iterate
             # Resolve varargs calls
             print(io,
 """
     $head f === Core._apply_iterate
-        argswrapped = getargs(args, frame)
+        argswrapped = getargs(interp, args, frame)
         if !expand
             return Some{Any}(Core._apply_iterate(argswrapped...))
         end
@@ -229,7 +229,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         for x in argsflat
             push!(new_expr.args, QuoteNode(x))
         end
-        return maybe_recurse_expanded_builtin(frame, new_expr)
+        return maybe_recurse_expanded_builtin(interp, frame, new_expr)
 """)
             continue
         elseif f === Core.invoke
@@ -238,7 +238,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
 """
     $head f === $fstr
         if !expand
-            argswrapped = getargs(args, frame)
+            argswrapped = getargs(interp, args, frame)
             return Some{Any}($fstr(argswrapped...))
         end
         # This uses the original arguments to avoid looking them up twice
@@ -257,7 +257,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
             end
             return Some{Any}(currscope)
         else
-            return Some{Any}(Core.current_scope(getargs(args, frame)...))
+            return Some{Any}(Core.current_scope(getargs(interp, args, frame)...))
         end
 """)
             continue
@@ -293,13 +293,13 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         if nargs == 1
             call_expr = copy(call_expr)
             args2 = args[2]
-            call_expr.args[2] = isa(args2, QuoteNode) ? args2 : lookup(frame, args2)
+            call_expr.args[2] = isa(args2, QuoteNode) ? args2 : lookup(interp, frame, args2)
             return Some{Any}(Core.eval(moduleof(frame), call_expr))
         elseif nargs == 2
             call_expr = copy(call_expr)
             args2 = args[2]
-            call_expr.args[2] = isa(args2, QuoteNode) ? args2 : lookup(frame, args2)
-            call_expr.args[3] = lookup(frame, args[3])
+            call_expr.args[2] = isa(args2, QuoteNode) ? args2 : lookup(interp, frame, args2)
+            call_expr.args[3] = lookup(interp, frame, args[3])
             return Some{Any}(Core.eval(moduleof(frame), call_expr))
         end
 """)
@@ -322,7 +322,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         _scopedname = "$mod.$name"
         fcall = name === :_call_latest ? CALL_LATEST :
                 maxarg < typemax(Int) ? generate_fcall_nargs(_scopedname, minarg, maxarg) :
-                                        "return Some{Any}($_scopedname(getargs(args, frame)...))"
+                                        "return Some{Any}($_scopedname(getargs(interp, args, frame)...))"
         rname = repr(name)
         print(io,
 """
@@ -361,7 +361,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
     print(io,
 """
     if isa(f, Core.IntrinsicFunction)
-        cargs = getargs(args, frame)
+        cargs = getargs(interp, args, frame)
         if f === Core.Intrinsics.have_fma && length(cargs) == 1
             cargs1 = cargs[1]
             if cargs1 == Float64
@@ -392,7 +392,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
 """
     end
     if isa(f, typeof(kwinvoke))
-        return Some{Any}(kwinvoke(getargs(args, frame)...))
+        return Some{Any}(kwinvoke(getargs(interp, args, frame)...))
     end
     return call_expr
 end

docs/src/dev_reference.md

@@ -6,6 +6,15 @@
 @interpret
 ```
 
+## Interpreter
+
+```@docs
+JuliaInterpreter.Interpreter
+JuliaInterpreter.RecursiveInterpreter
+JuliaInterpreter.NonRecursiveInterpreter
+JuliaInterpreter.Compiled
+```
+
 ## Frame creation
 
 ```@docs
@@ -31,11 +40,11 @@ leaf
 ## Frame execution
 
 ```@docs
-JuliaInterpreter.Compiled
 JuliaInterpreter.step_expr!
 JuliaInterpreter.finish!
 JuliaInterpreter.finish_and_return!
 JuliaInterpreter.finish_stack!
+JuliaInterpreter.finish_nested_frame!
 JuliaInterpreter.get_return
 JuliaInterpreter.next_until!
 JuliaInterpreter.maybe_next_until!
@@ -67,7 +76,7 @@ toggle
 break_on
 break_off
 breakpoints
-JuliaInterpreter.dummy_breakpoint
+JuliaInterpreter.BreakOnCall
 ```
 
 ## Types