Squashed 'packages/JuliaInterpreter/' changes from cc89e74..0b17ba7

0b17ba7 version 0.10.3
d357d35 make it easier to overload `evaluate_call!`
df933cc version 0.10.2
cf8f5fa update builtins.jl and add some simple tests for `using/import` evaluation (#688)
4ad9cbb version 0.10.1
d4c2351 quote rhs of `:const` expression (#687)
a29cc79 version 0.10.0
a8b4c95 update docs dependencies
ba766f9 implement a system for interpreting overlay methods (#682)
34cf432 remove the new `finish_nested_frame!` interface (#686)
a3ed4cc change the `recurse` interface to `AbstractInterpreter`-like interface (#683)
43f0215 rm `gen_lookup_ex`, and fix `@lookup`
0be2e0d version 0.9.46
837b899 deprecate `@lookup` macro in favor of `lookup` function (#685)
49220f9 deprecate 3-arg `@lookup` (#684)
93f6473 Support external method tables in method evaluation (#680)
77effa3 use `isdefinedglobal` instead of `isdefined(::Module, ::Symbol)` (#681)
0c37ff5 version 0.9.44
82e3897 make `llvmcall` search logic more robust (#679)
d66c5b9 Version 0.9.43 (#678)
3ab1279 Update builtins: `svec_ref` (#677)
0d1c51b Breakpoint setting for inverted linetable (#673)
c3739d1 Merge pull request #675 from JuliaDebug/teh/llvmcall_ega
b5340fd Assert that we only update pc in leaves
2ada170 Try harder to find llvmcalls
890cae3 Fix & improve `test/limits.jl` (#672)
b552e60 Update generated functions for Julia 1.12 (#668)
4ef265f Delay file removal (#674)
119b686 Improve breakpoints on Julia 1.12 (#669)
c12d8e0 Fix world age issues in test/toplevel (#671)
9f5d98d Fix kwprep stepping in Julia 1.12 and 1.11 (#667)
822632d Update builtins, allow stepping into invokelatest (#670)
22f186b Fix llvmcall/ccall and OpaqueClosure on nightly (#665)
2bbcead Fix builtins (#664)
aca346c Update all builtins
bd71e05 fix ir printing on 1.12 (#661)
33f1c25 bump version
cdefdcb allow getindex in lookup_or_eval :call (#651)
d88ba3a Bump Version
d37b7d8 Correct logic for ccall symbol lookup (#659)
126ab85 Bump Version
3ab3a11 More 1.12 world age adjustments (#658)
ba9783b fix builtins.jl
5fa1aaf update builtins, use `@invokelatest` instead of the function version
64cfc37 Bump Version
e807fdc Lookup bindings in the latest world (#657)
443d672 Bump codecov/codecov-action from 4 to 5 (#656)
b6cf649 update builtins.jl (#655)

git-subtree-dir: packages/JuliaInterpreter
git-subtree-split: 0b17ba720fa92256947a1015070ccbc98eb3a20d

Author: davidanthoff

.github/workflows/CI.yml

@@ -59,6 +59,6 @@ jobs:
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v4
+      - uses: codecov/codecov-action@v5
         with:
           file: lcov.info

Project.toml

@@ -1,6 +1,7 @@
 name = "JuliaInterpreter"
 uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a"
-version = "0.9.38"
+version = "0.10.3"
+authors = ["Tim Holy <[email protected]>, Kristoffer Carlsson <[email protected]>, Shuhei Kadowaki <[email protected]> and contributors"]
 
 [deps]
 CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
@@ -9,7 +10,7 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
-CodeTracking = "0.5.9, 1"
+CodeTracking = "1.3.9"
 julia = "1.10"
 
 [extras]
@@ -27,8 +28,7 @@ Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
 PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
-Tensors = "48a634ad-e948-5137-8d70-aa71f2a747f4"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["CassetteOverlay", "DataFrames", "Dates", "DeepDiffs", "Distributed", "FunctionWrappers", "HTTP", "LinearAlgebra", "Logging", "LoopVectorization", "Mmap", "PyCall", "SHA", "SparseArrays", "Tensors", "Test"]
+test = ["CassetteOverlay", "DataFrames", "Dates", "DeepDiffs", "Distributed", "FunctionWrappers", "HTTP", "LinearAlgebra", "Logging", "LoopVectorization", "Mmap", "PyCall", "SHA", "SparseArrays", "Test"]

README.md

@@ -8,8 +8,8 @@
 
 ## Installation
 
-```jl
-]add JuliaInterpreter
+```julia-repl
+pkg> add JuliaInterpreter
 ```
 
 ## Usage

bin/generate_builtins.jl

@@ -5,6 +5,8 @@ using InteractiveUtils
 # Builtins not present in 1.10 (the lowest supported version)
 const RECENTLY_ADDED = Core.Builtin[
     Core.current_scope,
+    isdefinedglobal,
+    Core.memorynew,
     Core.memoryref_isassigned,
     Core.memoryrefget,
     Core.memoryrefmodify!,
@@ -20,12 +22,38 @@ const RECENTLY_ADDED = Core.Builtin[
     setfieldonce!,
     setglobalonce!,
     swapglobal!,
+    Core._defaultctors,
+    Core._import,
+    Core._using,
+    # Recently became builtins
+    Base.invokelatest,
+    Base.invoke_in_world,
 ]
 # Builtins present from 1.10, not builtins (potentially still normal functions) anymore
 const RECENTLY_REMOVED = GlobalRef.(Ref(Core), [
-    :arrayref, :arrayset, :arrayset, :const_arrayref, :memoryref, :set_binding_type!
+    :arrayref, :arrayset, :arrayset, :const_arrayref, :memoryref, :set_binding_type!,
+    :_apply_pure, :_call_in_world, :_call_latest,
 ])
 const kwinvoke = Core.kwfunc(Core.invoke)
+const REQUIRES_WORLD = Core.Builtin[
+    setglobal!,
+    Core.get_binding_type,
+    swapglobal!,
+    modifyglobal!,
+    replaceglobal!,
+    setglobalonce!,
+]
+const CALL_LATEST = """args = getargs(interp, args, frame)
+        if !expand
+            return Some{Any}(Core._call_latest(args...))
+        end
+        new_expr = Expr(:call, args[1])
+        popfirst!(args)
+        for x in args
+            push!(new_expr.args, QuoteNode(x))
+        end
+        return maybe_recurse_expanded_builtin(interp, frame, new_expr)
+"""
 
 function scopedname(f)
     io = IOBuffer()
@@ -35,7 +63,7 @@ function scopedname(f)
     tn = typeof(f).name
     Base.isexported(tn.module, Symbol(fstr)) && return fstr
     fsym = Symbol(fstr)
-    isdefined(tn.module, fsym) && return string(tn.module) * '.' * fstr
+    isdefinedglobal(tn.module, fsym) && return string(tn.module) * '.' * fstr
     return "Base." * fstr
 end
 
@@ -49,13 +77,13 @@ function nargs(f, table, id)
     end
     # Specialize ~arrayref and arrayset~ memoryrefnew for small numbers of arguments
     # TODO: how about other memory intrinsics?
-    if (@static isdefined(Core, :memoryrefnew) ? f == Core.memoryrefnew : f == Core.memoryref)
+    if (@static isdefinedglobal(Core, :memoryrefnew) ? f == Core.memoryrefnew : f == Core.memoryref)
         maxarg = 5
     end
     return minarg, maxarg
 end
 
-function generate_fcall_nargs(fname, minarg, maxarg)
+function generate_fcall_nargs(fname, minarg, maxarg; requires_world::Bool=false)
     # Generate a separate call for each number of arguments
     maxarg < typemax(Int) || error("call this only for constrained number of arguments")
     annotation = fname == "fieldtype" ? "::Type" : ""
@@ -64,31 +92,41 @@ function generate_fcall_nargs(fname, minarg, maxarg)
         wrapper *= "$nargs\n            "
         argcall = ""
         for i = 1:nargs
-            argcall *= "@lookup(frame, args[$(i+1)])"
+            argcall *= "lookup(interp, frame, args[$(i+1)])"
             if i < nargs
                 argcall *= ", "
             end
         end
-        wrapper *= "return Some{Any}($fname($argcall)$annotation)"
+        wrapper *= requires_world ? "return Some{Any}(Base.invoke_in_world(frame.world, $fname, $argcall)$annotation)" :
+                                    "return Some{Any}($fname($argcall)$annotation)"
         if nargs < maxarg
             wrapper *= "\n        elseif nargs == "
         end
     end
     wrapper *= "\n        else"
-    wrapper *= "\n            return Some{Any}($fname(getargs(args, frame)...)$annotation)"  # to throw the correct error
+    # To throw the correct error
+    if requires_world
+        wrapper *= "\n            return Some{Any}(Base.invoke_in_world(frame.world, $fname, getargs(interp, args, frame)...)$annotation)"
+    else
+        wrapper *= "\n            return Some{Any}($fname(getargs(interp, args, frame)...)$annotation)"
+    end
     wrapper *= "\n        end"
     return wrapper
 end
 
 function generate_fcall(f, table, id)
     minarg, maxarg = nargs(f, table, id)
     fname = scopedname(f)
+    requires_world = f ∈ REQUIRES_WORLD
     if maxarg < typemax(Int)
-        return generate_fcall_nargs(fname, minarg, maxarg)
+        return generate_fcall_nargs(fname, minarg, maxarg; requires_world)
     end
     # A built-in with arbitrary or unknown number of arguments.
     # This will (unfortunately) use dynamic dispatch.
-    return "return Some{Any}($fname(getargs(args, frame)...))"
+    if requires_world
+        return "return Some{Any}(Base.invoke_in_world(frame.world, $fname, getargs(interp, args, frame)...))"
+    end
+    return "return Some{Any}($fname(getargs(interp, args, frame)...))"
 end
 
 # `io` is for the generated source file
@@ -104,42 +142,42 @@ function generate_builtins(io::IO)
 """
 # This file is generated by `generate_builtins.jl`. Do not edit by hand.
 
-function getargs(args, frame)
+function getargs(interp::Interpreter, args::Vector{Any}, frame::Frame)
     nargs = length(args)-1  # skip f
     callargs = resize!(frame.framedata.callargs, nargs)
     for i = 1:nargs
-        callargs[i] = @lookup(frame, args[i+1])
+        callargs[i] = lookup(interp, frame, args[i+1])
     end
     return callargs
 end
 
 const kwinvoke = Core.kwfunc(Core.invoke)
 
-function maybe_recurse_expanded_builtin(frame, new_expr)
+function maybe_recurse_expanded_builtin(interp::Interpreter, frame::Frame, new_expr::Expr)
     f = new_expr.args[1]
     if isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction)
-        return maybe_evaluate_builtin(frame, new_expr, true)
+        return maybe_evaluate_builtin(interp, frame, new_expr, true)
     else
         return new_expr
     end
 end
 
 \"\"\"
-    ret = maybe_evaluate_builtin(frame, call_expr, expand::Bool)
+    ret = maybe_evaluate_builtin(interp::Interpreter, frame::Frame, call_expr::Expr, expand::Bool)
 
 If `call_expr` is to a builtin function, evaluate it, returning the result inside a `Some` wrapper.
 Otherwise, return `call_expr`.
 
 If `expand` is true, `Core._apply_iterate` calls will be resolved as a call to the applied function.
 \"\"\"
-function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
+function maybe_evaluate_builtin(interp::Interpreter, frame::Frame, call_expr::Expr, expand::Bool)
     args = call_expr.args
     nargs = length(args) - 1
     fex = args[1]
     if isa(fex, QuoteNode)
         f = fex.value
     else
-        f = @lookup(frame, fex)
+        f = lookup(interp, frame, fex)
     end
 
     if f isa Core.OpaqueClosure
@@ -172,15 +210,15 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
             print(io,
 """
     $head f === tuple
-        return Some{Any}(ntupleany(i->@lookup(frame, args[i+1]), length(args)-1))
+        return Some{Any}(ntupleany(i::Int->lookup(interp, frame, args[i+1]), length(args)-1))
 """)
             continue
         elseif f === Core._apply_iterate
             # Resolve varargs calls
             print(io,
 """
     $head f === Core._apply_iterate
-        argswrapped = getargs(args, frame)
+        argswrapped = getargs(interp, args, frame)
         if !expand
             return Some{Any}(Core._apply_iterate(argswrapped...))
         end
@@ -193,7 +231,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         for x in argsflat
             push!(new_expr.args, QuoteNode(x))
         end
-        return maybe_recurse_expanded_builtin(frame, new_expr)
+        return maybe_recurse_expanded_builtin(interp, frame, new_expr)
 """)
             continue
         elseif f === Core.invoke
@@ -202,42 +240,26 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
 """
     $head f === $fstr
         if !expand
-            argswrapped = getargs(args, frame)
+            argswrapped = getargs(interp, args, frame)
             return Some{Any}($fstr(argswrapped...))
         end
         # This uses the original arguments to avoid looking them up twice
         # See #442
         return Expr(:call, invoke, args[2:end]...)
-""")
-            continue
-        elseif f === Core._call_latest
-            print(io,
-"""
-    elseif f === Core._call_latest
-        args = getargs(args, frame)
-        if !expand
-            return Some{Any}(Core._call_latest(args...))
-        end
-        new_expr = Expr(:call, args[1])
-        popfirst!(args)
-        for x in args
-            push!(new_expr.args, QuoteNode(x))
-        end
-        return maybe_recurse_expanded_builtin(frame, new_expr)
 """)
             continue
         elseif f === Core.current_scope
             print(io,
 """
-    elseif @static isdefined(Core, :current_scope) && f === Core.current_scope
+    elseif @static isdefinedglobal(Core, :current_scope) && f === Core.current_scope
         if nargs == 0
             currscope = Core.current_scope()
             for scope in frame.framedata.current_scopes
                 currscope = Scope(currscope, scope.values...)
             end
             return Some{Any}(currscope)
         else
-            return Some{Any}(Core.current_scope(getargs(args, frame)...))
+            return Some{Any}(Core.current_scope(getargs(interp, args, frame)...))
         end
 """)
             continue
@@ -246,9 +268,12 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         id = findfirst(isequal(f), Core.Compiler.T_FFUNC_KEY)
         fcall = generate_fcall(f, Core.Compiler.T_FFUNC_VAL, id)
         if f in RECENTLY_ADDED
+            if f === Core.invokelatest
+                fcall = replace(CALL_LATEST, "_call_latest" => "invokelatest")
+            end
             print(io,
 """
-    $head @static isdefined($(ft.name.module), $(repr(nameof(f)))) && f === $fname
+    $head @static isdefinedglobal($(ft.name.module), $(repr(nameof(f)))) && f === $fname
         $fcall
 """)
         else
@@ -270,13 +295,13 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         if nargs == 1
             call_expr = copy(call_expr)
             args2 = args[2]
-            call_expr.args[2] = isa(args2, QuoteNode) ? args2 : @lookup(frame, args2)
+            call_expr.args[2] = isa(args2, QuoteNode) ? args2 : lookup(interp, frame, args2)
             return Some{Any}(Core.eval(moduleof(frame), call_expr))
         elseif nargs == 2
             call_expr = copy(call_expr)
             args2 = args[2]
-            call_expr.args[2] = isa(args2, QuoteNode) ? args2 : @lookup(frame, args2)
-            call_expr.args[3] = @lookup(frame, args[3])
+            call_expr.args[2] = isa(args2, QuoteNode) ? args2 : lookup(interp, frame, args2)
+            call_expr.args[3] = lookup(interp, frame, args[3])
             return Some{Any}(Core.eval(moduleof(frame), call_expr))
         end
 """)
@@ -292,13 +317,18 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         elseif name === :set_binding_type!
             minarg = 2
             maxarg = 3
+        elseif name in (:_apply_pure, :_call_in_world, :_call_latest)
+            minarg = 0
+            maxarg = typemax(Int)
         end
         _scopedname = "$mod.$name"
-        fcall = generate_fcall_nargs(_scopedname, minarg, maxarg)
+        fcall = name === :_call_latest ? CALL_LATEST :
+                maxarg < typemax(Int) ? generate_fcall_nargs(_scopedname, minarg, maxarg) :
+                                        "return Some{Any}($_scopedname(getargs(interp, args, frame)...))"
         rname = repr(name)
         print(io,
 """
-    elseif @static (isdefined($mod, $rname) && $_scopedname isa Core.Builtin) && f === $_scopedname
+    elseif @static (isdefinedglobal($mod, $rname) && $_scopedname isa Core.Builtin) && f === $_scopedname
         $fcall
 """)
     end
@@ -307,7 +337,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
     minmin, maxmax = typemax(Int), 0
     for fsym in names(Core.Intrinsics)
         fsym === :Intrinsics && continue
-        isdefined(Base, fsym) || continue
+        isdefinedglobal(Base, fsym) || continue
         f = getfield(Base, fsym)
         id = reinterpret(Int32, f) + 1
         minarg, maxarg = nargs(f, Core.Compiler.T_IFUNC, id)
@@ -333,7 +363,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
     print(io,
 """
     if isa(f, Core.IntrinsicFunction)
-        cargs = getargs(args, frame)
+        cargs = getargs(interp, args, frame)
         if f === Core.Intrinsics.have_fma && length(cargs) == 1
             cargs1 = cargs[1]
             if cargs1 == Float64
@@ -364,7 +394,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
 """
     end
     if isa(f, typeof(kwinvoke))
-        return Some{Any}(kwinvoke(getargs(args, frame)...))
+        return Some{Any}(kwinvoke(getargs(interp, args, frame)...))
     end
     return call_expr
 end