Merge pull request #58 from JuliaDebug/teh/rename_pt

Rename prepare_toplevel -> split_expressions

Author: timholy

docs/src/dev_reference.md

@@ -16,7 +16,7 @@ JuliaInterpreter.determine_method_for_expr
 JuliaInterpreter.prepare_args
 JuliaInterpreter.prepare_call
 JuliaInterpreter.prepare_thunk
-JuliaInterpreter.prepare_toplevel
+JuliaInterpreter.split_expressions
 JuliaInterpreter.get_call_framecode
 JuliaInterpreter.optimize!
 ```

docs/src/internals.md

@@ -246,7 +246,7 @@ Here's a more fine-grained look at what's happening under the hood (and a robust
 for more complex situations where there may be nested calls of new methods):
 
 ```julia
-modexs, _ = JuliaInterpreter.prepare_toplevel(Main, ex)
+modexs, _ = JuliaInterpreter.split_expressions(Main, ex)
 stack = JuliaStackFrame[]
 for (mod, e) in modexs
     frame = JuliaInterpreter.prepare_thunk(mod, e)

src/JuliaInterpreter.jl

@@ -362,7 +362,7 @@ end
     frame = prepare_thunk(mod::Module, expr::Expr)
 
 Prepare `expr` for evaluation in `mod`. `expr` should be a "straightforward" expression,
-one that does not require special top-level handling (see [`JuliaInterpreter.prepare_toplevel`](@ref)).
+one that does not require special top-level handling (see [`JuliaInterpreter.split_expressions`](@ref)).
 """
 function prepare_thunk(mod::Module, thunk::Expr, recursive=false)
     if isexpr(thunk, :thunk)
@@ -382,7 +382,7 @@ function prepare_thunk((mod, ex)::Tuple{Module,Expr})
     if isexpr(lwr, :thunk)
         return prepare_thunk(mod, lwr)
     # elseif isexpr(lwr, :toplevel)
-    #     return prepare_toplevel!(frames, docexprs, lex, mod, lwr; extract_docexprs=extract_docexprs, filename=filename)
+    #     return split_expressions!(frames, docexprs, lex, mod, lwr; extract_docexprs=extract_docexprs, filename=filename)
     # elseif isa(lwr, Expr) && (lwr.head == :export || lwr.head == :using || lwr.head == :import)
     #     @show lwr
     #     push!(modexs, (mod, ex, lwr))
@@ -394,15 +394,20 @@ function prepare_thunk((mod, ex)::Tuple{Module,Expr})
 end
 
 """
-    modexs, docexprs = prepare_toplevel(mod::Module, expr::Expr; extract_docexprs=false)
+    modexs, docexprs = split_expressions(mod::Module, expr::Expr; extract_docexprs=false)
 
-Break `expr` into a list `modexs` of blocks to be successively executed at top level.
-This is used when `expr` defines new structs, new methods, or new modules.
+Break `expr` into a list `modexs` of sequential blocks. This is often needed when `expr`
+needs to be evaluated at top level.
 
-`modexs[i]` is a `(Module, Expr)` tuple. A prototype of how to use these is:
+`modexs[i]` is a `(mod::Module, ex::Expr)` tuple, where `ex` is to be evaluated in `mod`.
+
+# Toplevel evaluation
+
+For code that defines new structs, new methods, or new macros, it can be important to evaluate
+these expressions carefully:
 
     stack = JuliaStackFrame[]
-    for modex in modexs
+    for modex in modexs    # or use `for (mod, ex) in modexs` to split the tuple
         frame = JuliaInterpreter.prepare_thunk(modex)
         while true
             JuliaInterpreter.through_methoddef_or_done!(stack, frame) === nothing && break
@@ -412,21 +417,22 @@ This is used when `expr` defines new structs, new methods, or new modules.
 The `while` loop here deserves some explanation. Occasionally, a frame may define new methods
 (e.g., anonymous or local functions) and then call those methods. In such cases, running
 the entire frame as a single block (e.g., with [`JuliaInterpreter.finish_and_return!`](@ref)
-can trigger "method is too new..." errors. Instead, this runs each frame, but returns to the caller
-after any new method is defined. When this loop is running at top level (e.g., in the REPL),
-this allows the world age to update and thus avoid "method is too new..." errors.
+can trigger "method is too new..." errors. Instead, the approach above runs each frame,
+but returns to the caller after any new method is defined. When this loop is running at
+top level (e.g., in the REPL), this allows the world age to update and thus avoid
+"method is too new..." errors.
 
-Putting the above nested loop inside a function defeats the entire purpose of
-`prepare_toplevel`. If necessary, run that loop as
+Putting the above nested loop inside a function defeats its purpose, because inside a
+compiled function the world age will not update. If necessary, use the following strategy:
 
     Core.eval(somemodule, Expr(:toplevel, quote
         body
     ))
 
-where `body` executes the loop plus any preparatory statements required to make the
+where `body` contains the nested loop, plus any preparatory statements required to make the
 necessary variables available at top level in `somemodule`.
 """
-function prepare_toplevel(mod::Module, expr::Expr; filename=nothing, kwargs...)
+function split_expressions(mod::Module, expr::Expr; filename=nothing, kwargs...)
     modexs = Tuple{Module,Expr}[]
     docexprs = Dict{Module,Vector{Expr}}()
     if filename === nothing
@@ -437,7 +443,7 @@ function prepare_toplevel(mod::Module, expr::Expr; filename=nothing, kwargs...)
             filename="toplevel"
         end
     end
-    return prepare_toplevel!(modexs, docexprs, mod, expr; filename=filename, kwargs...)
+    return split_expressions!(modexs, docexprs, mod, expr; filename=filename, kwargs...)
 end
 
 """
@@ -459,18 +465,18 @@ function isdocexpr(ex)
     return false
 end
 
-prepare_toplevel!(modexs, docexprs, mod::Module, ex::Expr; kwargs...) =
-    prepare_toplevel!(modexs, docexprs, Expr(:block), mod, ex; kwargs...)
+split_expressions!(modexs, docexprs, mod::Module, ex::Expr; kwargs...) =
+    split_expressions!(modexs, docexprs, Expr(:block), mod, ex; kwargs...)
 
-function prepare_toplevel!(modexs, docexprs, lex::Expr, mod::Module, ex::Expr; extract_docexprs=false, filename="toplevel")
+function split_expressions!(modexs, docexprs, lex::Expr, mod::Module, ex::Expr; extract_docexprs=false, filename="toplevel")
     # lex is the expression we'll lower; it will accumulate LineNumberNodes and a
     # single top-level expression. We split blocks, module defs, etc.
     if ex.head == :toplevel || ex.head == :block
-        prepare_toplevel!(modexs, docexprs, lex, mod, ex.args; extract_docexprs=extract_docexprs, filename=filename)
+        split_expressions!(modexs, docexprs, lex, mod, ex.args; extract_docexprs=extract_docexprs, filename=filename)
     elseif ex.head == :module
         modname = ex.args[2]::Symbol
         newmod = isdefined(mod, modname) ? getfield(mod, modname) : Core.eval(mod, :(module $modname end))
-        prepare_toplevel!(modexs, docexprs, lex, newmod, ex.args[3]; extract_docexprs=extract_docexprs, filename=filename)
+        split_expressions!(modexs, docexprs, lex, newmod, ex.args[3]; extract_docexprs=extract_docexprs, filename=filename)
     elseif extract_docexprs && isdocexpr(ex)
         docexs = get(docexprs, mod, nothing)
         if docexs === nothing
@@ -479,7 +485,7 @@ function prepare_toplevel!(modexs, docexprs, lex::Expr, mod::Module, ex::Expr; e
         push!(docexs, ex)
         body = ex.args[4]
         if isa(body, Expr) && body.head != :call
-            prepare_toplevel!(modexs, docexprs, lex, mod, body; extract_docexprs=extract_docexprs, filename=filename)
+            split_expressions!(modexs, docexprs, lex, mod, body; extract_docexprs=extract_docexprs, filename=filename)
         end
     else
         if isempty(lex.args)
@@ -492,10 +498,10 @@ function prepare_toplevel!(modexs, docexprs, lex::Expr, mod::Module, ex::Expr; e
     return modexs, docexprs
 end
 
-function prepare_toplevel!(frames, docexprs, lex, mod::Module, args::Vector{Any}; filename="toplevel", kwargs...)
+function split_expressions!(frames, docexprs, lex, mod::Module, args::Vector{Any}; filename="toplevel", kwargs...)
     for a in args
         if isa(a, Expr)
-            prepare_toplevel!(frames, docexprs, lex, mod, a; filename=filename, kwargs...)
+            split_expressions!(frames, docexprs, lex, mod, a; filename=filename, kwargs...)
         elseif isa(a, LineNumberNode)
             if a.file === nothing  # happens with toplevel expressions on Julia 1.2
                 push!(lex.args, LineNumberNode(a.line, Symbol(filename)))

src/interpret.jl

@@ -380,7 +380,7 @@ function _step_expr!(stack, frame, @nospecialize(node), pc::JuliaProgramCounter,
                 elseif node.head == :primitive_type
                     evaluate_primitivetype!(stack, frame, node, pc)
                 elseif node.head == :module
-                    error("this should have been handled by prepare_toplevel")
+                    error("this should have been handled by split_expressions")
                 elseif node.head == :using || node.head == :import || node.head == :export
                     Core.eval(moduleof(frame), node)
                 elseif node.head == :const
@@ -409,7 +409,7 @@ function _step_expr!(stack, frame, @nospecialize(node), pc::JuliaProgramCounter,
                 elseif node.head == :toplevel
                     mod = moduleof(frame)
                     newstack = similar(stack, 0)
-                    modexs, _ = prepare_toplevel(mod, node)
+                    modexs, _ = split_expressions(mod, node)
                     Core.eval(mod, Expr(:toplevel,
                         :(for modex in $modexs
                               newframe = ($prepare_thunk)(modex)
@@ -477,7 +477,7 @@ function handle_err(frame, err)
         if (err.world != typemax(UInt) &&
             hasmethod(err.f, arg_types) &&
             !hasmethod(err.f, arg_types, world = err.world))
-            @warn "likely failure to return to toplevel, try JuliaInterpreter.prepare_toplevel"
+            @warn "likely failure to return to toplevel, try JuliaInterpreter.split_expressions"
             rethrow(err)
         end
     end

src/precompile.jl

@@ -23,7 +23,7 @@ function _precompile_()
         precompile(Tuple{typeof(f), Compiled, JuliaStackFrame, JuliaProgramCounter, Bool})
         precompile(Tuple{typeof(f), Vector{JuliaStackFrame}, JuliaStackFrame, JuliaProgramCounter, Bool})
     end
-    precompile(Tuple{typeof(prepare_toplevel), Module, Expr})
+    precompile(Tuple{typeof(split_expressions), Module, Expr})
     precompile(Tuple{typeof(prepare_thunk), Module, Expr})
     precompile(Tuple{typeof(prepare_locals), JuliaFrameCode, Vector{Any}})
     precompile(Tuple{typeof(prepare_args), Any, Vector{Any}, Vector{Any}})

test/limits.jl

@@ -15,7 +15,7 @@ using Test
     end
     @elapsed sum(rand(5))
     """; filename="fake.jl")
-    modexs, _ = JuliaInterpreter.prepare_toplevel(Main, ex; filename="fake.jl")
+    modexs, _ = JuliaInterpreter.split_expressions(Main, ex; filename="fake.jl")
     # find the 3rd assignment statement in the 2nd frame (corresponding to the x += 1 line)
     frame = JuliaInterpreter.prepare_thunk(modexs[2])
     i = 0
@@ -51,7 +51,7 @@ module EvalLimited end
         s += 1
     end
     """)
-    modexs, _ = JuliaInterpreter.prepare_toplevel(EvalLimited, ex)
+    modexs, _ = JuliaInterpreter.split_expressions(EvalLimited, ex)
     nstmts = 1000  # enough to ensure it finishes
     for modex in modexs
         frame = JuliaInterpreter.prepare_thunk(modex)
@@ -78,7 +78,7 @@ module EvalLimited end
         insert!(ex.args, 2, LineNumberNode(2, Symbol("fake.jl")))
         insert!(ex.args, 1, LineNumberNode(1, Symbol("fake.jl")))
     end
-    modexs, _ = JuliaInterpreter.prepare_toplevel(EvalLimited, ex)
+    modexs, _ = JuliaInterpreter.split_expressions(EvalLimited, ex)
     nstmts = 100 # enough to ensure it gets into the loop but doesn't finish
     for modex in modexs
         frame = JuliaInterpreter.prepare_thunk(modex)
@@ -96,7 +96,7 @@ module EvalLimited end
 
     # Now try again with recursive stack
     empty!(aborts)
-    modexs, _ = JuliaInterpreter.prepare_toplevel(EvalLimited, ex)
+    modexs, _ = JuliaInterpreter.split_expressions(EvalLimited, ex)
     for modex in modexs
         frame = JuliaInterpreter.prepare_thunk(modex)
         @test isa(frame, JuliaStackFrame)

test/toplevel.jl

@@ -17,7 +17,7 @@ module Toplevel end
 
 @testset "toplevel" begin
     stack = JuliaInterpreter.JuliaStackFrame[]
-    modexs, _ = JuliaInterpreter.prepare_toplevel(Toplevel, read_and_parse("toplevel_script.jl"))
+    modexs, _ = JuliaInterpreter.split_expressions(Toplevel, read_and_parse("toplevel_script.jl"))
     for modex in modexs
         frame = JuliaInterpreter.prepare_thunk(modex)
         while true
@@ -167,7 +167,7 @@ module Toplevel end
        end
        end
    end
-   modexs, _ = JuliaInterpreter.prepare_toplevel(Toplevel, ex)
+   modexs, _ = JuliaInterpreter.split_expressions(Toplevel, ex)
    for modex in modexs
        frame = JuliaInterpreter.prepare_thunk(modex)
        while true
@@ -208,7 +208,7 @@ ex = quote
         test_15703()
     end
 end
-modexs, _ = JuliaInterpreter.prepare_toplevel(IncTest, ex)
+modexs, _ = JuliaInterpreter.split_expressions(IncTest, ex)
 stack = JuliaStackFrame[]
 for (i, modex) in enumerate(modexs)
     frame = JuliaInterpreter.prepare_thunk(modex)
@@ -226,7 +226,7 @@ end
               EnumChild0
               EnumChild1
           end))
-    modexs, _ = JuliaInterpreter.prepare_toplevel(Toplevel, ex)
+    modexs, _ = JuliaInterpreter.split_expressions(Toplevel, ex)
     stack = JuliaStackFrame[]
     for modex in modexs
         frame = JuliaInterpreter.prepare_thunk(modex)
@@ -250,7 +250,7 @@ end
         ret[] = map(x->parse(Int16, x), AbstractString[])
     end
     stack = JuliaStackFrame[]
-    modexs, _ = JuliaInterpreter.prepare_toplevel(LowerAnon, ex1)
+    modexs, _ = JuliaInterpreter.split_expressions(LowerAnon, ex1)
     for modex in modexs
         frame = JuliaInterpreter.prepare_thunk(modex)
         while true
@@ -259,7 +259,7 @@ end
     end
     @test isa(LowerAnon.ret[], Vector{Int16})
     LowerAnon.ret[] = nothing
-    modexs, _ = JuliaInterpreter.prepare_toplevel(LowerAnon, ex2)
+    modexs, _ = JuliaInterpreter.split_expressions(LowerAnon, ex2)
     for modex in modexs
         frame = JuliaInterpreter.prepare_thunk(modex)
         while true
@@ -272,7 +272,7 @@ end
     ex3 = quote
         const BitIntegerType = Union{map(T->Type{T}, Base.BitInteger_types)...}
     end
-    modexs, _ = JuliaInterpreter.prepare_toplevel(LowerAnon, ex3)
+    modexs, _ = JuliaInterpreter.split_expressions(LowerAnon, ex3)
     for modex in modexs
         frame = JuliaInterpreter.prepare_thunk(modex)
         while true
@@ -286,7 +286,7 @@ end
         z = map(x->x^2+y, [1,2,3])
         y = 4
     end
-    modexs, _ = JuliaInterpreter.prepare_toplevel(LowerAnon, ex4)
+    modexs, _ = JuliaInterpreter.split_expressions(LowerAnon, ex4)
     for modex in modexs
         frame = JuliaInterpreter.prepare_thunk(modex)
         while true
@@ -319,7 +319,7 @@ end
         end
     end
     Core.eval(Toplevel, Expr(:toplevel, ex.args...))
-    modexs, docexprs = JuliaInterpreter.prepare_toplevel(Toplevel, ex; extract_docexprs=true)
+    modexs, docexprs = JuliaInterpreter.split_expressions(Toplevel, ex; extract_docexprs=true)
     for (mod, ex) in modexs
         frame = JuliaInterpreter.prepare_thunk(mod, ex)
         while true

test/utils.jl

@@ -168,7 +168,7 @@ function run_test_by_eval(test, fullpath, nstmts)
         ts = Test.DefaultTestSet($test)
         Test.push_testset(ts)
         current_task().storage[:SOURCE_PATH] = $fullpath
-        modexs, _ = JuliaInterpreter.prepare_toplevel(JuliaTests, ex)
+        modexs, _ = JuliaInterpreter.split_expressions(JuliaTests, ex)
         stack = JuliaStackFrame[]
         for (i, modex) in enumerate(modexs)  # having the index can be useful for debugging
             nstmtsleft = $nstmts