Merge pull request #72 from c42f/avi/generated-module

Some general clean ups and type stability improvements.

Author: aviatesk

src/eval.jl

@@ -120,8 +120,7 @@ end
 
 # Convert SyntaxTree to the CodeInfo+Expr data stuctures understood by the
 # Julia runtime
-function to_code_info(ex, mod, funcname, slots)
-    input_code = children(ex)
+function to_code_info(ex::SyntaxTree, slots::Vector{Slot})
     stmts = Any[]
 
     current_codelocs_stack = ir_debug_info_state(ex)
@@ -156,7 +155,7 @@ function to_code_info(ex, mod, funcname, slots)
 
     stmt_offset = length(stmts)
     for stmt in children(ex)
-        push!(stmts, _to_lowered_expr(mod, stmt, stmt_offset))
+        push!(stmts, _to_lowered_expr(stmt, stmt_offset))
         add_ir_debug_info!(current_codelocs_stack, stmt)
     end
 
@@ -223,11 +222,11 @@ function to_code_info(ex, mod, funcname, slots)
     )
 end
 
-@fzone "JL: to_lowered_expr" function to_lowered_expr(mod, ex)
-    _to_lowered_expr(mod, ex, 0)
+@fzone "JL: to_lowered_expr" function to_lowered_expr(ex::SyntaxTree)
+    _to_lowered_expr(ex, 0)
 end
 
-function _to_lowered_expr(mod, ex, stmt_offset)
+function _to_lowered_expr(ex::SyntaxTree, stmt_offset::Int)
     k = kind(ex)
     if is_literal(k)
         ex.value
@@ -263,15 +262,12 @@ function _to_lowered_expr(mod, ex, stmt_offset)
     elseif k == K"SSAValue"
         Core.SSAValue(ex.var_id + stmt_offset)
     elseif k == K"return"
-        Core.ReturnNode(_to_lowered_expr(mod, ex[1], stmt_offset))
+        Core.ReturnNode(_to_lowered_expr(ex[1], stmt_offset))
     elseif k == K"inert"
         e1 = ex[1]
         getmeta(ex, :as_Expr, false) ? QuoteNode(Expr(e1)) : e1
     elseif k == K"code_info"
-        funcname = ex.is_toplevel_thunk ?
-            "top-level scope" :
-            "none"              # FIXME
-        ir = to_code_info(ex[1], mod, funcname, ex.slots)
+        ir = to_code_info(ex[1], ex.slots)
         if ex.is_toplevel_thunk
             Expr(:thunk, ir)
         else
@@ -282,36 +278,40 @@ function _to_lowered_expr(mod, ex, stmt_offset)
     elseif k == K"goto"
         Core.GotoNode(ex[1].id + stmt_offset)
     elseif k == K"gotoifnot"
-        Core.GotoIfNot(_to_lowered_expr(mod, ex[1], stmt_offset), ex[2].id + stmt_offset)
+        Core.GotoIfNot(_to_lowered_expr(ex[1], stmt_offset), ex[2].id + stmt_offset)
     elseif k == K"enter"
         catch_idx = ex[1].id
         numchildren(ex) == 1 ?
             Core.EnterNode(catch_idx) :
-            Core.EnterNode(catch_idx, _to_lowered_expr(mod, ex[2], stmt_offset))
+            Core.EnterNode(catch_idx, _to_lowered_expr(ex[2], stmt_offset))
     elseif k == K"method"
-        cs = map(e->_to_lowered_expr(mod, e, stmt_offset), children(ex))
+        cs = map(e->_to_lowered_expr(e, stmt_offset), children(ex))
         # Ad-hoc unwrapping to satisfy `Expr(:method)` expectations
-        c1 = cs[1] isa QuoteNode ? cs[1].value : cs[1]
+        cs1 = cs[1]
+        c1 = cs1 isa QuoteNode ? cs1.value : cs1
         Expr(:method, c1, cs[2:end]...)
     elseif k == K"newvar"
-        Core.NewvarNode(_to_lowered_expr(mod, ex[1], stmt_offset))
+        Core.NewvarNode(_to_lowered_expr(ex[1], stmt_offset))
     elseif k == K"opaque_closure_method"
-        args = map(e->_to_lowered_expr(mod, e, stmt_offset), children(ex))
+        args = map(e->_to_lowered_expr(e, stmt_offset), children(ex))
         # opaque_closure_method has special non-evaluated semantics for the
         # `functionloc` line number node so we need to undo a level of quoting
-        @assert args[4] isa QuoteNode
-        args[4] = args[4].value
+        arg4 = args[4]
+        @assert arg4 isa QuoteNode
+        args[4] = arg4.value
         Expr(:opaque_closure_method, args...)
     elseif k == K"meta"
-        args = Any[_to_lowered_expr(mod, e, stmt_offset) for e in children(ex)]
+        args = Any[_to_lowered_expr(e, stmt_offset) for e in children(ex)]
         # Unpack K"Symbol" QuoteNode as `Expr(:meta)` requires an identifier here.
-        args[1] = args[1].value
+        arg1 = args[1]
+        @assert arg1 isa QuoteNode
+        args[1] = arg1.value
         Expr(:meta, args...)
     elseif k == K"static_eval"
         @assert numchildren(ex) == 1
-        _to_lowered_expr(mod, ex[1], stmt_offset)
+        _to_lowered_expr(ex[1], stmt_offset)
     elseif k == K"cfunction"
-        args = Any[_to_lowered_expr(mod, e, stmt_offset) for e in children(ex)]
+        args = Any[_to_lowered_expr(e, stmt_offset) for e in children(ex)]
         if kind(ex[2]) == K"static_eval"
             args[2] = QuoteNode(args[2])
         end
@@ -343,7 +343,11 @@ function _to_lowered_expr(mod, ex, stmt_offset)
         if isnothing(head)
             throw(LoweringError(ex, "Unhandled form for kind $k"))
         end
-        Expr(head, map(e->_to_lowered_expr(mod, e, stmt_offset), children(ex))...)
+        ret = Expr(head)
+        for e in children(ex)
+            push!(ret.args, _to_lowered_expr(e, stmt_offset))
+        end
+        return ret
     end
 end
 
@@ -359,7 +363,7 @@ end
         return x
     end
     linear_ir = lower(mod, ex; expr_compat_mode)
-    thunk = to_lowered_expr(mod, linear_ir)
+    thunk = to_lowered_expr(linear_ir)
     Core.eval(mod, thunk)
 end
 
@@ -400,4 +404,3 @@ function include_string(mod::Module, code::AbstractString, filename::AbstractStr
                         expr_compat_mode=false)
     eval(mod, parseall(SyntaxTree, code; filename=filename); expr_compat_mode)
 end
-

src/hooks.jl

@@ -22,7 +22,7 @@ function core_lowering_hook(@nospecialize(code), mod::Module,
         ctx3, st3 = resolve_scopes(  ctx2, st2)
         ctx4, st4 = convert_closures(ctx3, st3)
         ctx5, st5 = linearize_ir(    ctx4, st4)
-        ex = to_lowered_expr(mod, st5)
+        ex = to_lowered_expr(st5)
         return Core.svec(ex, st5, ctx5)
     catch exc
         @info("JuliaLowering threw given input:", code=code, st0=st0, file=file, line=line, mod=mod)

src/macro_expansion.jl

@@ -147,7 +147,7 @@ function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex::Syn
     ctx4, ex4 = convert_closures(ctx3, ex3)
     ctx5, ex5 = linearize_ir(ctx4, ex4)
     mod = current_layer(ctx).mod
-    expr_form = to_lowered_expr(mod, ex5)
+    expr_form = to_lowered_expr(ex5)
     try
         Core.eval(mod, expr_form)
     catch err
@@ -216,7 +216,7 @@ function expand_macro(ctx, ex)
     # We use a specific well defined world age for the next checks and macro
     # expansion invocations. This avoids inconsistencies if the latest world
     # age changes concurrently.
-    # 
+    #
     # TODO: Allow this to be passed in
     if hasmethod(macfunc, Tuple{typeof(mctx), typeof.(raw_args)...}; world=ctx.macro_world)
         macro_args = prepare_macro_args(ctx, mctx, raw_args)

src/precompile.jl

@@ -21,7 +21,7 @@ if Base.get_bool_env("JULIA_LOWERING_PRECOMPILE", true)
         JuliaSyntax.parse!(stream; rule=:all)
         st0 = JuliaSyntax.build_tree(SyntaxTree, stream; filename=@__FILE__)
         lwrst = lower(@__MODULE__, st0[1])
-        lwr = to_lowered_expr(@__MODULE__, lwrst)
+        lwr = to_lowered_expr(lwrst)
         @assert Meta.isexpr(lwr, :thunk) && only(lwr.args) isa Core.CodeInfo
     end
 end

src/runtime.jl

@@ -195,10 +195,10 @@ function eval_closure_type(mod::Module, closure_type_name::Symbol, field_names,
 end
 
 # Interpolate captured local variables into the CodeInfo for a global method
-function replace_captured_locals!(codeinfo, locals)
+function replace_captured_locals!(codeinfo::Core.CodeInfo, locals::Core.SimpleVector)
     for (i, ex) in enumerate(codeinfo.code)
         if Meta.isexpr(ex, :captured_local)
-            codeinfo.code[i] = locals[ex.args[1]]
+            codeinfo.code[i] = locals[ex.args[1]::Int]
         end
     end
     codeinfo
@@ -352,11 +352,13 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
                               is_toplevel_thunk=Bool
                               )
 
+    __module__ = source.module
+
     # Macro expansion. Looking at Core.GeneratedFunctionStub, it seems that
     # macros emitted by the generator are currently expanded in the latest
     # world, so do that for compatibility.
     macro_world = typemax(UInt)
-    ctx1 = MacroExpansionContext(graph, source.module, false, macro_world)
+    ctx1 = MacroExpansionContext(graph, __module__, false, macro_world)
 
     # Run code generator - this acts like a macro expander and like a macro
     # expander it gets a MacroContext.
@@ -397,7 +399,7 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
     # Rest of lowering
     ctx4, ex4 = convert_closures(ctx3, ex3)
     ctx5, ex5 = linearize_ir(ctx4, ex4)
-    ci = to_lowered_expr(mod, ex5)
+    ci = to_lowered_expr(ex5)
     @assert ci isa Core.CodeInfo
 
     # See GeneratedFunctionStub code in base/expr.jl

test/demo.jl

@@ -32,7 +32,7 @@ function formatsrc(ex; kws...)
     Text(JuliaSyntaxFormatter.formatsrc(ex; kws...))
 end
 
-function debug_lower(mod, ex; expr_compat_mode=false, verbose=false, do_eval=false)
+function debug_lower(mod::Module, ex::SyntaxTree; expr_compat_mode::Bool=false, verbose::Bool=false, do_eval::Bool=false)
     ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(mod, ex, expr_compat_mode, Base.get_world_counter())
 
     verbose && @info "Macro expanded" formatsrc(ex_macroexpand, color_by=:scope_layer)
@@ -49,7 +49,7 @@ function debug_lower(mod, ex; expr_compat_mode=false, verbose=false, do_eval=fal
     ctx5, ex_compiled = JuliaLowering.linearize_ir(ctx4, ex_converted)
     verbose && @info "Linear IR" formatsrc(ex_compiled, color_by=:var_id) Text(sprint(JuliaLowering.print_ir, ex_compiled))
 
-    ex_expr = JuliaLowering.to_lowered_expr(mod, ex_compiled)
+    ex_expr = JuliaLowering.to_lowered_expr(ex_compiled)
     verbose && @info "CodeInfo" ex_expr
 
     if do_eval
@@ -502,7 +502,7 @@ end
 
 src = """
 function f(y)
-    x = 
+    x =
     try
         try
             error("hi")
@@ -908,4 +908,3 @@ ex = parsestmt(SyntaxTree, src, filename="foo.jl")
 # for e in Meta.parseall(text).args
 #     Meta.lower(JuliaLowering, e)
 # end
-

test/repl_mode.jl

@@ -19,7 +19,7 @@ function is_incomplete(prompt_state)
     end
 end
 
-function eval_ish(mod, ex, do_eval, do_print_ir)
+function eval_ish(mod::Module, ex::SyntaxTree, do_eval::Bool, do_print_ir::Bool)
     k = kind(ex)
     if k == K"toplevel"
         x = nothing
@@ -34,7 +34,7 @@ function eval_ish(mod, ex, do_eval, do_print_ir)
     end
     if do_eval
         println(stdout, "#----------------------")
-        expr_form = JuliaLowering.to_lowered_expr(mod, linear_ir)
+        expr_form = JuliaLowering.to_lowered_expr(linear_ir)
         Base.eval(mod, expr_form)
     end
 end
@@ -79,4 +79,3 @@ function __init__()
 end
 
 end
-

test/utils.jl

@@ -262,9 +262,9 @@ function watch_ir_tests(dir, delay=0.5)
     end
 end
 
-function lower_str(mod, s)
+function lower_str(mod::Module, s::AbstractString)
     ex = parsestmt(JuliaLowering.SyntaxTree, s)
-    return JuliaLowering.to_lowered_expr(mod, JuliaLowering.lower(mod, ex))
+    return JuliaLowering.to_lowered_expr(JuliaLowering.lower(mod, ex))
 end
 
 # See Julia Base tests in "test/docs.jl"
@@ -356,13 +356,13 @@ end
 # Parse a file and lower the top level expression one child at a time, finding
 # any top level statement that fails lowering and producing a partially reduced
 # test case.
-function reduce_any_failing_toplevel(mod, filename; do_eval=false)
+function reduce_any_failing_toplevel(mod::Module, filename::AbstractString; do_eval::Bool=false)
     text = read(filename, String)
     ex0 = parseall(SyntaxTree, text; filename)
     for ex in children(ex0)
         try
             ex_compiled = JuliaLowering.lower(mod, ex)
-            ex_expr = JuliaLowering.to_lowered_expr(mod, ex_compiled)
+            ex_expr = JuliaLowering.to_lowered_expr(ex_compiled)
             if do_eval
                 Base.eval(mod, ex_expr)
             end
@@ -373,7 +373,7 @@ function reduce_any_failing_toplevel(mod, filename; do_eval=false)
             end
             (reduced,was_reduced) = block_reduction(e->throws_lowering_exc(mod,e), ex)
             if !was_reduced
-                @info "No reduction possible" 
+                @info "No reduction possible"
                 return ex
             else
                 @info "Reduced code" reduced
@@ -383,4 +383,3 @@ function reduce_any_failing_toplevel(mod, filename; do_eval=false)
     end
     nothing
 end
-