Interpret module expressions at top level in eval()

This is a step toward an iteration interface for lowering which can
return a sequence of CodeInfo to be evaluated for top level and module
expressions.

This also restricts lowering of module expressions to be syntactically
at top level (ie, not inside a top level thunk), consistent with the
existing way that they're handled in eval.

Author: c42f

src/desugaring.jl

@@ -19,7 +19,7 @@ function DesugaringContext(ctx, expr_compat_mode::Bool)
     DesugaringContext(graph,
                       ctx.bindings,
                       ctx.scope_layers,
-                      first(ctx.scope_layers).mod,
+                      current_layer(ctx).mod,
                       expr_compat_mode)
 end
 
@@ -4270,39 +4270,6 @@ function expand_public(ctx, ex)
     ]
 end
 
-#-------------------------------------------------------------------------------
-# Expand module definitions
-
-function expand_module(ctx, ex::SyntaxTree)
-    modname_ex = ex[1]
-    @chk kind(modname_ex) == K"Identifier"
-    modname = modname_ex.name_val
-
-    std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG)
-
-    body = ex[2]
-    @chk kind(body) == K"block"
-
-    @ast ctx ex [K"block"
-        [K"assert"
-            "global_toplevel_only"::K"Symbol"
-            [K"inert" ex]
-        ]
-        [K"call"
-            eval_module           ::K"Value"
-            ctx.mod               ::K"Value"
-            modname               ::K"String"
-            std_defs              ::K"Bool"
-            ctx.expr_compat_mode  ::K"Bool"
-            [K"inert"(body)
-                [K"toplevel"
-                    children(body)...
-                ]
-            ]
-        ]
-    ]
-end
-
 #-------------------------------------------------------------------------------
 # Expand docstring-annotated expressions
 
@@ -4477,7 +4444,7 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing)
     elseif k == K"$"
         throw(LoweringError(ex, "`\$` expression outside string or quote block"))
     elseif k == K"module"
-        expand_module(ctx, ex)
+        throw(LoweringError(ex, "`module` is only allowed at top level"))
     elseif k == K"import" || k == K"using"
         expand_import_or_using(ctx, ex)
     elseif k == K"export" || k == K"public"

src/eval.jl

@@ -358,18 +358,63 @@ end
 
 #-------------------------------------------------------------------------------
 # Our version of eval takes our own data structures
-@fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree; expr_compat_mode::Bool=false)
+@fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree;
+        expr_compat_mode::Bool=false, macro_world=Base.get_world_counter())
+    graph = ensure_macro_attributes(syntax_graph(ex))
+    ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world)
+    _eval(ctx, ex)
+end
+
+function _eval_stmts(ctx, exs)
+    res = nothing
+    for ex in exs
+        res = _eval(ctx, ex)
+    end
+    res
+end
+
+function _eval_module_body(ctx, mod, ex)
+    new_layer = ScopeLayer(length(ctx.scope_layers)+1, mod,
+                           current_layer_id(ctx), false)
+    push!(ctx.scope_layers, new_layer)
+    push!(ctx.scope_layer_stack, new_layer.id)
+    stmts = kind(ex[2]) == K"block" ? ex[2][1:end] : ex[2:2]
+    _eval_stmts(ctx, stmts)
+    pop!(ctx.scope_layer_stack)
+end
+
+function _eval_module(ctx, ex)
+    # Here we just use `eval()` with an Expr to create a module.
+    # TODO: Refactor jl_eval_module_expr() in the runtime so that we can avoid
+    # eval.
+    std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG)
+    newmod_name = Symbol(ex[1].name_val)
+    Core.eval(current_layer(ctx).mod,
+              Expr(:module, std_defs, newmod_name,
+                   Expr(:block, Expr(:call,
+                                     newmod->_eval_module_body(ctx, newmod, ex),
+                                     newmod_name))))
+end
+
+function _eval(ctx, ex::SyntaxTree)
     k = kind(ex)
     if k == K"toplevel"
-        x = nothing
-        for e in children(ex)
-            x = eval(mod, e; expr_compat_mode)
+        _eval_stmts(ctx, children(ex))
+    elseif k == K"module"
+        _eval_module(ctx, ex)
+    else
+        ex1 = expand_forms_1(ctx, ex)
+        if kind(ex1) in KSet"toplevel module"
+            _eval(ctx, ex1)
+        else
+            ctx2, ex2 = expand_forms_2(ctx, ex1)
+            ctx3, ex3 = resolve_scopes(ctx2, ex2)
+            ctx4, ex4 = convert_closures(ctx3, ex3)
+            ctx5, ex5 = linearize_ir(ctx4, ex4)
+            thunk = to_lowered_expr(ex5)
+            Core.eval(current_layer(ctx).mod, thunk)
         end
-        return x
     end
-    linear_ir = lower(mod, ex; expr_compat_mode)
-    thunk = to_lowered_expr(linear_ir)
-    Core.eval(mod, thunk)
 end
 
 """

src/macro_expansion.jl

@@ -520,23 +520,28 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree)
     end
 end
 
+function ensure_macro_attributes(graph)
+    ensure_attributes(graph,
+                      var_id=IdTag,
+                      scope_layer=LayerId,
+                      __macro_ctx__=Nothing,
+                      meta=CompileHints)
+end
+
 @fzone "JL: macroexpand" function expand_forms_1(mod::Module, ex::SyntaxTree, expr_compat_mode::Bool, macro_world::UInt)
     if kind(ex) == K"local"
         # This error assumes we're expanding the body of a top level thunk but
         # we might want to make that more explicit in the pass system.
         throw(LoweringError(ex, "local declarations have no effect outside a scope"))
     end
-    graph = ensure_attributes(syntax_graph(ex),
-                              var_id=IdTag,
-                              scope_layer=LayerId,
-                              __macro_ctx__=Nothing,
-                              meta=CompileHints)
+    graph = ensure_macro_attributes(syntax_graph(ex))
     ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world)
     ex2 = expand_forms_1(ctx, reparent(ctx, ex))
     graph2 = delete_attributes(graph, :__macro_ctx__)
     # TODO: Returning the context with pass-specific mutable data is a bad way
     # to carry state into the next pass. We might fix this by attaching such
     # data to the graph itself as global attributes?
-    ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, LayerId[], expr_compat_mode, macro_world)
+    ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, ctx.scope_layer_stack,
+                                 expr_compat_mode, macro_world)
     return ctx2, reparent(ctx2, ex2)
 end

src/runtime.jl

@@ -207,35 +207,6 @@ end
 #--------------------------------------------------
 # Functions which create modules or mutate their bindings
 
-# Construct new bare module including only the "default names"
-#
-#     using Core
-#     const modname = modval
-#     public modname
-#
-# And run statments in the toplevel expression `body`
-function eval_module(parentmod::Module, modname::AbstractString, std_defs::Bool,
-                     expr_compat_mode::Bool, body::SyntaxTree)
-    # Here we just use `eval()` with an Expr.
-    # If we wanted to avoid this we'd need to reproduce a lot of machinery from
-    # jl_eval_module_expr()
-    #
-    # 1. Register / deparent toplevel modules
-    # 2. Set binding in parent module
-    # 3. Deal with replacing modules
-    #    * Warn if replacing
-    #    * Root old module being replaced
-    # 4. Run __init__
-    #    * Also run __init__ for any children after parent is defined
-    # mod = @ccall jl_new_module(Symbol(modname)::Symbol, parentmod::Module)::Any
-    # ...
-    name = Symbol(modname)
-    eval_module_body(mod) = eval(mod, body; expr_compat_mode=expr_compat_mode)
-    Core.eval(parentmod,
-        Expr(:module, std_defs, name,
-             Expr(:block, Expr(:call, eval_module_body, name))))
-end
-
 const _Base_has_eval_import = isdefined(Base, :_eval_import)
 
 function eval_import(imported::Bool, to::Module, from::Union{Expr, Nothing}, paths::Expr...)
@@ -357,11 +328,7 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
     graph = ensure_attributes(SyntaxGraph(), kind=Kind, syntax_flags=UInt16, source=SourceAttrType,
                               value=Any, name_val=String)
     # Attributes for macro expansion
-    graph = ensure_attributes(graph,
-                              var_id=IdTag,
-                              scope_layer=LayerId,
-                              __macro_ctx__=Nothing,
-                              meta=CompileHints,
+    graph = ensure_attributes(ensure_macro_attributes(graph),
                               # Additional attribute for resolve_scopes, for
                               # adding our custom lambda below
                               is_toplevel_thunk=Bool
@@ -393,7 +360,7 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
     ex1 = expand_forms_1(ctx1, reparent(ctx1, ex0))
     ctx1 = MacroExpansionContext(delete_attributes(graph, :__macro_ctx__),
                                  ctx1.bindings, ctx1.scope_layers,
-                                 LayerId[], false, macro_world)
+                                 ctx1.scope_layer_stack, false, macro_world)
     ex1 = reparent(ctx1, ex1)
 
     # Desugaring

test/demo.jl

@@ -893,7 +893,7 @@ ex = parsestmt(SyntaxTree, src, filename="foo.jl")
  ctx3, ex_scoped,
  ctx4, ex_converted,
  ctx5, ex_compiled,
- ex_expr, eval_result) = debug_lower(M, ex; verbose=true)
+ ex_expr, eval_result) = debug_lower(M, ex; verbose=true, do_eval=true)
 
 # Automatic test reduction
 # bad = reduce_any_failing_toplevel(JuliaLowering, joinpath(@__DIR__, "../src/desugaring.jl"))

test/misc_ir.jl

@@ -183,38 +183,18 @@ LoweringError:
 #       └─┘ ── Invalid named tuple element
 
 ########################################
-# Module lowering
-module Mod
-    body
-    stmts
-end
-#---------------------
-1   (call JuliaLowering.eval_module TestMod "Mod" true false (inert (toplevel body stmts)))
-2   (return %₁)
-
-########################################
-# Bare module lowering
-baremodule BareMod
-    body
-    stmts
-end
-#---------------------
-1   (call JuliaLowering.eval_module TestMod "BareMod" false false (inert (toplevel body stmts)))
-2   (return %₁)
-
-########################################
-# Error: Modules not allowed in local scope
-let
+# Error: Modules not allowed inside blocks
+begin
     module C
     end
 end
 #---------------------
 LoweringError:
-let
+begin
 #   ┌───────
     module C
     end
-#─────┘ ── module is only allowed in global scope
+#─────┘ ── `module` is only allowed at top level
 end
 
 ########################################
@@ -229,7 +209,7 @@ function f()
 #   ┌───────
     module C
     end
-#─────┘ ── module is only allowed in global scope
+#─────┘ ── `module` is only allowed at top level
 end
 
 ########################################

test/modules.jl

@@ -1,4 +1,4 @@
-@testset "JuliaLowering.jl" begin
+@testset "modules" begin
 
 test_mod = Module()
 
@@ -26,12 +26,29 @@ end
 @test !isdefined(B, :eval)
 @test !isdefined(B, :Base)
 
-# modules allowed in nested code in global scope
-@test typeof(JuliaLowering.include_string(test_mod, """
-begin
+# Module init order
+Amod = JuliaLowering.include_string(test_mod, """
+module A
+    init_order = []
+    __init__() = push!(init_order, "A")
+    module B
+        using ..A
+        __init__() = push!(A.init_order, "B")
+    end
     module C
+        using ..A
+        __init__() = push!(A.init_order, "C")
+        module D
+            using ...A
+            __init__() = push!(A.init_order, "D")
+        end
+        module E
+            using ...A
+            __init__() = push!(A.init_order, "E")
+        end
     end
 end
-""")) == Module
+""")
+@test Amod.init_order == ["B", "D", "E", "C", "A"]
 
 end