Compat behaviours: typeof quoted expressions, macro def signatures (#44)

Co-authored-by: Shuhei Kadowaki <[email protected]>
Co-authored-by: Claire Foster <[email protected]>

Author: 3 people

src/desugaring.jl

@@ -5,17 +5,22 @@ struct DesugaringContext{GraphType} <: AbstractLoweringContext
     bindings::Bindings
     scope_layers::Vector{ScopeLayer}
     mod::Module
+    expr_compat_mode::Bool
 end
 
-function DesugaringContext(ctx)
+function DesugaringContext(ctx, expr_compat_mode::Bool)
     graph = ensure_attributes(syntax_graph(ctx),
                               kind=Kind, syntax_flags=UInt16,
                               source=SourceAttrType,
                               value=Any, name_val=String,
                               scope_type=Symbol, # :hard or :soft
                               var_id=IdTag,
                               is_toplevel_thunk=Bool)
-    DesugaringContext(graph, ctx.bindings, ctx.scope_layers, first(ctx.scope_layers).mod)
+    DesugaringContext(graph,
+                      ctx.bindings,
+                      ctx.scope_layers,
+                      first(ctx.scope_layers).mod,
+                      expr_compat_mode)
 end
 
 #-------------------------------------------------------------------------------
@@ -3246,20 +3251,39 @@ function expand_macro_def(ctx, ex)
     name = sig[1]
     args = remove_empty_parameters(children(sig))
     @chk kind(args[end]) != K"parameters" (args[end], "macros cannot accept keyword arguments")
-    ret = @ast ctx ex [K"function"
-        [K"call"(sig)
-            _make_macro_name(ctx, name)
-            [K"::"
-                adopt_scope(@ast(ctx, sig, "__context__"::K"Identifier"),
-                            kind(name) == K"." ? name[1] : name)
-                MacroContext::K"Value"
+    scope_ref = kind(name) == K"." ? name[1] : name
+    if ctx.expr_compat_mode
+        @ast ctx ex [K"function"
+            [K"call"(sig)
+                _make_macro_name(ctx, name)
+                [K"::"
+                    # TODO: should we be adopting the scope of the K"macro" expression itself?
+                    adopt_scope(@ast(ctx, sig, "__source__"::K"Identifier"), scope_ref)
+                    LineNumberNode::K"Value"
+                ]
+                [K"::"
+                    adopt_scope(@ast(ctx, sig, "__module__"::K"Identifier"), scope_ref)
+                    Module::K"Value"
+                ]
+                map(e->_apply_nospecialize(ctx, e), args[2:end])...
             ]
-            # flisp: We don't mark these @nospecialize because all arguments to
-            # new macros will be of type SyntaxTree
-            args[2:end]...
+            ex[2]
         ]
-        ex[2]
-    ]
+    else
+        @ast ctx ex [K"function"
+            [K"call"(sig)
+                _make_macro_name(ctx, name)
+                [K"::"
+                    adopt_scope(@ast(ctx, sig, "__context__"::K"Identifier"), scope_ref)
+                    MacroContext::K"Value"
+                ]
+                # flisp: We don't mark these @nospecialize because all arguments to
+                # new macros will be of type SyntaxTree
+                args[2:end]...
+            ]
+            ex[2]
+        ]
+    end
 end
 
 #-------------------------------------------------------------------------------
@@ -4284,9 +4308,10 @@ function expand_module(ctx, ex::SyntaxTree)
             [K"inert" ex]
         ]
         [K"call"
-            eval_module ::K"Value"
-            ctx.mod     ::K"Value"
-            modname     ::K"String"
+            eval_module           ::K"Value"
+            ctx.mod               ::K"Value"
+            modname               ::K"String"
+            ctx.expr_compat_mode  ::K"Bool"
             [K"inert"(body)
                 [K"toplevel"
                     std_defs
@@ -4477,14 +4502,21 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing)
     elseif k == K"toplevel"
         # The toplevel form can't be lowered here - it needs to just be quoted
         # and passed through to a call to eval.
-        @ast ctx ex [K"block"
+        ex2 = @ast ctx ex [K"block"
             [K"assert" "toplevel_only"::K"Symbol" [K"inert" ex]]
             [K"call"
-                eval          ::K"Value"
-                ctx.mod       ::K"Value"
+                eval                  ::K"Value"
+                ctx.mod               ::K"Value"
                 [K"inert" ex]
+                [K"parameters"
+                    [K"="
+                        "expr_compat_mode"::K"Identifier"
+                        ctx.expr_compat_mode::K"Bool"
+                    ]
+                ]
             ]
         ]
+        expand_forms_2(ctx, ex2)
     elseif k == K"vect"
         check_no_parameters(ex, "unexpected semicolon in array expression")
         expand_array(ctx, ex, "vect")
@@ -4546,7 +4578,7 @@ function expand_forms_2(ctx::StatementListCtx, args...)
 end
 
 function expand_forms_2(ctx::MacroExpansionContext, ex::SyntaxTree)
-    ctx1 = DesugaringContext(ctx)
+    ctx1 = DesugaringContext(ctx, ctx.expr_compat_mode)
     ex1 = expand_forms_2(ctx1, reparent(ctx1, ex))
     ctx1, ex1
 end

src/eval.jl

@@ -1,14 +1,14 @@
-function lower(mod::Module, ex0)
-    ctx1, ex1 = expand_forms_1(  mod,  ex0)
+function lower(mod::Module, ex0, expr_compat_mode=false)
+    ctx1, ex1 = expand_forms_1(  mod,  ex0, expr_compat_mode)
     ctx2, ex2 = expand_forms_2(  ctx1, ex1)
     ctx3, ex3 = resolve_scopes(  ctx2, ex2)
     ctx4, ex4 = convert_closures(ctx3, ex3)
     ctx5, ex5 = linearize_ir(    ctx4, ex4)
     ex5
 end
 
-function macroexpand(mod::Module, ex)
-    ctx1, ex1 = expand_forms_1(mod, ex)
+function macroexpand(mod::Module, ex, expr_compat_mode=false)
+    ctx1, ex1 = expand_forms_1(mod, ex, expr_compat_mode)
     ex1
 end
 
@@ -344,16 +344,16 @@ end
 
 #-------------------------------------------------------------------------------
 # Our version of eval takes our own data structures
-function Core.eval(mod::Module, ex::SyntaxTree)
+function Core.eval(mod::Module, ex::SyntaxTree; expr_compat_mode::Bool=false)
     k = kind(ex)
     if k == K"toplevel"
         x = nothing
         for e in children(ex)
-            x = eval(mod, e)
+            x = eval(mod, e; expr_compat_mode)
         end
         return x
     end
-    linear_ir = lower(mod, ex)
+    linear_ir = lower(mod, ex, expr_compat_mode)
     expr_form = to_lowered_expr(mod, linear_ir)
     eval(mod, expr_form)
 end

src/hooks.jl

@@ -17,7 +17,7 @@ function core_lowering_hook(@nospecialize(code), mod::Module,
 
     st0 = code isa Expr ? expr_to_syntaxtree(code, LineNumberNode(line, file)) : code
     try
-        ctx1, st1 = expand_forms_1(  mod,  st0)
+        ctx1, st1 = expand_forms_1(  mod,  st0, true)
         ctx2, st2 = expand_forms_2(  ctx1, st1)
         ctx3, st3 = resolve_scopes(  ctx2, st2)
         ctx4, st4 = convert_closures(ctx3, st3)

src/macro_expansion.jl

@@ -20,11 +20,12 @@ struct MacroExpansionContext{GraphType} <: AbstractLoweringContext
     bindings::Bindings
     scope_layers::Vector{ScopeLayer}
     scope_layer_stack::Vector{LayerId}
+    expr_compat_mode::Bool
 end
 
-function MacroExpansionContext(graph::SyntaxGraph, mod::Module)
+function MacroExpansionContext(graph::SyntaxGraph, mod::Module, expr_compat_mode::Bool)
     layers = ScopeLayer[ScopeLayer(1, mod, 0, false)]
-    MacroExpansionContext(graph, Bindings(), layers, LayerId[length(layers)])
+    MacroExpansionContext(graph, Bindings(), layers, LayerId[length(layers)], expr_compat_mode)
 end
 
 current_layer(ctx::MacroExpansionContext) = ctx.scope_layers[last(ctx.scope_layer_stack)]
@@ -67,6 +68,7 @@ function expand_quote(ctx, ex)
     # (ex, @HERE) ?
     @ast ctx ex [K"call"
         interpolate_ast::K"Value"
+        (ctx.expr_compat_mode ? Expr : SyntaxTree)::K"Value"
         [K"inert" ex]
         unquoted...
     ]
@@ -447,18 +449,18 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree)
     end
 end
 
-function expand_forms_1(mod::Module, ex::SyntaxTree)
+function expand_forms_1(mod::Module, ex::SyntaxTree, expr_compat_mode::Bool)
     graph = ensure_attributes(syntax_graph(ex),
                               var_id=IdTag,
                               scope_layer=LayerId,
                               __macro_ctx__=Nothing,
                               meta=CompileHints)
-    ctx = MacroExpansionContext(graph, mod)
+    ctx = MacroExpansionContext(graph, mod, expr_compat_mode)
     ex2 = expand_forms_1(ctx, reparent(ctx, ex))
     graph2 = delete_attributes(graph, :__macro_ctx__)
     # TODO: Returning the context with pass-specific mutable data is a bad way
     # to carry state into the next pass. We might fix this by attaching such
     # data to the graph itself as global attributes?
-    ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, LayerId[])
+    ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, LayerId[], expr_compat_mode)
     return ctx2, reparent(ctx2, ex2)
 end

src/runtime.jl

@@ -71,7 +71,7 @@ function _interpolate_ast(ctx::InterpolationContext, ex, depth)
     makenode(ctx, ex, head(ex), expanded_children)
 end
 
-function interpolate_ast(ex, values...)
+function interpolate_ast(::Type{SyntaxTree}, ex, values...)
     # Construct graph for interpolation context. We inherit this from the macro
     # context where possible by detecting it using __macro_ctx__. This feels
     # hacky though.
@@ -108,6 +108,14 @@ function interpolate_ast(ex, values...)
     end
 end
 
+function interpolate_ast(::Type{Expr}, ex, values...)
+    # TODO: Adjust `_interpolated_value` to ensure that incoming `Expr` data
+    # structures are treated as AST in Expr compat mode, rather than `K"Value"`?
+    # Or convert `ex` to `Expr` early during lowering and implement
+    # `interpolate_ast` for `Expr`?
+    Expr(interpolate_ast(SyntaxTree, ex, values...))
+end
+
 #--------------------------------------------------
 # Functions called by closure conversion
 function eval_closure_type(mod::Module, closure_type_name::Symbol, field_names, field_is_box)
@@ -154,7 +162,7 @@ end
 #     public modname
 #
 # And run statments in the toplevel expression `body`
-function eval_module(parentmod, modname, body)
+function eval_module(parentmod, modname, expr_compat_mode, body)
     # Here we just use `eval()` with an Expr.
     # If we wanted to avoid this we'd need to reproduce a lot of machinery from
     # jl_eval_module_expr()
@@ -171,7 +179,7 @@ function eval_module(parentmod, modname, body)
     name = Symbol(modname)
     eval(parentmod, :(
         baremodule $name
-            $eval($name, $body)
+            $eval($name, $body; expr_compat_mode=$expr_compat_mode)
         end
     ))
 end
@@ -296,7 +304,7 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
                               )
 
     # Macro expansion
-    ctx1 = MacroExpansionContext(graph, mod)
+    ctx1 = MacroExpansionContext(graph, mod, false)
 
     # Run code generator - this acts like a macro expander and like a macro
     # expander it gets a MacroContext.
@@ -315,7 +323,7 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
     # Expand any macros emitted by the generator
     ex1 = expand_forms_1(ctx1, reparent(ctx1, ex0))
     ctx1 = MacroExpansionContext(delete_attributes(graph, :__macro_ctx__),
-                                 ctx1.bindings, ctx1.scope_layers, LayerId[])
+                                 ctx1.bindings, ctx1.scope_layers, LayerId[], false)
     ex1 = reparent(ctx1, ex1)
 
     # Desugaring

src/syntax_graph.jl

@@ -653,9 +653,10 @@ macro SyntaxTree(ex_old)
     ex = _find_SyntaxTree_macro(full_ex, __source__.line)
     # 4. Do the first step of JuliaLowering's syntax lowering to get
     # syntax interpolations to work
-    _, ex1 = expand_forms_1(__module__, ex)
+    _, ex1 = expand_forms_1(__module__, ex, false)
     @assert kind(ex1) == K"call" && ex1[1].value == interpolate_ast
-    Expr(:call, :interpolate_ast, ex1[2][1], map(e->_scope_layer_1_to_esc!(Expr(e)), ex1[3:end])...)
+    Expr(:call, :interpolate_ast, SyntaxTree, ex1[3][1],
+         map(e->_scope_layer_1_to_esc!(Expr(e)), ex1[4:end])...)
 end
 
 #-------------------------------------------------------------------------------

test/arrays_ir.jl

@@ -496,3 +496,4 @@ a[] = rhs
 2   TestMod.a
 3   (call top.setindex! %₂ %₁)
 4   (return %₁)
+

test/demo.jl

@@ -35,7 +35,7 @@ end
 # Currently broken - need to push info back onto src
 # function annotate_scopes(mod, ex)
 #     ex = ensure_attributes(ex, var_id=Int)
-#     ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(mod, ex)
+#     ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(mod, ex, false)
 #     ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand)
 #     ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar)
 #     ex
@@ -859,7 +859,7 @@ ex = ensure_attributes(ex, var_id=Int)
 
 in_mod = M
 # in_mod=Main
-ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex)
+ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex, false)
 @info "Macro expanded" formatsrc(ex_macroexpand, color_by=:scope_layer)
 #@info "Macro expanded" formatsrc(ex_macroexpand, color_by=e->JuliaLowering.flattened_provenance(e)[1:end-1])
 

test/functions_ir.jl

@@ -1565,9 +1565,9 @@ end
 10  (call core.svec %₇ %₈ %₉)
 11  --- method core.nothing %₁₀
     slots: [slot₁/#self#(!read) slot₂/__context__(!read) slot₃/#self#(!read) slot₄/x(nospecialize,!read) slot₅/y(nospecialize,!read)]
-    1   (call JuliaLowering.interpolate_ast (inert (block (= maybe_gen_stuff (call some_gen_stuff x y)))))
+    1   (call JuliaLowering.interpolate_ast SyntaxTree (inert (block (= maybe_gen_stuff (call some_gen_stuff x y)))))
     2   (call core.tuple %₁)
-    3   (call JuliaLowering.interpolate_ast (inert (block (block (= nongen_stuff (call bothgen x y)) ($ (block (call JuliaLowering.interpolate_ast (inert (block (= maybe_gen_stuff (call some_gen_stuff x y))))))) (tuple-p nongen_stuff maybe_gen_stuff)))) %₂)
+    3   (call JuliaLowering.interpolate_ast SyntaxTree (inert (block (block (= nongen_stuff (call bothgen x y)) ($ (block (call JuliaLowering.interpolate_ast SyntaxTree (inert (block (= maybe_gen_stuff (call some_gen_stuff x y))))))) (tuple-p nongen_stuff maybe_gen_stuff)))) %₂)
     4   (return %₃)
 12  latestworld
 13  TestMod.f_partially_generated

test/generators_ir.jl

@@ -171,7 +171,7 @@ LoweringError:
 9   (call core.svec %₆ %₇ %₈)
 10  --- method core.nothing %₉
     slots: [slot₁/#self#(!read) slot₂/_(!read)]
-    1   (call JuliaLowering.interpolate_ast (inert (return x)))
+    1   (call JuliaLowering.interpolate_ast SyntaxTree (inert (return x)))
     2   (return %₁)
 11  latestworld
 12  TestMod.#->##6