diff --git a/README.md b/README.md index 83b10ebb..0918fb0f 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,65 @@ discussed in Adams' paper: TODO: Write more here... + +### Compatibility with `Expr` macros + +In order to have compatibility with old-style macros which expect an `Expr`-based +data structure as input, we convert `SyntaxTree` to `Expr`, call the old-style +macro, then convert `SyntaxTree` back to `Expr` and continue with the expansion +process. This involves some loss of provenance precision but allows full +interoperability in the package ecosystem without a need to make breaking +changes. + +Let's look at an example. Suppose a manually escaped old-style macro +`@oldstyle` is implemented as + +```julia +macro oldstyle(a, b) + quote + x = "x in @oldstyle" + @newstyle $(esc(a)) $(esc(b)) x + end +end +``` + +along with two correctly escaped new-style macros: + +```julia +macro call_oldstyle_macro(y) + quote + x = "x in call_oldstyle_macro" + @oldstyle $y x + end +end + +macro newstyle(x, y, z) + quote + x = "x in @newstyle" + ($x, $y, $z, x) + end +end +``` + +Then want some code like the following to "just work" with respect to hygiene + +```julia +let + x = "x in outer ctx" + @call_oldstyle_macro x +end +``` + +When calling `@oldstyle`, we must convert `SyntaxTree` into `Expr`, but we need +to preserve the scope layer of the `x` from the outer context as it is passed +into `@oldstyle` as a macro argument. To do this, we use `Expr(:scope_layer, +:x, outer_layer_id)`. (In the old system, this would be `Expr(:escape, :x)` +instead, presuming that `@call_oldstyle_macro` was implemented using `esc()`.) + +When receiving output from old style macro invocations, we preserve the escape +handling of the existing system for any symbols which aren't tagged with a +scope layer. + ## Pass 2: Syntax desugaring This pass recursively converts many special surface syntax forms to a smaller diff --git a/src/ast.jl b/src/ast.jl index bbe59015..46aa82e5 100644 --- a/src/ast.jl +++ b/src/ast.jl @@ -662,8 +662,8 @@ function to_symbol(ctx, ex) @ast ctx ex ex=>K"Symbol" end -function new_scope_layer(ctx, mod_ref::Module=ctx.mod; is_macro_expansion=true) - new_layer = ScopeLayer(length(ctx.scope_layers)+1, ctx.mod, is_macro_expansion) +function new_scope_layer(ctx, mod_ref::Module=ctx.mod) + new_layer = ScopeLayer(length(ctx.scope_layers)+1, ctx.mod, 0, false) push!(ctx.scope_layers, new_layer) new_layer.id end diff --git a/src/compat.jl b/src/compat.jl index 76dad46c..f12f0e7b 100644 --- a/src/compat.jl +++ b/src/compat.jl @@ -24,7 +24,8 @@ function expr_to_syntaxtree(@nospecialize(e), lnn::Union{LineNumberNode, Nothing SyntaxGraph(), kind=Kind, syntax_flags=UInt16, source=SourceAttrType, var_id=Int, value=Any, - name_val=String, is_toplevel_thunk=Bool) + name_val=String, is_toplevel_thunk=Bool, + scope_layer=LayerId) expr_to_syntaxtree(graph, e, lnn) end @@ -423,6 +424,13 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA @assert nargs === 1 child_exprs[1] = Expr(:quoted_symbol, e.args[1]) end + elseif e.head === :scope_layer + @assert nargs === 2 + @assert e.args[1] isa Symbol + @assert e.args[2] isa LayerId + st_id, src = _insert_convert_expr(e.args[1], graph, src) + setattr!(graph, st_id, scope_layer=e.args[2]) + return st_id, src elseif e.head === :symbolicgoto || e.head === :symboliclabel @assert nargs === 1 st_k = e.head === :symbolicgoto ? K"symbolic_label" : K"symbolic_goto" diff --git a/src/desugaring.jl b/src/desugaring.jl index 4a4d4e29..573b712c 100644 --- a/src/desugaring.jl +++ b/src/desugaring.jl @@ -15,7 +15,7 @@ function DesugaringContext(ctx) scope_type=Symbol, # :hard or :soft var_id=IdTag, is_toplevel_thunk=Bool) - DesugaringContext(graph, ctx.bindings, ctx.scope_layers, ctx.current_layer.mod) + DesugaringContext(graph, ctx.bindings, ctx.scope_layers, first(ctx.scope_layers).mod) end #------------------------------------------------------------------------------- @@ -2555,7 +2555,7 @@ function keyword_function_defs(ctx, srcref, callex_srcref, name_str, typevar_nam end # TODO: Is the layer correct here? Which module should be the parent module # of this body function? - layer = new_scope_layer(ctx; is_macro_expansion=false) + layer = new_scope_layer(ctx) body_func_name = adopt_scope(@ast(ctx, callex_srcref, mangled_name::K"Identifier"), layer) kwcall_arg_names = SyntaxList(ctx) diff --git a/src/kinds.jl b/src/kinds.jl index 7aa17aa4..4cc0afe8 100644 --- a/src/kinds.jl +++ b/src/kinds.jl @@ -48,8 +48,11 @@ function _register_kinds() # Internal initializer for struct types, for inner constructors/functions "new" "splatnew" - # For expr-macro compatibility; gone after expansion + # Used for converting `esc()`'d expressions arising from old macro + # invocations during macro expansion (gone after macro expansion) "escape" + # Used for converting the old-style macro hygienic-scope form (gone + # after macro expansion) "hygienic_scope" # An expression which will eventually be evaluated "statically" in # the context of a CodeInfo and thus allows access only to globals diff --git a/src/macro_expansion.jl b/src/macro_expansion.jl index f02f2fa8..e321af03 100644 --- a/src/macro_expansion.jl +++ b/src/macro_expansion.jl @@ -11,6 +11,7 @@ generates a new layer. struct ScopeLayer id::LayerId mod::Module + parent_layer::LayerId # Index of parent layer in a macro expansion. Equal to 0 for no parent is_macro_expansion::Bool # FIXME end @@ -18,9 +19,17 @@ struct MacroExpansionContext{GraphType} <: AbstractLoweringContext graph::GraphType bindings::Bindings scope_layers::Vector{ScopeLayer} - current_layer::ScopeLayer + scope_layer_stack::Vector{LayerId} end +function MacroExpansionContext(graph::SyntaxGraph, mod::Module) + layers = ScopeLayer[ScopeLayer(1, mod, 0, false)] + MacroExpansionContext(graph, Bindings(), layers, LayerId[length(layers)]) +end + +current_layer(ctx::MacroExpansionContext) = ctx.scope_layers[last(ctx.scope_layer_stack)] +current_layer_id(ctx::MacroExpansionContext) = last(ctx.scope_layer_stack) + #-------------------------------------------------- # Expansion of quoted expressions function collect_unquoted!(ctx, unquoted, ex, depth) @@ -130,7 +139,7 @@ function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex::Syn ctx3, ex3 = resolve_scopes(ctx2, ex2) ctx4, ex4 = convert_closures(ctx3, ex3) ctx5, ex5 = linearize_ir(ctx4, ex4) - mod = ctx.current_layer.mod + mod = current_layer(ctx).mod expr_form = to_lowered_expr(mod, ex5) try eval(mod, expr_form) @@ -139,54 +148,136 @@ function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex::Syn end end -function expand_macro(ctx::MacroExpansionContext, ex::SyntaxTree) +# Record scope layer information for symbols passed to a macro by setting +# scope_layer for each expression and also processing any K"escape" arising +# from previous expansion of old-style macros. +# +# See also set_scope_layer() +function set_macro_arg_hygiene(ctx, ex, layer_ids, layer_idx) + k = kind(ex) + scope_layer = get(ex, :scope_layer, layer_ids[layer_idx]) + if k == K"module" || k == K"toplevel" || k == K"inert" + makenode(ctx, ex, ex, children(ex); + scope_layer=scope_layer) + elseif k == K"." + makenode(ctx, ex, ex, set_macro_arg_hygiene(ctx, ex[1], layer_ids, layer_idx), ex[2], + scope_layer=scope_layer) + elseif !is_leaf(ex) + inner_layer_idx = layer_idx + if k == K"escape" + inner_layer_idx = layer_idx - 1 + if inner_layer_idx < 1 + # If we encounter too many escape nodes, there's probably been + # an error in the previous macro expansion. + # todo: The error here isn't precise about that - maybe we + # should record that macro call expression with the scope layer + # if we want to report the error against the macro call? + throw(MacroExpansionError(ex, "`escape` node in outer context")) + end + end + mapchildren(e->set_macro_arg_hygiene(ctx, e, layer_ids, inner_layer_idx), + ctx, ex; scope_layer=scope_layer) + else + makeleaf(ctx, ex, ex; scope_layer=scope_layer) + end +end + +function prepare_macro_args(ctx, mctx, raw_args) + macro_args = Any[mctx] + for arg in raw_args + # Add hygiene information to be carried along with macro arguments. + # + # Macro call arguments may be either + # * Unprocessed by the macro expansion pass + # * Previously processed, but spliced into a further macro call emitted by + # a macro expansion. + # In either case, we need to set scope layers before passing the + # arguments to the macro call. + push!(macro_args, set_macro_arg_hygiene(ctx, arg, ctx.scope_layer_stack, + length(ctx.scope_layer_stack))) + end + return macro_args +end + +function expand_macro(ctx, ex) @assert kind(ex) == K"macrocall" macname = ex[1] - mctx = MacroContext(ctx.graph, ex, ctx.current_layer) + mctx = MacroContext(ctx.graph, ex, current_layer(ctx)) macfunc = eval_macro_name(ctx, mctx, macname) - # Macro call arguments may be either - # * Unprocessed by the macro expansion pass - # * Previously processed, but spliced into a further macro call emitted by - # a macro expansion. - # In either case, we need to set any unset scope layers before passing the - # arguments to the macro call. - macro_args = Any[mctx] - for i in 2:numchildren(ex) - push!(macro_args, set_scope_layer(ctx, ex[i], ctx.current_layer.id, false)) - end - macro_invocation_world = Base.get_world_counter() - expanded = try - # TODO: Allow invoking old-style macros for compat - invokelatest(macfunc, macro_args...) - catch exc - if exc isa MacroExpansionError - # Add context to the error. - newexc = MacroExpansionError(mctx, exc.ex, exc.msg, exc.position, exc.err) + raw_args = ex[2:end] + # We use a specific well defined world age for the next checks and macro + # expansion invocations. This avoids inconsistencies if the latest world + # age changes concurrently. + # + # TODO: Allow this to be passed in + macro_world = Base.get_world_counter() + if hasmethod(macfunc, Tuple{typeof(mctx), typeof.(raw_args)...}; world=macro_world) + macro_args = prepare_macro_args(ctx, mctx, raw_args) + expanded = try + Base.invoke_in_world(macro_world, macfunc, macro_args...) + catch exc + newexc = exc isa MacroExpansionError ? + MacroExpansionError(mctx, exc.ex, exc.msg, exc.position, exc.err) : + MacroExpansionError(mctx, ex, "Error expanding macro", :all, exc) + # TODO: We can delete this rethrow when we move to AST-based error propagation. + rethrow(newexc) + end + if expanded isa SyntaxTree + if !is_compatible_graph(ctx, expanded) + # If the macro has produced syntax outside the macro context, + # copy it over. TODO: Do we expect this always to happen? What + # is the API for access to the macro expansion context? + expanded = copy_ast(ctx, expanded) + end else - newexc = MacroExpansionError(mctx, ex, "Error expanding macro", :all, exc) + expanded = @ast ctx ex expanded::K"Value" + end + else + # Compat: attempt to invoke an old-style macro if there's no applicable + # method for new-style macro arguments. + macro_loc = source_location(LineNumberNode, ex) + macro_args = Any[macro_loc, current_layer(ctx).mod] + for arg in raw_args + # For hygiene in old-style macros, we omit any additional scope + # layer information from macro arguments. Old-style macros will + # handle that using manual escaping in the macro itself. + # + # Note that there's one slight incompatibility here for identifiers + # interpolated into the `raw_args` from outer macro expansions of + # new-style macros which call old-style macros. Instead of seeing + # `Expr(:escape)` in such situations, old-style macros will now see + # `Expr(:scope_layer)` inside `macro_args`. + push!(macro_args, Expr(arg)) end - # TODO: We can delete this rethrow when we move to AST-based error propagation. - rethrow(newexc) + expanded = try + Base.invoke_in_world(macro_world, macfunc, macro_args...) + catch exc + if exc isa MethodError && exc.f === macfunc + if !isempty(methods_in_world(macfunc, Tuple{typeof(mctx), Vararg{Any}}, macro_world)) + # If the macro has at least some methods implemented in the + # new style, assume the user meant to call one of those + # rather than any old-style macro methods which might exist + exc = MethodError(macfunc, (prepare_macro_args(ctx, mctx, raw_args)..., ), macro_world) + end + end + rethrow(MacroExpansionError(mctx, ex, "Error expanding macro", :all, exc)) + end + expanded = expr_to_syntaxtree(ctx, expanded, macro_loc) end - if expanded isa SyntaxTree - if !is_compatible_graph(ctx, expanded) - # If the macro has produced syntax outside the macro context, copy it over. - # TODO: Do we expect this always to happen? What is the API for access - # to the macro expansion context? - expanded = copy_ast(ctx, expanded) - end + if kind(expanded) != K"Value" expanded = append_sourceref(ctx, expanded, ex) # Module scope for the returned AST is the module where this particular # method was defined (may be different from `parentmodule(macfunc)`) - mod_for_ast = lookup_method_instance(macfunc, macro_args, macro_invocation_world).def.module - new_layer = ScopeLayer(length(ctx.scope_layers)+1, mod_for_ast, true) + mod_for_ast = lookup_method_instance(macfunc, macro_args, + macro_world).def.module + new_layer = ScopeLayer(length(ctx.scope_layers)+1, mod_for_ast, + current_layer_id(ctx), true) push!(ctx.scope_layers, new_layer) - inner_ctx = MacroExpansionContext(ctx.graph, ctx.bindings, ctx.scope_layers, new_layer) - expanded = expand_forms_1(inner_ctx, expanded) - else - expanded = @ast ctx ex expanded::K"Value" + push!(ctx.scope_layer_stack, new_layer.id) + expanded = expand_forms_1(ctx, expanded) + pop!(ctx.scope_layer_stack) end return expanded end @@ -231,18 +322,37 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree) # turned into normal bindings (eg, assigned to) @ast ctx ex name_str::K"core" else - layerid = get(ex, :scope_layer, ctx.current_layer.id) + layerid = get(ex, :scope_layer, current_layer_id(ctx)) makeleaf(ctx, ex, ex, kind=K"Identifier", scope_layer=layerid) end elseif k == K"Identifier" || k == K"MacroName" || k == K"StringMacroName" - layerid = get(ex, :scope_layer, ctx.current_layer.id) + layerid = get(ex, :scope_layer, current_layer_id(ctx)) makeleaf(ctx, ex, ex, kind=K"Identifier", scope_layer=layerid) elseif k == K"var" || k == K"char" || k == K"parens" # Strip "container" nodes @chk numchildren(ex) == 1 expand_forms_1(ctx, ex[1]) + elseif k == K"escape" + # For processing of old-style macros + @chk numchildren(ex) >= 1 "`escape` requires an argument" + if length(ctx.scope_layer_stack) === 1 + throw(MacroExpansionError(ex, "`escape` node in outer context")) + end + top_layer = pop!(ctx.scope_layer_stack) + escaped_ex = expand_forms_1(ctx, ex[1]) + push!(ctx.scope_layer_stack, top_layer) + escaped_ex + elseif k == K"hygienic_scope" + @chk numchildren(ex) >= 2 && ex[2].value isa Module (ex,"`hygienic_scope` requires an AST and a module") + new_layer = ScopeLayer(length(ctx.scope_layers)+1, ex[2].value, + current_layer_id(ctx), true) + push!(ctx.scope_layers, new_layer) + push!(ctx.scope_layer_stack, new_layer.id) + hyg_ex = expand_forms_1(ctx, ex[1]) + pop!(ctx.scope_layer_stack) + hyg_ex elseif k == K"juxtapose" - layerid = get(ex, :scope_layer, ctx.current_layer.id) + layerid = get(ex, :scope_layer, current_layer_id(ctx)) @chk numchildren(ex) == 2 @ast ctx ex [K"call" "*"::K"Identifier"(scope_layer=layerid) @@ -330,7 +440,7 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree) elseif k == K"<:" || k == K">:" || k == K"-->" # TODO: Should every form get layerid systematically? Or only the ones # which expand_forms_2 needs? - layerid = get(ex, :scope_layer, ctx.current_layer.id) + layerid = get(ex, :scope_layer, current_layer_id(ctx)) mapchildren(e->expand_forms_1(ctx,e), ctx, ex; scope_layer=layerid) else mapchildren(e->expand_forms_1(ctx,e), ctx, ex) @@ -343,13 +453,12 @@ function expand_forms_1(mod::Module, ex::SyntaxTree) scope_layer=LayerId, __macro_ctx__=Nothing, meta=CompileHints) - layers = ScopeLayer[ScopeLayer(1, mod, false)] - ctx = MacroExpansionContext(graph, Bindings(), layers, layers[1]) + ctx = MacroExpansionContext(graph, mod) ex2 = expand_forms_1(ctx, reparent(ctx, ex)) graph2 = delete_attributes(graph, :__macro_ctx__) # TODO: Returning the context with pass-specific mutable data is a bad way - # to carry state into the next pass. - ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, - ctx.current_layer) + # to carry state into the next pass. We might fix this by attaching such + # data to the graph itself as global attributes? + ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, LayerId[]) return ctx2, reparent(ctx2, ex2) end diff --git a/src/runtime.jl b/src/runtime.jl index 23252400..9d9e0e50 100644 --- a/src/runtime.jl +++ b/src/runtime.jl @@ -306,12 +306,11 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a ) # Macro expansion - layers = ScopeLayer[ScopeLayer(1, mod, false)] - ctx1 = MacroExpansionContext(graph, Bindings(), layers, layers[1]) + ctx1 = MacroExpansionContext(graph, mod) # Run code generator - this acts like a macro expander and like a macro # expander it gets a MacroContext. - mctx = MacroContext(syntax_graph(ctx1), g.srcref, layers[1]) + mctx = MacroContext(syntax_graph(ctx1), g.srcref, ctx1.scope_layers[end]) ex0 = g.gen(mctx, args...) if ex0 isa SyntaxTree if !is_compatible_graph(ctx1, ex0) @@ -326,11 +325,11 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a # Expand any macros emitted by the generator ex1 = expand_forms_1(ctx1, reparent(ctx1, ex0)) ctx1 = MacroExpansionContext(delete_attributes(graph, :__macro_ctx__), - ctx1.bindings, ctx1.scope_layers, ctx1.current_layer) + ctx1.bindings, ctx1.scope_layers, LayerId[]) ex1 = reparent(ctx1, ex1) # Desugaring - ctx2, ex2 = expand_forms_2( ctx1, ex1) + ctx2, ex2 = expand_forms_2(ctx1, ex1) # Wrap expansion in a non-toplevel lambda and run scope resolution ex2 = @ast ctx2 ex0 [K"lambda"(is_toplevel_thunk=false) @@ -342,12 +341,11 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a ] ex2 ] - ctx3, ex3 = resolve_scopes( ctx2, ex2) - + ctx3, ex3 = resolve_scopes(ctx2, ex2) # Rest of lowering ctx4, ex4 = convert_closures(ctx3, ex3) - ctx5, ex5 = linearize_ir( ctx4, ex4) + ctx5, ex5 = linearize_ir(ctx4, ex4) ci = to_lowered_expr(mod, ex5) @assert ci isa Core.CodeInfo return ci @@ -414,3 +412,8 @@ function lookup_method_instance(func, args, world::Integer) world::Csize_t)::Ptr{Cvoid} return mi == C_NULL ? nothing : unsafe_pointer_to_objref(mi) end + +# Like `Base.methods()` but with world age support +function methods_in_world(func, arg_sig, world) + Base._methods(func, arg_sig, -1, world) +end diff --git a/src/syntax_graph.jl b/src/syntax_graph.jl index 6eb0737c..d883b246 100644 --- a/src/syntax_graph.jl +++ b/src/syntax_graph.jl @@ -429,7 +429,7 @@ attrsummary(name, value::Number) = "$name=$value" function _value_string(ex) k = kind(ex) str = k == K"Identifier" || k == K"MacroName" || is_operator(k) ? ex.name_val : - k == K"Placeholder" ? ex.name_val : + k == K"Placeholder" ? ex.name_val : k == K"SSAValue" ? "%" : k == K"BindingId" ? "#" : k == K"label" ? "label" : @@ -546,9 +546,16 @@ JuliaSyntax.byte_range(ex::SyntaxTree) = byte_range(sourceref(ex)) function JuliaSyntax._expr_leaf_val(ex::SyntaxTree, _...) name = get(ex, :name_val, nothing) - !isnothing(name) && return Symbol(name) - name = get(ex, :value, nothing) - return name + if !isnothing(name) + n = Symbol(name) + if hasattr(ex, :scope_layer) + Expr(:scope_layer, n, ex.scope_layer) + else + n + end + else + get(ex, :value, nothing) + end end Base.Expr(ex::SyntaxTree) = JuliaSyntax.to_expr(ex) @@ -588,6 +595,21 @@ function _find_SyntaxTree_macro(ex, line) return nothing # Will get here if multiple children are on the same line. end +# Translate JuliaLowering hygiene to esc() for use in @SyntaxTree +function _scope_layer_1_to_esc!(ex) + if ex isa Expr + if ex.head == :scope_layer + @assert ex.args[2] === 1 + return esc(_scope_layer_1_to_esc!(ex.args[1])) + else + map!(_scope_layer_1_to_esc!, ex.args, ex.args) + return ex + end + else + return ex + end +end + """ Macro to construct quoted SyntaxTree literals (instead of quoted Expr literals) in normal Julia source code. @@ -630,10 +652,10 @@ macro SyntaxTree(ex_old) # discover the piece of AST which should be returned. ex = _find_SyntaxTree_macro(full_ex, __source__.line) # 4. Do the first step of JuliaLowering's syntax lowering to get - # synax interpolations to work + # syntax interpolations to work _, ex1 = expand_forms_1(__module__, ex) @assert kind(ex1) == K"call" && ex1[1].value == interpolate_ast - esc(Expr(:call, interpolate_ast, ex1[2][1], map(Expr, ex1[3:end])...)) + Expr(:call, :interpolate_ast, ex1[2][1], map(e->_scope_layer_1_to_esc!(Expr(e)), ex1[3:end])...) end #------------------------------------------------------------------------------- diff --git a/test/demo.jl b/test/demo.jl index 3aa83937..c0698b02 100644 --- a/test/demo.jl +++ b/test/demo.jl @@ -43,7 +43,7 @@ end #------------------------------------------------------------------------------- # Module containing macros used in the demo. -define_macros = false +define_macros = true if !define_macros eval(:(module M end)) else @@ -95,6 +95,34 @@ eval(JuliaLowering.@SyntaxTree :(baremodule M end end + macro call_show(x) + quote + z = "z in @call_show" + @show z $x + end + end + + macro call_info(x) + quote + z = "z in @call_info" + @info "hi" z $x + end + end + + macro call_oldstyle_macro(y) + quote + x = "x in call_oldstyle_macro" + @oldstyle $y x + end + end + + macro newstyle(x, y, z) + quote + x = "x in @newstyle" + ($x, $y, $z, x) + end + end + macro set_a_global(val) quote global a_global = $val @@ -182,6 +210,16 @@ eval(JuliaLowering.@SyntaxTree :(baremodule M end)) end + +Base.eval(M, :( +macro oldstyle(a, b) + quote + x = "x in @oldstyle" + @newstyle $(esc(a)) $(esc(b)) x + end +end +)) + # #------------------------------------------------------------------------------- # Demos of the prototype @@ -794,7 +832,24 @@ end """ src = """ -cglobal(:jl_uv_stdin, Ptr{Cvoid}) +let + z = "z in outer ctx" + @call_show z +end +""" + +src = """ +let + x = "x in outer ctx" + @call_oldstyle_macro x +end +""" + +src = """ +let + z = "z in outer ctx" + @call_info z +end """ ex = parsestmt(SyntaxTree, src, filename="foo.jl") diff --git a/test/macros.jl b/test/macros.jl index a75d0fec..34d02887 100644 --- a/test/macros.jl +++ b/test/macros.jl @@ -4,7 +4,7 @@ using JuliaLowering, Test module test_mod end -JuliaLowering.include_string(test_mod, """ +JuliaLowering.include_string(test_mod, raw""" module M using JuliaLowering: JuliaLowering, @ast, @chk, adopt_scope using JuliaSyntax @@ -28,51 +28,49 @@ module M macro foo(ex) :(begin x = "`x` from @foo" - (x, someglobal, \$ex) + (x, someglobal, $ex) end) end # Set `a_global` in M macro set_a_global(val) :(begin - global a_global = \$val + global a_global = $val end) end macro set_other_global(ex, val) :(begin - global \$ex = \$val + global $ex = $val end) end macro set_global_in_parent(ex) e1 = adopt_scope(:(sym_introduced_from_M), __context__) quote - \$e1 = \$ex + $e1 = $ex nothing end end macro inner() - :(2) + :(y) end macro outer() - :((1, @inner)) - end - - # # Recursive macro call - # # TODO: Need branching! - # macro recursive(N) - # Nval = N.value #::Int - # if Nval < 1 - # return N - # end - # quote - # x = \$N - # (@recursive \$(Nval-1), x) - # end - # end + :((x, @inner)) + end + + macro recursive(N) + Nval = N.value::Int + if Nval < 1 + return N + end + quote + x = $N + (x, @recursive $(Nval-1)) + end + end end """) @@ -84,6 +82,7 @@ end """) == ("`x` from @foo", "global in module M", "`x` from outer scope") @test !isdefined(test_mod.M, :x) + @test JuliaLowering.include_string(test_mod, """ #line1 (M.@__MODULE__(), M.@__FILE__(), M.@__LINE__()) @@ -106,43 +105,21 @@ JuliaLowering.include_string(test_mod, "M.@set_other_global global_in_test_mod 1 @test !isdefined(test_mod.M, :global_in_test_mod) @test test_mod.global_in_test_mod == 100 -Base.eval(test_mod.M, :( -# Recursive macro call -function var"@recursive"(mctx, N) - @chk kind(N) == K"Integer" - Nval = N.value::Int - if Nval < 1 - return N - end - @ast mctx (@HERE) [K"block" - [K"="(@HERE) - "x"::K"Identifier"(@HERE) - N - ] - [K"tuple"(@HERE) - "x"::K"Identifier"(@HERE) - [K"macrocall"(@HERE) - "@recursive"::K"Identifier" - (Nval-1)::K"Integer" - ] - ] - ] -end -)) - @test JuliaLowering.include_string(test_mod, """ M.@recursive 3 """) == (3, (2, (1, 0))) -@test let - ex = JuliaLowering.parsestmt(JuliaLowering.SyntaxTree, "M.@outer()", filename="foo.jl") - expanded = JuliaLowering.macroexpand(test_mod, ex) - JuliaLowering.sourcetext.(JuliaLowering.flattened_provenance(expanded[2])) -end == [ +ex = JuliaLowering.parsestmt(JuliaLowering.SyntaxTree, "M.@outer()", filename="foo.jl") +ctx, expanded = JuliaLowering.expand_forms_1(test_mod, ex) +@test JuliaLowering.sourcetext.(JuliaLowering.flattened_provenance(expanded[2])) == [ "M.@outer()" "@inner" - "2" + "y" ] +# Layer parenting +@test expanded[1].scope_layer == 2 +@test expanded[2].scope_layer == 3 +@test getfield.(ctx.scope_layers, :parent_layer) == [0,1,2] JuliaLowering.include_string(test_mod, """ f_throw(x) = throw(x) @@ -194,4 +171,126 @@ let (err, st) = try @test any(sf->sf.func===:ccall_macro_parse, st) end +# Tests for interop between old and new-style macros + +# Hygiene interop +JuliaLowering.include_string(test_mod, raw""" + macro call_oldstyle_macro(a) + quote + x = "x in call_oldstyle_macro" + @oldstyle $a x + end + end + + macro newstyle(a, b, c) + quote + x = "x in @newstyle" + ($a, $b, $c, x) + end + end +""") +# TODO: Make this macro lowering go via JuliaSyntax rather than the flisp code +# (JuliaSyntax needs support for old-style quasiquote processing) +Base.eval(test_mod, :( +macro oldstyle(a, b) + quote + x = "x in @oldstyle" + @newstyle $(esc(a)) $(esc(b)) x + end +end +)) +@test JuliaLowering.include_string(test_mod, """ +let x = "x in outer scope" + @call_oldstyle_macro x +end +""") == ("x in outer scope", + "x in call_oldstyle_macro", + "x in @oldstyle", + "x in @newstyle") + +# Old style unhygenic escaping with esc() +Base.eval(test_mod, :( +macro oldstyle_unhygenic() + esc(:x) +end +)) +@test JuliaLowering.include_string(test_mod, """ +let x = "x in outer scope" + @oldstyle_unhygenic +end +""") == "x in outer scope" + +# Exceptions in old style macros +Base.eval(test_mod, :( +macro oldstyle_error() + error("Some error in old style macro") +end +)) +@test try + JuliaLowering.include_string(test_mod, """ + @oldstyle_error + """) +catch exc + sprint(showerror, exc) +end == """ +MacroExpansionError while expanding @oldstyle_error in module Main.macros.test_mod: +@oldstyle_error +└─────────────┘ ── Error expanding macro +Caused by: +Some error in old style macro""" + +# Old-style macros returning non-Expr values +Base.eval(test_mod, :( +macro oldstyle_non_Expr() + 42 +end +)) +@test JuliaLowering.include_string(test_mod, """ +@oldstyle_non_Expr +""") === 42 + +# New-style macros called with the wrong arguments +JuliaLowering.include_string(test_mod, raw""" +macro method_error_test(a) +end +""") +Base.eval(test_mod, :( +macro method_error_test() +end +)) +try + JuliaLowering.include_string(test_mod, raw""" + @method_error_test x y + """) + @test false +catch exc + @test exc isa JuliaLowering.MacroExpansionError + mexc = exc.err + @test mexc isa MethodError + @test mexc.args isa Tuple{JuliaLowering.MacroContext, JuliaLowering.SyntaxTree, JuliaLowering.SyntaxTree} +end + +@testset "calling with old/new macro signatures" begin + # Old defined with 1 arg, new with 2 args, both with 3 (but with different values) + Base.eval(test_mod, :(macro sig_mismatch(x); x; end)) + Base.eval(test_mod, :(macro sig_mismatch(x, y, z); z; end)) + JuliaLowering.include_string(test_mod, "macro sig_mismatch(x, y); x; end") + JuliaLowering.include_string(test_mod, "macro sig_mismatch(x, y, z); x; end") + + @test JuliaLowering.include_string(test_mod, "@sig_mismatch(1)") === 1 + @test JuliaLowering.include_string(test_mod, "@sig_mismatch(1, 2)") === 1 + @test JuliaLowering.include_string(test_mod, "@sig_mismatch(1, 2, 3)") === 1 # 3 if we prioritize old sig + err = try + JuliaLowering.include_string(test_mod, "@sig_mismatch(1, 2, 3, 4)") === 1 + catch exc + sprint(showerror, exc, context=:module=>@__MODULE__) + end + @test startswith(err, """ + MacroExpansionError while expanding @sig_mismatch in module Main.macros.test_mod: + @sig_mismatch(1, 2, 3, 4) + └───────────────────────┘ ── Error expanding macro + Caused by: + MethodError: no method matching var"@sig_mismatch"(::JuliaLowering.MacroContext, ::JuliaLowering.SyntaxTree""") +end + end # module macros