Macro expansion for old-style Expr macros

Implements mixed macro expansion for old-style macros (those written to
expect an `Expr` data structure) and new-style macros (those written to
expect `SyntaxTree`).

The main difficulty here is managing hygiene correctly. We choose to
represent new-style scoped identifiers passed to old macros using
`Expr(:scope_layer, name, layer_id)` where necessary. But only where
necessary - in most contexts, old-style macros will see unadorned
identifiers just as they currently do. The only time the new `Expr`
construct is visible is when new macros interpolate an expression into a
call to an old-style macro in the returned code. Previously, such
macro-calling-macro situations would result in the inner macro call
seeing `Expr(:escape, ...)` but they now see `Expr(:scope_layer)`.
However, and it's rare for old-style macros to de- and re-construct
escaped expressions correctly so this should be a minor issue for
compatibility.

Old-style macros may still return `Expr(:escape)` expressions resulting
from manual escaping. When consuming the output of old macros, we
process these manual escapes by escaping up the macro expansion stack in
the same way we currently do.

Also add `parent_layer` id to `ScopeLayer` to preserve the macro
expansion stack there for use by JETLS.

Co-authored-by: Em Chu <[email protected]>

Author: c42f

README.md

@@ -288,6 +288,65 @@ discussed in Adams' paper:
 
 TODO: Write more here...
 
+
+### Compatibility with `Expr` macros
+
+In order to have compatibility with old-style macros which expect an `Expr`-based
+data structure as input, we convert `SyntaxTree` to `Expr`, call the old-style
+macro, then convert `SyntaxTree` back to `Expr` and continue with the expansion
+process. This involves some loss of provenance precision but allows full
+interoperability in the package ecosystem without a need to make breaking
+changes.
+
+Let's look at an example. Suppose a manually escaped old-style macro
+`@oldstyle` is implemented as
+
+```julia
+macro oldstyle(a, b)
+    quote
+        x = "x in @oldstyle"
+        @newstyle $(esc(a)) $(esc(b)) x
+    end
+end
+```
+
+along with two correctly escaped new-style macros:
+
+```julia
+macro call_oldstyle_macro(y)
+    quote
+        x = "x in call_oldstyle_macro"
+        @oldstyle $y x
+    end
+end
+
+macro newstyle(x, y, z)
+    quote
+        x = "x in @newstyle"
+        ($x, $y, $z, x)
+    end
+end
+```
+
+Then want some code like the following to "just work" with respect to hygiene
+
+```julia
+let
+    x = "x in outer ctx"
+    @call_oldstyle_macro x
+end
+```
+
+When calling `@oldstyle`, we must convert `SyntaxTree` into `Expr`, but we need
+to preserve the scope layer of the `x` from the outer context as it is passed
+into `@oldstyle` as a macro argument. To do this, we use `Expr(:scope_layer,
+:x, outer_layer_id)`. (In the old system, this would be `Expr(:escape, :x)`
+instead, presuming that `@call_oldstyle_macro` was implemented using `esc()`.)
+
+When receiving output from old style macro invocations, we preserve the escape
+handling of the existing system for any symbols which aren't tagged with a
+scope layer.
+
 ## Pass 2: Syntax desugaring
 
 This pass recursively converts many special surface syntax forms to a smaller

src/ast.jl

@@ -662,8 +662,8 @@ function to_symbol(ctx, ex)
     @ast ctx ex ex=>K"Symbol"
 end
 
-function new_scope_layer(ctx, mod_ref::Module=ctx.mod; is_macro_expansion=true)
-    new_layer = ScopeLayer(length(ctx.scope_layers)+1, ctx.mod, is_macro_expansion)
+function new_scope_layer(ctx, mod_ref::Module=ctx.mod)
+    new_layer = ScopeLayer(length(ctx.scope_layers)+1, ctx.mod, 0, false)
     push!(ctx.scope_layers, new_layer)
     new_layer.id
 end

src/compat.jl

@@ -24,7 +24,8 @@ function expr_to_syntaxtree(@nospecialize(e), lnn::Union{LineNumberNode, Nothing
         SyntaxGraph(),
         kind=Kind, syntax_flags=UInt16,
         source=SourceAttrType, var_id=Int, value=Any,
-        name_val=String, is_toplevel_thunk=Bool)
+        name_val=String, is_toplevel_thunk=Bool,
+        scope_layer=LayerId)
     expr_to_syntaxtree(graph, e, lnn)
 end
 
@@ -423,6 +424,13 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
             @assert nargs === 1
             child_exprs[1] = Expr(:quoted_symbol, e.args[1])
         end
+    elseif e.head === :scope_layer 
+        @assert nargs === 2
+        @assert e.args[1] isa Symbol
+        @assert e.args[2] isa LayerId
+        st_id, src = _insert_convert_expr(e.args[1], graph, src)
+        setattr!(graph, st_id, scope_layer=e.args[2])
+        return st_id, src
     elseif e.head === :symbolicgoto || e.head === :symboliclabel
         @assert nargs === 1
         st_k = e.head === :symbolicgoto ? K"symbolic_label" : K"symbolic_goto"

src/desugaring.jl

@@ -15,7 +15,7 @@ function DesugaringContext(ctx)
                               scope_type=Symbol, # :hard or :soft
                               var_id=IdTag,
                               is_toplevel_thunk=Bool)
-    DesugaringContext(graph, ctx.bindings, ctx.scope_layers, ctx.current_layer.mod)
+    DesugaringContext(graph, ctx.bindings, ctx.scope_layers, first(ctx.scope_layers).mod)
 end
 
 #-------------------------------------------------------------------------------
@@ -2555,7 +2555,7 @@ function keyword_function_defs(ctx, srcref, callex_srcref, name_str, typevar_nam
     end
     # TODO: Is the layer correct here? Which module should be the parent module
     # of this body function?
-    layer = new_scope_layer(ctx; is_macro_expansion=false)
+    layer = new_scope_layer(ctx)
     body_func_name = adopt_scope(@ast(ctx, callex_srcref, mangled_name::K"Identifier"), layer)
 
     kwcall_arg_names = SyntaxList(ctx)

src/kinds.jl

@@ -48,8 +48,11 @@ function _register_kinds()
             # Internal initializer for struct types, for inner constructors/functions
             "new"
             "splatnew"
-            # For expr-macro compatibility; gone after expansion
+            # Used for converting `esc()`'d expressions arising from old macro
+            # invocations during macro expansion (gone after macro expansion)
             "escape"
+            # Used for converting the old-style macro hygienic-scope form (gone
+            # after macro expansion)
             "hygienic_scope"
             # An expression which will eventually be evaluated "statically" in
             # the context of a CodeInfo and thus allows access only to globals