Skip to content

Compat behaviours: typeof quoted expressions, macro def signatures #44

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 22, 2025
Merged

Compat behaviours: typeof quoted expressions, macro def signatures #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
36962e0
Compat behaviours: typeof quoted expressions, macro def signatures
mlechu Aug 20, 2025
225c8a6
Thread `expr_compat_mode` param through eval
mlechu Aug 21, 2025
4abba47
Update src/runtime.jl
mlechu Aug 21, 2025
5ec41d3
Apply suggestions from code review
mlechu Aug 22, 2025
ada69fb
Fix unexpanded `K"parameters"`
mlechu Aug 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 53 additions & 21 deletions src/desugaring.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,22 @@ struct DesugaringContext{GraphType} <: AbstractLoweringContext
bindings::Bindings
scope_layers::Vector{ScopeLayer}
mod::Module
expr_compat_mode::Bool
end

function DesugaringContext(ctx)
function DesugaringContext(ctx, expr_compat_mode::Bool)
graph = ensure_attributes(syntax_graph(ctx),
kind=Kind, syntax_flags=UInt16,
source=SourceAttrType,
value=Any, name_val=String,
scope_type=Symbol, # :hard or :soft
var_id=IdTag,
is_toplevel_thunk=Bool)
DesugaringContext(graph, ctx.bindings, ctx.scope_layers, first(ctx.scope_layers).mod)
DesugaringContext(graph,
ctx.bindings,
ctx.scope_layers,
first(ctx.scope_layers).mod,
expr_compat_mode)
end

#-------------------------------------------------------------------------------
Expand Down Expand Up @@ -3246,20 +3251,39 @@ function expand_macro_def(ctx, ex)
name = sig[1]
args = remove_empty_parameters(children(sig))
@chk kind(args[end]) != K"parameters" (args[end], "macros cannot accept keyword arguments")
ret = @ast ctx ex [K"function"
[K"call"(sig)
_make_macro_name(ctx, name)
[K"::"
adopt_scope(@ast(ctx, sig, "__context__"::K"Identifier"),
kind(name) == K"." ? name[1] : name)
MacroContext::K"Value"
scope_ref = kind(name) == K"." ? name[1] : name
if ctx.expr_compat_mode
@ast ctx ex [K"function"
[K"call"(sig)
_make_macro_name(ctx, name)
[K"::"
# TODO: should we be adopting the scope of the K"macro" expression itself?
adopt_scope(@ast(ctx, sig, "__source__"::K"Identifier"), scope_ref)
LineNumberNode::K"Value"
]
[K"::"
adopt_scope(@ast(ctx, sig, "__module__"::K"Identifier"), scope_ref)
Module::K"Value"
]
map(e->_apply_nospecialize(ctx, e), args[2:end])...
]
# flisp: We don't mark these @nospecialize because all arguments to
# new macros will be of type SyntaxTree
args[2:end]...
ex[2]
]
ex[2]
]
else
@ast ctx ex [K"function"
[K"call"(sig)
_make_macro_name(ctx, name)
[K"::"
adopt_scope(@ast(ctx, sig, "__context__"::K"Identifier"), scope_ref)
MacroContext::K"Value"
]
# flisp: We don't mark these @nospecialize because all arguments to
# new macros will be of type SyntaxTree
args[2:end]...
]
ex[2]
]
end
end

#-------------------------------------------------------------------------------
Expand Down Expand Up @@ -4284,9 +4308,10 @@ function expand_module(ctx, ex::SyntaxTree)
[K"inert" ex]
]
[K"call"
eval_module ::K"Value"
ctx.mod ::K"Value"
modname ::K"String"
eval_module ::K"Value"
ctx.mod ::K"Value"
modname ::K"String"
ctx.expr_compat_mode ::K"Bool"
[K"inert"(body)
[K"toplevel"
std_defs
Expand Down Expand Up @@ -4477,14 +4502,21 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing)
elseif k == K"toplevel"
# The toplevel form can't be lowered here - it needs to just be quoted
# and passed through to a call to eval.
@ast ctx ex [K"block"
ex2 = @ast ctx ex [K"block"
[K"assert" "toplevel_only"::K"Symbol" [K"inert" ex]]
[K"call"
eval ::K"Value"
ctx.mod ::K"Value"
eval ::K"Value"
ctx.mod ::K"Value"
[K"inert" ex]
[K"parameters"
[K"="
"expr_compat_mode"::K"Identifier"
ctx.expr_compat_mode::K"Bool"
]
]
]
]
expand_forms_2(ctx, ex2)
elseif k == K"vect"
check_no_parameters(ex, "unexpected semicolon in array expression")
expand_array(ctx, ex, "vect")
Expand Down Expand Up @@ -4546,7 +4578,7 @@ function expand_forms_2(ctx::StatementListCtx, args...)
end

function expand_forms_2(ctx::MacroExpansionContext, ex::SyntaxTree)
ctx1 = DesugaringContext(ctx)
ctx1 = DesugaringContext(ctx, ctx.expr_compat_mode)
ex1 = expand_forms_2(ctx1, reparent(ctx1, ex))
ctx1, ex1
end
14 changes: 7 additions & 7 deletions src/eval.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
function lower(mod::Module, ex0)
ctx1, ex1 = expand_forms_1( mod, ex0)
function lower(mod::Module, ex0, expr_compat_mode=false)
ctx1, ex1 = expand_forms_1( mod, ex0, expr_compat_mode)
ctx2, ex2 = expand_forms_2( ctx1, ex1)
ctx3, ex3 = resolve_scopes( ctx2, ex2)
ctx4, ex4 = convert_closures(ctx3, ex3)
ctx5, ex5 = linearize_ir( ctx4, ex4)
ex5
end

function macroexpand(mod::Module, ex)
ctx1, ex1 = expand_forms_1(mod, ex)
function macroexpand(mod::Module, ex, expr_compat_mode=false)
ctx1, ex1 = expand_forms_1(mod, ex, expr_compat_mode)
ex1
end

Expand Down Expand Up @@ -344,16 +344,16 @@ end

#-------------------------------------------------------------------------------
# Our version of eval takes our own data structures
function Core.eval(mod::Module, ex::SyntaxTree)
function Core.eval(mod::Module, ex::SyntaxTree; expr_compat_mode::Bool=false)
k = kind(ex)
if k == K"toplevel"
x = nothing
for e in children(ex)
x = eval(mod, e)
x = eval(mod, e; expr_compat_mode)
end
return x
end
linear_ir = lower(mod, ex)
linear_ir = lower(mod, ex, expr_compat_mode)
expr_form = to_lowered_expr(mod, linear_ir)
eval(mod, expr_form)
end
Expand Down
2 changes: 1 addition & 1 deletion src/hooks.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ function core_lowering_hook(@nospecialize(code), mod::Module,

st0 = code isa Expr ? expr_to_syntaxtree(code, LineNumberNode(line, file)) : code
try
ctx1, st1 = expand_forms_1( mod, st0)
ctx1, st1 = expand_forms_1( mod, st0, true)
ctx2, st2 = expand_forms_2( ctx1, st1)
ctx3, st3 = resolve_scopes( ctx2, st2)
ctx4, st4 = convert_closures(ctx3, st3)
Expand Down
12 changes: 7 additions & 5 deletions src/macro_expansion.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@ struct MacroExpansionContext{GraphType} <: AbstractLoweringContext
bindings::Bindings
scope_layers::Vector{ScopeLayer}
scope_layer_stack::Vector{LayerId}
expr_compat_mode::Bool
end

function MacroExpansionContext(graph::SyntaxGraph, mod::Module)
function MacroExpansionContext(graph::SyntaxGraph, mod::Module, expr_compat_mode::Bool)
layers = ScopeLayer[ScopeLayer(1, mod, 0, false)]
MacroExpansionContext(graph, Bindings(), layers, LayerId[length(layers)])
MacroExpansionContext(graph, Bindings(), layers, LayerId[length(layers)], expr_compat_mode)
end

current_layer(ctx::MacroExpansionContext) = ctx.scope_layers[last(ctx.scope_layer_stack)]
Expand Down Expand Up @@ -67,6 +68,7 @@ function expand_quote(ctx, ex)
# (ex, @HERE) ?
@ast ctx ex [K"call"
interpolate_ast::K"Value"
(ctx.expr_compat_mode ? Expr : SyntaxTree)::K"Value"
[K"inert" ex]
unquoted...
]
Expand Down Expand Up @@ -447,18 +449,18 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree)
end
end

function expand_forms_1(mod::Module, ex::SyntaxTree)
function expand_forms_1(mod::Module, ex::SyntaxTree, expr_compat_mode::Bool)
graph = ensure_attributes(syntax_graph(ex),
var_id=IdTag,
scope_layer=LayerId,
__macro_ctx__=Nothing,
meta=CompileHints)
ctx = MacroExpansionContext(graph, mod)
ctx = MacroExpansionContext(graph, mod, expr_compat_mode)
ex2 = expand_forms_1(ctx, reparent(ctx, ex))
graph2 = delete_attributes(graph, :__macro_ctx__)
# TODO: Returning the context with pass-specific mutable data is a bad way
# to carry state into the next pass. We might fix this by attaching such
# data to the graph itself as global attributes?
ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, LayerId[])
ctx2 = MacroExpansionContext(graph2, ctx.bindings, ctx.scope_layers, LayerId[], expr_compat_mode)
return ctx2, reparent(ctx2, ex2)
end
18 changes: 13 additions & 5 deletions src/runtime.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ function _interpolate_ast(ctx::InterpolationContext, ex, depth)
makenode(ctx, ex, head(ex), expanded_children)
end

function interpolate_ast(ex, values...)
function interpolate_ast(::Type{SyntaxTree}, ex, values...)
# Construct graph for interpolation context. We inherit this from the macro
# context where possible by detecting it using __macro_ctx__. This feels
# hacky though.
Expand Down Expand Up @@ -108,6 +108,14 @@ function interpolate_ast(ex, values...)
end
end

function interpolate_ast(::Type{Expr}, ex, values...)
# TODO: Adjust `_interpolated_value` to ensure that incoming `Expr` data
# structures are treated as AST in Expr compat mode, rather than `K"Value"`?
# Or convert `ex` to `Expr` early during lowering and implement
# `interpolate_ast` for `Expr`?
Expr(interpolate_ast(SyntaxTree, ex, values...))
Copy link
Owner

@c42f c42f Aug 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, this is a rather interesting approach. Preserving SyntaxTree in the IR as the definitive as-parsed representation is cute.

With this approach I'm concerned the logic in _interpolated_value doesn't make sense.

  • When interpolating values into SyntaxTree, we currently assume SyntaxTree is AST, and everything else is K"Value" (ie, a leaf)
  • When interpolating values into Expr, the existing system assumes Expr and LineNumberNode are pieces of AST, and everything else is a leaf (QuoteNode is a bit of an oddity ... is it both?)

There seems to be two ways out:

  1. We could take a union of all these rules and support bidirectional interpolation of Expr-into-SyntaxTree and vice versa.
  2. We can convert ex to an Expr at lowering time then implement interpolate_ast for Expr separately.

I feel like (2) would be a good conservative option for now. But at least we should make a note about these issues.

Suggested change
Expr(interpolate_ast(SyntaxTree, ex, values...))
# TODO: Adjust `_interpolated_value` to ensure that incoming `Expr` data structures are treated
# as AST in Expr compat mode, rather than `K"Value"`?
# Or convert `ex` to `Expr` early during lowering and implement `interpolate_ast` for `Expr`?
Expr(interpolate_ast(SyntaxTree, ex, values...))

end

#--------------------------------------------------
# Functions called by closure conversion
function eval_closure_type(mod::Module, closure_type_name::Symbol, field_names, field_is_box)
Expand Down Expand Up @@ -154,7 +162,7 @@ end
# public modname
#
# And run statments in the toplevel expression `body`
function eval_module(parentmod, modname, body)
function eval_module(parentmod, modname, expr_compat_mode, body)
# Here we just use `eval()` with an Expr.
# If we wanted to avoid this we'd need to reproduce a lot of machinery from
# jl_eval_module_expr()
Expand All @@ -171,7 +179,7 @@ function eval_module(parentmod, modname, body)
name = Symbol(modname)
eval(parentmod, :(
baremodule $name
$eval($name, $body)
$eval($name, $body; expr_compat_mode=$expr_compat_mode)
end
))
end
Expand Down Expand Up @@ -296,7 +304,7 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
)

# Macro expansion
ctx1 = MacroExpansionContext(graph, mod)
ctx1 = MacroExpansionContext(graph, mod, false)

# Run code generator - this acts like a macro expander and like a macro
# expander it gets a MacroContext.
Expand All @@ -315,7 +323,7 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
# Expand any macros emitted by the generator
ex1 = expand_forms_1(ctx1, reparent(ctx1, ex0))
ctx1 = MacroExpansionContext(delete_attributes(graph, :__macro_ctx__),
ctx1.bindings, ctx1.scope_layers, LayerId[])
ctx1.bindings, ctx1.scope_layers, LayerId[], false)
ex1 = reparent(ctx1, ex1)

# Desugaring
Expand Down
5 changes: 3 additions & 2 deletions src/syntax_graph.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -653,9 +653,10 @@ macro SyntaxTree(ex_old)
ex = _find_SyntaxTree_macro(full_ex, __source__.line)
# 4. Do the first step of JuliaLowering's syntax lowering to get
# syntax interpolations to work
_, ex1 = expand_forms_1(__module__, ex)
_, ex1 = expand_forms_1(__module__, ex, false)
@assert kind(ex1) == K"call" && ex1[1].value == interpolate_ast
Expr(:call, :interpolate_ast, ex1[2][1], map(e->_scope_layer_1_to_esc!(Expr(e)), ex1[3:end])...)
Expr(:call, :interpolate_ast, SyntaxTree, ex1[3][1],
map(e->_scope_layer_1_to_esc!(Expr(e)), ex1[4:end])...)
end

#-------------------------------------------------------------------------------
Expand Down
1 change: 1 addition & 0 deletions test/arrays_ir.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -496,3 +496,4 @@ a[] = rhs
2 TestMod.a
3 (call top.setindex! %₂ %₁)
4 (return %₁)

4 changes: 2 additions & 2 deletions test/demo.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ end
# Currently broken - need to push info back onto src
# function annotate_scopes(mod, ex)
# ex = ensure_attributes(ex, var_id=Int)
# ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(mod, ex)
# ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(mod, ex, false)
# ctx2, ex_desugar = JuliaLowering.expand_forms_2(ctx1, ex_macroexpand)
# ctx3, ex_scoped = JuliaLowering.resolve_scopes(ctx2, ex_desugar)
# ex
Expand Down Expand Up @@ -859,7 +859,7 @@ ex = ensure_attributes(ex, var_id=Int)

in_mod = M
# in_mod=Main
ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex)
ctx1, ex_macroexpand = JuliaLowering.expand_forms_1(in_mod, ex, false)
@info "Macro expanded" formatsrc(ex_macroexpand, color_by=:scope_layer)
#@info "Macro expanded" formatsrc(ex_macroexpand, color_by=e->JuliaLowering.flattened_provenance(e)[1:end-1])

Expand Down
4 changes: 2 additions & 2 deletions test/functions_ir.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1565,9 +1565,9 @@ end
10 (call core.svec %₇ %₈ %₉)
11 --- method core.nothing %₁₀
slots: [slot₁/#self#(!read) slot₂/__context__(!read) slot₃/#self#(!read) slot₄/x(nospecialize,!read) slot₅/y(nospecialize,!read)]
1 (call JuliaLowering.interpolate_ast (inert (block (= maybe_gen_stuff (call some_gen_stuff x y)))))
1 (call JuliaLowering.interpolate_ast SyntaxTree (inert (block (= maybe_gen_stuff (call some_gen_stuff x y)))))
2 (call core.tuple %₁)
3 (call JuliaLowering.interpolate_ast (inert (block (block (= nongen_stuff (call bothgen x y)) ($ (block (call JuliaLowering.interpolate_ast (inert (block (= maybe_gen_stuff (call some_gen_stuff x y))))))) (tuple-p nongen_stuff maybe_gen_stuff)))) %₂)
3 (call JuliaLowering.interpolate_ast SyntaxTree (inert (block (block (= nongen_stuff (call bothgen x y)) ($ (block (call JuliaLowering.interpolate_ast SyntaxTree (inert (block (= maybe_gen_stuff (call some_gen_stuff x y))))))) (tuple-p nongen_stuff maybe_gen_stuff)))) %₂)
4 (return %₃)
12 latestworld
13 TestMod.f_partially_generated
Expand Down
2 changes: 1 addition & 1 deletion test/generators_ir.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ LoweringError:
9 (call core.svec %₆ %₇ %₈)
10 --- method core.nothing %₉
slots: [slot₁/#self#(!read) slot₂/_(!read)]
1 (call JuliaLowering.interpolate_ast (inert (return x)))
1 (call JuliaLowering.interpolate_ast SyntaxTree (inert (return x)))
2 (return %₁)
11 latestworld
12 TestMod.#->##6
Expand Down
2 changes: 1 addition & 1 deletion test/macros.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ M.@recursive 3
""") == (3, (2, (1, 0)))

ex = JuliaLowering.parsestmt(JuliaLowering.SyntaxTree, "M.@outer()", filename="foo.jl")
ctx, expanded = JuliaLowering.expand_forms_1(test_mod, ex)
ctx, expanded = JuliaLowering.expand_forms_1(test_mod, ex, false)
@test JuliaLowering.sourcetext.(JuliaLowering.flattened_provenance(expanded[2])) == [
"M.@outer()"
"@inner"
Expand Down
2 changes: 1 addition & 1 deletion test/macros_ir.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ end
9 --- method core.nothing %₈
slots: [slot₁/#self#(!read) slot₂/__context__(!read) slot₃/ex]
1 (call core.tuple slot₃/ex)
2 (call JuliaLowering.interpolate_ast (inert (block (call-i ($ ex) + 1))) %₁)
2 (call JuliaLowering.interpolate_ast SyntaxTree (inert (block (call-i ($ ex) + 1))) %₁)
3 (return %₂)
10 latestworld
11 TestMod.@add_one
Expand Down
6 changes: 3 additions & 3 deletions test/quoting_ir.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ end
#---------------------
1 TestMod.x
2 (call core.tuple %₁)
3 (call JuliaLowering.interpolate_ast (inert (block (call-i ($ x) + 1))) %₂)
3 (call JuliaLowering.interpolate_ast SyntaxTree (inert (block (call-i ($ x) + 1))) %₂)
4 (return %₃)

########################################
Expand All @@ -15,7 +15,7 @@ end
#---------------------
1 TestMod.x
2 (call core.tuple %₁)
3 (call JuliaLowering.interpolate_ast (inert ($ x)) %₂)
3 (call JuliaLowering.interpolate_ast SyntaxTree (inert ($ x)) %₂)
4 (return %₃)

########################################
Expand All @@ -28,7 +28,7 @@ end
#---------------------
1 TestMod.x
2 (call core.tuple %₁)
3 (call JuliaLowering.interpolate_ast (inert (block (quote (block (call-i ($ ($ x)) + 1))))) %₂)
3 (call JuliaLowering.interpolate_ast SyntaxTree (inert (block (quote (block (call-i ($ ($ x)) + 1))))) %₂)
4 (return %₃)

########################################
Expand Down
Loading