From 4e8ec55abb8e161e8070e9bb4f13ea2584185bab Mon Sep 17 00:00:00 2001 From: Claire Foster Date: Fri, 26 Sep 2025 10:55:57 +1000 Subject: [PATCH 1/5] `mapexpr` support + refine incremental lowering API * Fix macro expansion world age in new `eval()` to always use the latest world; make it the duty of `eval()` to manage this global state and pass it to lowering steps. * Add support for `mapexpr` in `include_string()`. When `expr_compat_mode=true`, this function will take an `Expr`, otherwise a `SyntaxTree`. * Remove detailed type information from `LoweringIterator` because it seems unlikely that we'll obtain much advantage from this, it'll just cause additional codegen. --- src/compat.jl | 2 +- src/eval.jl | 54 ++++++++++++++++++++++++++++++++----------------- test/modules.jl | 40 +++++++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 20 deletions(-) diff --git a/src/compat.jl b/src/compat.jl index bb2edc3e..72967e3e 100644 --- a/src/compat.jl +++ b/src/compat.jl @@ -29,7 +29,7 @@ function expr_to_syntaxtree(@nospecialize(e), lnn::Union{LineNumberNode, Nothing expr_to_syntaxtree(graph, e, lnn) end -@fzone "JL: expr_to_syntaxtree" function expr_to_syntaxtree(ctx, @nospecialize(e), lnn::Union{LineNumberNode, Nothing}) +@fzone "JL: expr_to_syntaxtree" function expr_to_syntaxtree(ctx, @nospecialize(e), lnn::Union{LineNumberNode, Nothing}=nothing) graph = syntax_graph(ctx) toplevel_src = if isnothing(lnn) # Provenance sinkhole for all nodes until we hit a linenode diff --git a/src/eval.jl b/src/eval.jl index fe859ae0..ce206601 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -32,19 +32,23 @@ end # We might consider changing at least the second of these choices, depending on # how we end up putting this into Base. -struct LoweringIterator{GraphType} - ctx::MacroExpansionContext{GraphType} - todo::Vector{Tuple{SyntaxTree{GraphType}, Bool, Int}} +struct LoweringIterator + # frontend::JuliaLoweringFrontend # TODO: world age? + ctx::MacroExpansionContext + todo::Vector{Tuple{SyntaxTree, Bool, Int}} + mapexpr::Any end -function lower_init(ex::SyntaxTree, mod::Module, macro_world::UInt; expr_compat_mode::Bool=false) +function lower_init(mod::Module, ex::SyntaxTree, mapexpr; + expr_compat_mode::Bool=false) graph = ensure_macro_attributes(syntax_graph(ex)) - ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world) + dummy_world = zero(UInt) + ctx = MacroExpansionContext(graph, mod, expr_compat_mode, dummy_world) ex = reparent(ctx, ex) - LoweringIterator{typeof(graph)}(ctx, [(ex, false, 0)]) + LoweringIterator(ctx, [(ex, false, 0)], mapexpr) end -function lower_step(iter, push_mod=nothing) +function lower_step(iter, macro_world, push_mod=nothing) if !isnothing(push_mod) push_layer!(iter.ctx, push_mod, false) end @@ -64,19 +68,28 @@ function lower_step(iter, push_mod=nothing) pop_layer!(iter.ctx) return Core.svec(:end_module) else - return lower_step(iter) + return lower_step(iter, macro_world) end end end k = kind(ex) if !(k in KSet"toplevel module") - ex = expand_forms_1(iter.ctx, ex) + if !is_module_body && !isnothing(iter.mapexpr) + ex = iter.ctx.expr_compat_mode ? + expr_to_syntaxtree(iter.ctx, iter.mapexpr(Expr(ex))) : + iter.mapexpr(ex) + end + c = iter.ctx + # Copy context in order to update macro_world + ctx = MacroExpansionContext(c.graph, c.bindings, c.scope_layers, + c.scope_layer_stack, c.expr_compat_mode, macro_world) + ex = expand_forms_1(ctx, ex) k = kind(ex) end if k == K"toplevel" push!(iter.todo, (ex, false, 1)) - return lower_step(iter) + return lower_step(iter, macro_world) elseif k == K"module" name = ex[1] if kind(name) != K"Identifier" @@ -451,10 +464,8 @@ end #------------------------------------------------------------------------------- # Our version of eval - should be upstreamed though? -@fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree; - macro_world::UInt=Base.get_world_counter(), - opts...) - iter = lower_init(ex, mod, macro_world; opts...) +@fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree; mapexpr=nothing, opts...) + iter = lower_init(mod, ex, mapexpr; opts...) _eval(mod, iter) end @@ -465,7 +476,7 @@ function _eval(mod, iter) new_mod = nothing result = nothing while true - thunk = lower_step(iter, new_mod)::Core.SimpleVector + thunk = lower_step(iter, Base.get_world_counter(), new_mod)::Core.SimpleVector new_mod = nothing type = thunk[1]::Symbol if type == :done @@ -494,7 +505,7 @@ function _eval(mod, iter, new_mod=nothing) in_new_mod = !isnothing(new_mod) result = nothing while true - thunk = lower_step(iter, new_mod)::Core.SimpleVector + thunk = lower_step(iter, Base.get_world_counter(), new_mod)::Core.SimpleVector new_mod = nothing type = thunk[1]::Symbol if type == :done @@ -553,7 +564,14 @@ end Like `include`, except reads code from the given string rather than from a file. """ +function include_string(mapexpr::Function, mod::Module, + code::AbstractString, filename::AbstractString="string"; + opts...) + ex = parseall(SyntaxTree, code; filename=filename) + eval(mod, ex; mapexpr=(mapexpr === identity ? nothing : mapexpr), opts...) +end + function include_string(mod::Module, code::AbstractString, filename::AbstractString="string"; - expr_compat_mode=false) - eval(mod, parseall(SyntaxTree, code; filename=filename); expr_compat_mode) + opts...) + include_string(identity, mod, code, filename; opts...) end diff --git a/test/modules.jl b/test/modules.jl index a68c5f8a..e32e1165 100644 --- a/test/modules.jl +++ b/test/modules.jl @@ -1,4 +1,4 @@ -@testset "modules" begin +@testset "modules and top level code" begin test_mod = Module() @@ -51,4 +51,42 @@ end """) @test Amod.init_order == ["B", "D", "E", "C", "A"] +# Macros in top level and module expressions may be used immediately in the +# next top level statement after their definition +@test JuliaLowering.include_string(test_mod, """ +macro mac1() + :(42) +end + +@mac1 +""") === 42 + +@test JuliaLowering.include_string(test_mod, """ +module ModuleTopLevelEvalWorldTest + macro mac2() + :(101) + end + + xx = @mac2 +end + +ModuleTopLevelEvalWorldTest.xx +""") === 101 + +function test_mapexpr_1(ex::JuliaLowering.SyntaxTree) + JuliaLowering.@ast ex ex 10101::K"Integer" +end + +@test JuliaLowering.include_string(test_mapexpr_1, test_mod, """ +this + expression + will + be + replaced +""") === 10101 + +function test_mapexpr_2(ex::Expr) + 20202 +end + +@test JuliaLowering.include_string(test_mapexpr_2, test_mod, """ +this + expression + will + be + replaced +"""; expr_compat_mode=true) === 20202 + end From f889a143e3c4df56c47bf5103ef468e763d27540 Mon Sep 17 00:00:00 2001 From: Claire Foster Date: Fri, 26 Sep 2025 19:31:42 +1000 Subject: [PATCH 2/5] Compiler frontend API and sketch for Core integration Here I've generalized and refactored the incremental lowering code from PR #84, splitting out the interface from the implementation and factoring the parts which should go into `Core` into a `_Core` module. `_Core.eval()` is implemented in terms of a new `_Core.simple_eval()` which can evaluate only `CodeInfo` and the new types `TopLevelCodeIterator`, `BeginModule` and `EndModule`. Evaluation of `TopLevelCodeIterator` is implemented in terms of the incremental lowering interface which must be provided by a compiler frontend. A compiler frontend (subtype of `CompilerFrontend`) must come with implementations of the functions `lower_init` and `parseall` and `lower_step` must be being defined on the return type of `lower_init`. Having a type for the frontend solves the issue of how to tie parsing and lowering together without needing to convert to `Expr`. `parseall(frontend, code, ...)` returns a syntax tree of the preferred expression type for the frontend which can be fed into `lower_init`. This function is just a placeholder - it needs to be generalized to allow `Meta.parse()`, `Meta.parseatom()` and `Meta.parseall()` to be implemented in terms of it. Parser and lowering diagnostics are handled by throwing an exception from the frontend but we may want a fuller diagnostics API instead. --- src/eval.jl | 214 ------------------------- src/hooks.jl | 422 +++++++++++++++++++++++++++++++++++++++++++++++++- test/hooks.jl | 17 ++ 3 files changed, 435 insertions(+), 218 deletions(-) diff --git a/src/eval.jl b/src/eval.jl index ce206601..b9c3e5fb 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -15,107 +15,6 @@ function macroexpand(mod::Module, ex; expr_compat_mode=false, world=Base.get_wor ex1 end -# Incremental lowering API which can manage toplevel and module expressions. -# -# This iteration API is oddly bespoke and arguably somewhat non-Julian for two -# reasons: -# -# * Lowering knows when new modules are required, and may request them with -# `:begin_module`. However `eval()` generates those modules so they need to -# be passed back into lowering. So we can't just use `Base.iterate()`. (Put a -# different way, we have a situation which is suited to coroutines but we -# don't want to use full Julia `Task`s for this.) -# * We might want to implement this `eval()` in Julia's C runtime code or early -# in bootstrap. Hence using SimpleVector and Symbol as the return values of -# `lower_step()` -# -# We might consider changing at least the second of these choices, depending on -# how we end up putting this into Base. - -struct LoweringIterator - # frontend::JuliaLoweringFrontend # TODO: world age? - ctx::MacroExpansionContext - todo::Vector{Tuple{SyntaxTree, Bool, Int}} - mapexpr::Any -end - -function lower_init(mod::Module, ex::SyntaxTree, mapexpr; - expr_compat_mode::Bool=false) - graph = ensure_macro_attributes(syntax_graph(ex)) - dummy_world = zero(UInt) - ctx = MacroExpansionContext(graph, mod, expr_compat_mode, dummy_world) - ex = reparent(ctx, ex) - LoweringIterator(ctx, [(ex, false, 0)], mapexpr) -end - -function lower_step(iter, macro_world, push_mod=nothing) - if !isnothing(push_mod) - push_layer!(iter.ctx, push_mod, false) - end - - if isempty(iter.todo) - return Core.svec(:done) - end - - ex, is_module_body, child_idx = pop!(iter.todo) - if child_idx > 0 - next_child = child_idx + 1 - if child_idx <= numchildren(ex) - push!(iter.todo, (ex, is_module_body, next_child)) - ex = ex[child_idx] - else - if is_module_body - pop_layer!(iter.ctx) - return Core.svec(:end_module) - else - return lower_step(iter, macro_world) - end - end - end - - k = kind(ex) - if !(k in KSet"toplevel module") - if !is_module_body && !isnothing(iter.mapexpr) - ex = iter.ctx.expr_compat_mode ? - expr_to_syntaxtree(iter.ctx, iter.mapexpr(Expr(ex))) : - iter.mapexpr(ex) - end - c = iter.ctx - # Copy context in order to update macro_world - ctx = MacroExpansionContext(c.graph, c.bindings, c.scope_layers, - c.scope_layer_stack, c.expr_compat_mode, macro_world) - ex = expand_forms_1(ctx, ex) - k = kind(ex) - end - if k == K"toplevel" - push!(iter.todo, (ex, false, 1)) - return lower_step(iter, macro_world) - elseif k == K"module" - name = ex[1] - if kind(name) != K"Identifier" - throw(LoweringError(name, "Expected module name")) - end - newmod_name = Symbol(name.name_val) - body = ex[2] - if kind(body) != K"block" - throw(LoweringError(body, "Expected block in module body")) - end - std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG) - loc = source_location(LineNumberNode, ex) - push!(iter.todo, (body, true, 1)) - return Core.svec(:begin_module, newmod_name, std_defs, loc) - else - # Non macro expansion parts of lowering - ctx2, ex2 = expand_forms_2(iter.ctx, ex) - ctx3, ex3 = resolve_scopes(ctx2, ex2) - ctx4, ex4 = convert_closures(ctx3, ex3) - ctx5, ex5 = linearize_ir(ctx4, ex4) - thunk = to_lowered_expr(ex5) - return Core.svec(:thunk, thunk) - end -end - - #------------------------------------------------------------------------------- function codeinfo_has_image_globalref(@nospecialize(e)) @@ -462,116 +361,3 @@ function _to_lowered_expr(ex::SyntaxTree, stmt_offset::Int) end end -#------------------------------------------------------------------------------- -# Our version of eval - should be upstreamed though? -@fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree; mapexpr=nothing, opts...) - iter = lower_init(mod, ex, mapexpr; opts...) - _eval(mod, iter) -end - -if VERSION >= v"1.13.0-DEV.1199" # https://github.com/JuliaLang/julia/pull/59604 - -function _eval(mod, iter) - modules = Module[] - new_mod = nothing - result = nothing - while true - thunk = lower_step(iter, Base.get_world_counter(), new_mod)::Core.SimpleVector - new_mod = nothing - type = thunk[1]::Symbol - if type == :done - break - elseif type == :begin_module - push!(modules, mod) - mod = @ccall jl_begin_new_module(mod::Any, thunk[2]::Symbol, thunk[3]::Cint, - thunk[4].file::Cstring, thunk[4].line::Cint)::Module - new_mod = mod - elseif type == :end_module - @ccall jl_end_new_module(mod::Module)::Cvoid - result = mod - mod = pop!(modules) - else - @assert type == :thunk - result = Core.eval(mod, thunk[2]) - end - end - @assert isempty(modules) - return result -end - -else - -function _eval(mod, iter, new_mod=nothing) - in_new_mod = !isnothing(new_mod) - result = nothing - while true - thunk = lower_step(iter, Base.get_world_counter(), new_mod)::Core.SimpleVector - new_mod = nothing - type = thunk[1]::Symbol - if type == :done - @assert !in_new_mod - break - elseif type == :begin_module - name = thunk[2]::Symbol - std_defs = thunk[3] - result = Core.eval(mod, - Expr(:module, std_defs, name, - Expr(:block, thunk[4], Expr(:call, m->_eval(m, iter, m), name))) - ) - elseif type == :end_module - @assert in_new_mod - return mod - else - @assert type == :thunk - result = Core.eval(mod, thunk[2]) - end - end - return result -end - -end - -""" - include(mod::Module, path::AbstractString) - -Evaluate the contents of the input source file in the global scope of module -`mod`. Every module (except those defined with baremodule) has its own -definition of `include()` omitting the `mod` argument, which evaluates the file -in that module. Returns the result of the last evaluated expression of the -input file. During including, a task-local include path is set to the directory -containing the file. Nested calls to include will search relative to that path. -This function is typically used to load source interactively, or to combine -files in packages that are broken into multiple source files. -""" -function include(mod::Module, path::AbstractString) - path, prev = Base._include_dependency(mod, path) - code = read(path, String) - tls = task_local_storage() - tls[:SOURCE_PATH] = path - try - return include_string(mod, code, path) - finally - if prev === nothing - delete!(tls, :SOURCE_PATH) - else - tls[:SOURCE_PATH] = prev - end - end -end - -""" - include_string(mod::Module, code::AbstractString, filename::AbstractString="string") - -Like `include`, except reads code from the given string rather than from a file. -""" -function include_string(mapexpr::Function, mod::Module, - code::AbstractString, filename::AbstractString="string"; - opts...) - ex = parseall(SyntaxTree, code; filename=filename) - eval(mod, ex; mapexpr=(mapexpr === identity ? nothing : mapexpr), opts...) -end - -function include_string(mod::Module, code::AbstractString, filename::AbstractString="string"; - opts...) - include_string(identity, mod, code, filename; opts...) -end diff --git a/src/hooks.jl b/src/hooks.jl index 31157632..10266661 100644 --- a/src/hooks.jl +++ b/src/hooks.jl @@ -1,3 +1,408 @@ +#------------------------------------------------------------------------------- +# Experimental functionality that should be moved into Base / Core when it +# seems ready + +baremodule _Core + +using Base +using Core: CodeInfo + +abstract type CompilerFrontend +end + +_compiler_frontend = nothing + +function _set_compiler_frontend!(frontend) + global _compiler_frontend + old = _compiler_frontend + _compiler_frontend = frontend + return old +end + +""" +parseall(frontend, text; filename="none") + +Parse Julia code with the provided Julia compiler `frontend`. + +TODO: Generalize to allow: +* Replacement of `Core._parse` hook + - Callable from C runtime code + - `Meta.parse()` `parseatom` and `parseall` +* Compiler diagnostics +""" +function parseall +end + +# Incremental lowering API which can manage toplevel and module expressions via +# iteration. The iterator API here is oddly bespoke because `eval()` and +# lowering must pass parameters back and forth. (Lowering to inform eval about +# modules begin/end and eval to create the actual module object and pass it back.) +abstract type TopLevelCodeIterator +end + +struct BeginModule + name::Symbol + standard_defs::Bool # If true, use `Base` and define `eval` and `include` + location # Currently LineNumberNode but might be something else in future +end + +struct EndModule +end + +# Return a subtype of `TopLevelCodeIterator` which may be passed to `lower_step` +function lower_init +end + +# Returns one of +# * `CodeInfo` - top level thunk to evaluate +# * `BeginModule` - start a new module (must be paired with EndModule later in iteration) +# * `EndModule` - finish current module +# * `Nothing` - iteration is finished +function lower_step +end + +#------------------------------------------------------------------------------- +# TODO: C entry points to make it easier for the runtime to call into parsing +# and lowering... ?? +# +# function __parse(text::String, offset::Int, filename::String, rule::Symbol) +# ex, off = parse_code(_compiler_frontend, text, filename; rule=rule) +# svec(ex, off) +# end +# +# function __parse(text::SimpleVector, offset::Int, filename::String, rule::Symbol) +# ?? unsafe_string(text[1], text[2]) +# ?? SubString(text[1]) +# ex, off = parse_code(_compiler_frontend, text, filename; rule=rule) +# svec(ex, off) +# end +# +# function __lower +# end + +#------------------------------------------------------------------------------- +eval(mod::Module, ex; opts...) = eval(_compiler_frontend, mod, ex; opts...) + +function eval(frontend::CompilerFrontend, mod::Module, ex; mapexpr=nothing, opts...) + iter = lower_init(_compiler_frontend, mod, ex, mapexpr; opts...) + simple_eval(mod, iter) +end + +# Barebones `eval()` based on top level iteration API +function simple_eval(mod::Module, thunk::CodeInfo) + # TODO: @ccall jl_eval_thunk instead? + Core.eval(mod, Expr(:thunk, thunk)) +end + +# Shim in case we want extend the allowed types of newmod.location +_module_loc(loc::LineNumberNode) = (loc.file, loc.line) + +if VERSION >= v"1.13.0-DEV.1199" # https://github.com/JuliaLang/julia/pull/59604 + +function simple_eval(mod::Module, newmod::BeginModule) + file, line = _module_loc(newmod.location) + @ccall jl_begin_new_module(mod::Module, newmod.name::Symbol, + newmod.standard_defs::Cint, + file::Cstring, line::Cint)::Module +end + +function simple_eval(mod::Module, ::EndModule) + @ccall jl_end_new_module(mod::Module)::Cvoid + return mod +end + +function simple_eval(mod::Module, iter::TopLevelCodeIterator) + modules = Module[] + new_mod = nothing + result = nothing + while true + thunk = lower_step(iter, Base.get_world_counter(), new_mod) + if isnothing(thunk) + @assert isempty(modules) + return result + end + result = simple_eval(mod, thunk) + if thunk isa BeginModule + push!(modules, mod) + mod = result + new_mod = mod + elseif thunk isa EndModule + result = mod + mod = pop!(modules) + new_mod = nothing + else + new_mod = nothing + end + end +end + +else + +# 1.12 compat +# +# We can't easily implement the following in pre 1.13 Julia. Possibly we could +# do something using a Task to manage the stack frame of Core.eval, but we'd +# need something other than `Module` as the first agument of `simple_eval` in +# order to hold the associated channels and it would be terribly heavy weight. +# +# simple_eval(mod::Module, newmod::BeginModule) +# simple_eval(mod::Module, ::EndModule) + +function simple_eval(mod::Module, iter::TopLevelCodeIterator, new_mod=nothing) + in_new_mod = !isnothing(new_mod) + result = nothing + while true + thunk = lower_step(iter, Base.get_world_counter(), new_mod)::Core.SimpleVector + new_mod = nothing + if isnothing(thunk) + @assert !in_new_mod + return result + elseif thunk isa BeginModule + file, line = _module_loc(thunk) + result = Core.eval(mod, + Expr(:module, thunk.standard_defs, thunk.name, + Expr(:block, LineNumberNode(line, file), + Expr(:call, m->simple_eval(m, iter, m), thunk.name))) + ) + elseif thunk isa EndModule + @assert in_new_mod + return mod + else + @assert thunk isa CodeInfo + result = simple_eval(mod, thunk) + end + end +end + +end + + +#------------------------------------------------------------------------------- +function include_string(frontend::CompilerFrontend, mod::Module, code::AbstractString; + filename::AbstractString="string", mapexpr=nothing, opts...) + ex = parseall(frontend, code; filename=filename) + eval(mod, ex; mapexpr=mapexpr, opts...) +end + +function include_string(mod::Module, code::AbstractString; opts...) + include_string(_compiler_frontend, mod, code; opts...) +end + +# TODO: Simple include() implementation would also be hooked up here. + +#------------------------------------------------------------------------------- +# Julia's builtin flisp-based compiler frontend +struct FlispCompilerFrontend <: CompilerFrontend +end + +# TODO +# function fl_parse(text::Union{Core.SimpleVector,String}, +# filename::String, lineno, offset, options) +# if text isa Core.SimpleVector +# # Will be generated by C entry points jl_parse_string etc +# text, text_len = text +# else +# text_len = sizeof(text) +# end +# ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Csize_t, Any), +# text, text_len, filename, lineno, offset, options) +# end +# +# function fl_parse(text::AbstractString, filename::AbstractString, lineno, offset, options) +# fl_parse(String(text), String(filename), lineno, offset, options) +# end +# +# function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none", +# lineno=0, world::Unsigned=typemax(Csize_t), warn::Bool=false) +# warn = warn ? 1 : 0 +# ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), +# ex, mod, filename, lineno, world, warn) +# end + +#------------------------------------------------------------------------------- +# TODO Current default frontend: JuliaSyntax for parsing plus flisp lowering +# implementation + +struct DefaultFrontend <: CompilerFrontend +end + +#------------------------------------------------------------------------------- + +end # module _Core + + +baremodule _Base + +using Base + +using .._Core + +# TODO: Meta integration... +# +# module _Meta +# +# function parseall +# ... +# end +# +# end + +function include_string(mapexpr::Function, mod::Module, + code::AbstractString, filename::AbstractString="string"; + opts...) + # `identity` is not defined in Core - need to special case it here if we + # want to elide Expr conversion in some cases. + _Core.include_string(mod, code; + mapexpr=(mapexpr===identity ? nothing : mapexpr), + filename=filename, + opts...) +end + +function include_string(mod::Module, code::AbstractString, filename::AbstractString="string"; + opts...) + include_string(identity, mod, code, filename; opts...) +end + +""" + include([mapexpr::Function], mod::Module, path::AbstractString) + +Evaluate the contents of the input source file in the global scope of module +`mod`. Every module (except those defined with baremodule) has its own +definition of `include()` omitting the `mod` argument, which evaluates the file +in that module. Returns the result of the last evaluated expression of the +input file. During including, a task-local include path is set to the directory +containing the file. Nested calls to include will search relative to that path. +This function is typically used to load source interactively, or to combine +files in packages that are broken into multiple source files. +""" +function include(mapexpr::Function, mod::Module, path::AbstractString) + path, prev = Base._include_dependency(mod, path) + code = read(path, String) + tls = task_local_storage() + tls[:SOURCE_PATH] = path + try + return include_string(mapexpr, mod, code, path) + finally + if prev === nothing + delete!(tls, :SOURCE_PATH) + else + tls[:SOURCE_PATH] = prev + end + end +end + +function include(mod::Module, path::AbstractString) + include(identity, mod, path) +end + +end # module _Base + + +#------------------------------------------------------------------------------- +struct JuliaLoweringFrontend <: _Core.CompilerFrontend + # world::UInt # TODO: fixed world age for frontend +end + +function _Core.parseall(::JuliaLoweringFrontend, code::AbstractString; filename="none") + JuliaSyntax.parseall(SyntaxTree, code; filename=filename, ignore_warnings=true) +end + +struct LoweringIterator <: _Core.TopLevelCodeIterator + # frontend::JuliaLoweringFrontend # TODO: world age? + ctx::MacroExpansionContext + todo::Vector{Tuple{SyntaxTree, Bool, Int}} + mapexpr::Any +end + +function _Core.lower_init(::JuliaLoweringFrontend, mod::Module, ex, mapexpr; + expr_compat_mode::Bool=false) + if !(ex isa SyntaxTree) + ex = expr_to_syntaxtree(ex) + else + # TODO: Copy `ex`? We don't want the underlying graph mutated :-( + end + graph = ensure_macro_attributes(syntax_graph(ex)) + dummy_world = zero(UInt) + ctx = MacroExpansionContext(graph, mod, expr_compat_mode, dummy_world) + ex = reparent(ctx, ex) + LoweringIterator(ctx, [(ex, false, 0)], mapexpr) +end + +using ._Core: lower_step, BeginModule, EndModule + +function _Core.lower_step(iter, macro_world, push_mod=nothing) + if !isnothing(push_mod) + push_layer!(iter.ctx, push_mod, false) + end + + if isempty(iter.todo) + return nothing + end + + ex, is_module_body, child_idx = pop!(iter.todo) + if child_idx > 0 + next_child = child_idx + 1 + if child_idx <= numchildren(ex) + push!(iter.todo, (ex, is_module_body, next_child)) + ex = ex[child_idx] + else + if is_module_body + pop_layer!(iter.ctx) + return EndModule() + else + return lower_step(iter, macro_world) + end + end + end + + k = kind(ex) + if !(k in KSet"toplevel module") + if !is_module_body && !isnothing(iter.mapexpr) + # TODO: `mapexpr` is a pretty niche tool and in principle could be + # generically implemented on top of expression iteration if we had + # an option to do that without macro expansion. + ex = iter.ctx.expr_compat_mode ? + expr_to_syntaxtree(iter.ctx, iter.mapexpr(Expr(ex))) : + iter.mapexpr(ex) + end + c = iter.ctx + # Copy context in order to update macro_world + ctx = MacroExpansionContext(c.graph, c.bindings, c.scope_layers, + c.scope_layer_stack, c.expr_compat_mode, macro_world) + ex = expand_forms_1(ctx, ex) + k = kind(ex) + end + if k == K"toplevel" + push!(iter.todo, (ex, false, 1)) + return lower_step(iter, macro_world) + elseif k == K"module" + name = ex[1] + if kind(name) != K"Identifier" + throw(LoweringError(name, "Expected module name")) + end + newmod_name = Symbol(name.name_val) + body = ex[2] + if kind(body) != K"block" + throw(LoweringError(body, "Expected block in module body")) + end + std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG) + loc = source_location(LineNumberNode, ex) + push!(iter.todo, (body, true, 1)) + return BeginModule(newmod_name, std_defs, loc) + else + # Non macro expansion parts of lowering + ctx2, ex2 = expand_forms_2(iter.ctx, ex) + ctx3, ex3 = resolve_scopes(ctx2, ex2) + ctx4, ex4 = convert_closures(ctx3, ex3) + ctx5, ex5 = linearize_ir(ctx4, ex4) + thunk = to_lowered_expr(ex5) + return thunk.args[1] # TODO: Remove wrapping elsewhere? + end +end + + +#------------------------------------------------------------------------------- +# Simple lowering hook. Can be removed once we complete the frontend API above. """ Becomes `Core._lower()` upon activating JuliaLowering. @@ -37,10 +442,6 @@ function core_lowering_hook(@nospecialize(code), mod::Module, end end -# TODO: Write a parser hook here. The input to `core_lowering_hook` should -# eventually be a (convertible to) SyntaxTree, but we need to make updates to -# the parsing API to include a parameter for AST type. - const _has_v1_13_hooks = isdefined(Core, :_lower) function activate!(enable=true) @@ -54,3 +455,16 @@ function activate!(enable=true) Core._setlowerer!(Base.fl_lower) end end + +#------------------------------------------------------------------------------- + +_Core._set_compiler_frontend!(JuliaLoweringFrontend()) + +# Pull implementations from _Base/_Core into JuliaLowering for now +# (Assumes frontend is in _Core not Core.) +@fzone "JL: eval" function eval(mod::Module, ex; opts...) + _Core.eval(mod, ex; opts...) +end + +const include = _Base.include +const include_string = _Base.include_string diff --git a/test/hooks.jl b/test/hooks.jl index d83c2b14..9e73a7fe 100644 --- a/test/hooks.jl +++ b/test/hooks.jl @@ -59,4 +59,21 @@ const JL = JuliaLowering # @test jeval("Base.@propagate_inbounds @inline meta_double_quote_issue(x) = x") isa Function end end + + @testset "CompilerFrontend" begin + _Core = JuliaLowering._Core + old_fe = _Core._set_compiler_frontend!(JuliaLowering.JuliaLoweringFrontend()) + try + # Expr works with eval() + _Core.eval(test_mod, :(xxx = 6)) + @test _Core.eval(test_mod, :(xxx / 2)) == 3 + + # SyntaxTree works with eval() + _Core.eval(test_mod, JuliaLowering.@SyntaxTree :(xxx = 8)) + @test _Core.eval(test_mod, JuliaLowering.@SyntaxTree :(xxx / 2)) == 4 + + finally + _Core._set_compiler_frontend!(old_fe) + end + end end From 1807b472728b814d2be5da2801308dc41cf7845a Mon Sep 17 00:00:00 2001 From: Claire Foster Date: Fri, 26 Sep 2025 20:27:19 +1000 Subject: [PATCH 3/5] fixup! Compiler frontend API and sketch for Core integration --- src/hooks.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hooks.jl b/src/hooks.jl index 10266661..4dfe304c 100644 --- a/src/hooks.jl +++ b/src/hooks.jl @@ -152,13 +152,13 @@ function simple_eval(mod::Module, iter::TopLevelCodeIterator, new_mod=nothing) in_new_mod = !isnothing(new_mod) result = nothing while true - thunk = lower_step(iter, Base.get_world_counter(), new_mod)::Core.SimpleVector + thunk = lower_step(iter, Base.get_world_counter(), new_mod) new_mod = nothing if isnothing(thunk) @assert !in_new_mod return result elseif thunk isa BeginModule - file, line = _module_loc(thunk) + file, line = _module_loc(thunk.location) result = Core.eval(mod, Expr(:module, thunk.standard_defs, thunk.name, Expr(:block, LineNumberNode(line, file), From 29b3aeafd04a300f0e772e32fb40a90b954e4026 Mon Sep 17 00:00:00 2001 From: Claire Foster Date: Sat, 4 Oct 2025 15:36:09 +1000 Subject: [PATCH 4/5] Flisp-based lowering iterator --- src/hooks.jl | 205 +++++++++++++++++++++++++++++++++++++++----------- test/hooks.jl | 97 ++++++++++++++++++++++++ 2 files changed, 260 insertions(+), 42 deletions(-) diff --git a/src/hooks.jl b/src/hooks.jl index 4dfe304c..3cc5d7df 100644 --- a/src/hooks.jl +++ b/src/hooks.jl @@ -49,6 +49,14 @@ end struct EndModule end +# In simple cases flisp lowering may want to produce `CodeInfo` with a trivial +# return inside (for example, `return 1`). In that case we just return a +# `LoweredValue`. +struct LoweredValue + val + location # Currently LineNumberNode +end + # Return a subtype of `TopLevelCodeIterator` which may be passed to `lower_step` function lower_init end @@ -94,6 +102,10 @@ function simple_eval(mod::Module, thunk::CodeInfo) Core.eval(mod, Expr(:thunk, thunk)) end +function simple_eval(mod::Module, v::LoweredValue) + return v.val +end + # Shim in case we want extend the allowed types of newmod.location _module_loc(loc::LineNumberNode) = (loc.file, loc.line) @@ -168,7 +180,6 @@ function simple_eval(mod::Module, iter::TopLevelCodeIterator, new_mod=nothing) @assert in_new_mod return mod else - @assert thunk isa CodeInfo result = simple_eval(mod, thunk) end end @@ -190,44 +201,6 @@ end # TODO: Simple include() implementation would also be hooked up here. -#------------------------------------------------------------------------------- -# Julia's builtin flisp-based compiler frontend -struct FlispCompilerFrontend <: CompilerFrontend -end - -# TODO -# function fl_parse(text::Union{Core.SimpleVector,String}, -# filename::String, lineno, offset, options) -# if text isa Core.SimpleVector -# # Will be generated by C entry points jl_parse_string etc -# text, text_len = text -# else -# text_len = sizeof(text) -# end -# ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Csize_t, Any), -# text, text_len, filename, lineno, offset, options) -# end -# -# function fl_parse(text::AbstractString, filename::AbstractString, lineno, offset, options) -# fl_parse(String(text), String(filename), lineno, offset, options) -# end -# -# function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none", -# lineno=0, world::Unsigned=typemax(Csize_t), warn::Bool=false) -# warn = warn ? 1 : 0 -# ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), -# ex, mod, filename, lineno, world, warn) -# end - -#------------------------------------------------------------------------------- -# TODO Current default frontend: JuliaSyntax for parsing plus flisp lowering -# implementation - -struct DefaultFrontend <: CompilerFrontend -end - -#------------------------------------------------------------------------------- - end # module _Core @@ -244,8 +217,156 @@ using .._Core # function parseall # ... # end -# -# end + +#------------------------------------------------------------------------------- +# Julia's builtin flisp-based compiler frontend +struct FlispCompilerFrontend <: _Core.CompilerFrontend +end + +function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none", + lineno=0, world::Unsigned=typemax(Csize_t), warn::Bool=false) + warn = warn ? 1 : 0 + ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), + ex, mod, filename, lineno, world, warn) +end + +function fl_parse(text::Union{Core.SimpleVector,String}, + filename::String, lineno, offset, options) + if text isa Core.SimpleVector + # Will be generated by C entry points jl_parse_string etc + text, text_len = text + else + text_len = sizeof(text) + end + ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Csize_t, Any), + text, text_len, filename, lineno, offset, options) +end + +function _Core.parseall(frontend::FlispCompilerFrontend, code::AbstractString; + filename="none") + ex, _ = fl_parse(code, filename, 1, 0, :all) + return ex +end + +mutable struct FlispLoweringIterator <: _Core.TopLevelCodeIterator + current_loc::LineNumberNode + do_warn::Bool + todo::Vector{Tuple{Module, Any, Bool, Int}} + mapexpr::Any +end + +function _Core.lower_init(::FlispCompilerFrontend, mod::Module, ex, mapexpr; filename="none", + lineno=0, warn=false, opts...) + FlispLoweringIterator(LineNumberNode(lineno, filename), warn, [(mod, ex, false, 0)], mapexpr) +end + +using ._Core: lower_step, BeginModule, EndModule, LoweredValue + +_expr_head(ex) = ex isa Expr ? ex.head : :none + +function _Core.lower_step(iter::FlispLoweringIterator, macro_world, push_mod=nothing) + if isempty(iter.todo) + return nothing + end + + mod, ex, is_module_body, child_idx = pop!(iter.todo) + + if is_module_body && child_idx == 1 + @assert !isnothing(push_mod) + mod = push_mod + end + + if child_idx > 0 + next_child = child_idx + 1 + if child_idx <= length(ex.args) + push!(iter.todo, (mod, ex, is_module_body, next_child)) + c = ex.args[child_idx] + if c isa LineNumberNode && next_child <= length(ex.args) + iter.current_loc = c + return lower_step(iter, macro_world) + else + ex = c + end + else + if is_module_body + return EndModule() + else + return lower_step(iter, macro_world) + end + end + end + + k = _expr_head(ex) + if !(k in (:toplevel, :module)) + if !is_module_body && !isnothing(iter.mapexpr) + ex = iter.mapexpr(ex) + end + # Expand macros so that we may consume :toplevel from macro expansions + # used at top level. + ex = macroexpand(mod, ex) + k = _expr_head(ex) + end + if k == :toplevel + push!(iter.todo, (mod, ex, false, 1)) + return lower_step(iter, macro_world) + elseif k == :module + if length(ex.args) != 3 + error("syntax: malformed module expression") + end + std_defs = ex.args[1] === true + newmod_name = ex.args[2] + if !(newmod_name isa Symbol) + throw(TypeError(:module, "", Symbol, newmod_name)) + end + body = ex.args[3] + if !(body isa Expr && body.head == :block) + error("syntax: malformed module expression") + end + loc = length(body.args) > 1 && body.args[1] isa LineNumberNode ? + body.args[1] : iter.current_loc + push!(iter.todo, (mod, body, true, 1)) + return BeginModule(newmod_name, std_defs, loc) + elseif ex isa LineNumberNode + # LineNumberNode in value position lowers to `nothing`. (fl_lower + # produces a line number node which then `eval()`'s to nothing but this + # seems incorrect) + return LoweredValue(nothing, ex) + else + # Non macro expansion parts of lowering + do_warn = true # TODO? + thunk = fl_lower(ex, mod, string(iter.current_loc.file), + iter.current_loc.line, macro_world, do_warn)[1] + if thunk isa Expr && thunk.head == :thunk + return thunk.args[1] + else + # For trivial expressions, flisp lowering decides a CodeInfo is not + # required and returns the value itself. + # + # It's unclear whether this is a good idea in the future (what + # about provenance), but for now we use `LoweredValue` for this case. + return LoweredValue(thunk, iter.current_loc) + end + end +end + +#------------------------------------------------------------------------------- +# Current default frontend: Core._parse hook for parsing plus flisp lowering +# implementation + +struct DefaultCompilerFrontend <: _Core.CompilerFrontend +end + +function _Core.parseall(::DefaultCompilerFrontend, code::AbstractString; filename="none") + ex, _ = Core._parse(code, filename, 1, 0, :all) + return ex +end + +function _Core.lower_init(::DefaultCompilerFrontend, mod::Module, ex, mapexpr; + filename="none", lineno=0, warn=false, opts...) + FlispLoweringIterator(LineNumberNode(lineno, filename), warn, [(mod, ex, false, 0)], mapexpr) +end + +#------------------------------------------------------------------------------- function include_string(mapexpr::Function, mod::Module, code::AbstractString, filename::AbstractString="string"; @@ -330,7 +451,7 @@ end using ._Core: lower_step, BeginModule, EndModule -function _Core.lower_step(iter, macro_world, push_mod=nothing) +function _Core.lower_step(iter::LoweringIterator, macro_world, push_mod=nothing) if !isnothing(push_mod) push_layer!(iter.ctx, push_mod, false) end diff --git a/test/hooks.jl b/test/hooks.jl index 9e73a7fe..1696bb0e 100644 --- a/test/hooks.jl +++ b/test/hooks.jl @@ -77,3 +77,100 @@ const JL = JuliaLowering end end end + +@testset "Compiler frontend $frontend" for frontend in [JuliaLowering._Base.FlispCompilerFrontend(), + JuliaLowering._Base.DefaultCompilerFrontend()] + test_mod = Module() + + _Core = JuliaLowering._Core + _Base = JuliaLowering._Base + old_fe = _Core._set_compiler_frontend!(frontend) + try + # Basic top level expressions + @test _Base.include_string(test_mod, + """ + x = 1 + y = 2 + + (x,y) + """) === (1,2) + + # Nested modules and module init order + Amod = _Base.include_string(test_mod, """ + module A + init_order = [] + __init__() = push!(init_order, "A") + module B + using ..A + __init__() = push!(A.init_order, "B") + end + module C + using ..A + __init__() = push!(A.init_order, "C") + module D + using ...A + __init__() = push!(A.init_order, "D") + end + module E + using ...A + __init__() = push!(A.init_order, "E") + end + end + end + """) + Core.@latestworld + @test nameof(Amod) == :A + @test Amod.C.D isa Module + @test Amod.init_order == ["B", "D", "E", "C", "A"] + + # Macro expansion world age + @test _Base.include_string(test_mod, + """ + module ModuleTopLevelEvalWorldTest + macro mac2() + :(101) + end + + xx = @mac2 + end + + ModuleTopLevelEvalWorldTest.xx + """) === 101 + + # Macros expanding to `Expr(:toplevel)` + @test _Base.include_string(test_mod, + """ + macro expand_to_toplevel() + esc(Expr(:toplevel, + :(a = 1), + :(b = 2), + :((a, b)), + )) + end + + @expand_to_toplevel + """) == (1,2) + + # Test that `mapexpr` argument to `include()` is applied + function test_mapexpr(ex) + if ex isa Expr && ex.head != :module + 10101 + else + ex + end + end + @test JuliaLowering.include_string(test_mapexpr, test_mod, """ + module ContentNotMapped + x = 1 # This line won't have mapexpr applied even though it's at + # top level in the module. + end + + this + expression + will + be + replaced + """) === 10101 + Core.@latestworld + @test test_mod.ContentNotMapped.x == 1 + + finally + _Core._set_compiler_frontend!(old_fe) + end +end From ad82a19c37510d7322711a73e167a28d4c177cad Mon Sep 17 00:00:00 2001 From: Claire Foster Date: Wed, 8 Oct 2025 21:31:50 +1000 Subject: [PATCH 5/5] Factor compiler frontend interface into a module --- src/hooks.jl | 277 +++++++++++++++++++++++++++++++++---------------- test/hooks.jl | 2 +- test/macros.jl | 4 +- 3 files changed, 188 insertions(+), 95 deletions(-) diff --git a/src/hooks.jl b/src/hooks.jl index 3cc5d7df..4bfa3872 100644 --- a/src/hooks.jl +++ b/src/hooks.jl @@ -4,33 +4,48 @@ baremodule _Core +module CompilerFrontend + +# Parsing +export AbstractCompilerFrontend, parsecode, syntaxtree, checkparse +# Lowering +export TopLevelCodeIterator, BeginModule, EndModule, LoweredValue, lower_init, lower_step + + using Base -using Core: CodeInfo -abstract type CompilerFrontend +abstract type AbstractCompilerFrontend end -_compiler_frontend = nothing +""" + (parse_result, next_index) = parsecode(frontend, rule, code, first_index; + filename, first_line) -function _set_compiler_frontend!(frontend) - global _compiler_frontend - old = _compiler_frontend - _compiler_frontend = frontend - return old +Parse Julia code with the provided Julia compiler `frontend`. `parse_result` is +a container for the "result of parsing" from which a syntax tree and compiler +errors / diagnostics may be extracted. +""" +function parsecode +end + +""" +tree = syntaxtree(frontend, [tree_type, ] parse_result) + +Return a syntax `tree` of type `tree_type` based on the `parse_result` coming +from `parse_code`. If `tree_type` is absent, return the preferred tree type for +the frontend as may be passed to `lower_init()`. +""" +function syntaxtree end """ -parseall(frontend, text; filename="none") +checkparse(frontend, parse_result) -Parse Julia code with the provided Julia compiler `frontend`. +Check that parsing was successful, throwing an error if not. -TODO: Generalize to allow: -* Replacement of `Core._parse` hook - - Callable from C runtime code - - `Meta.parse()` `parseatom` and `parseall` -* Compiler diagnostics +TODO: Add support for compiler warnings and other diagnostics here? """ -function parseall +function checkparse end # Incremental lowering API which can manage toplevel and module expressions via @@ -69,29 +84,37 @@ end function lower_step end +end # module CompilerFrontend + +using .CompilerFrontend + +using Core: CodeInfo, svec + #------------------------------------------------------------------------------- -# TODO: C entry points to make it easier for the runtime to call into parsing -# and lowering... ?? -# -# function __parse(text::String, offset::Int, filename::String, rule::Symbol) -# ex, off = parse_code(_compiler_frontend, text, filename; rule=rule) -# svec(ex, off) -# end -# -# function __parse(text::SimpleVector, offset::Int, filename::String, rule::Symbol) -# ?? unsafe_string(text[1], text[2]) -# ?? SubString(text[1]) -# ex, off = parse_code(_compiler_frontend, text, filename; rule=rule) -# svec(ex, off) -# end -# -# function __lower -# end +# Default compiler frontend +_compiler_frontend = nothing + +function _set_compiler_frontend!(frontend) + global _compiler_frontend + old = _compiler_frontend + _compiler_frontend = frontend + return old +end + +# Parsing entry point for Julia C runtime code +function _parsecode(code_ptr::Ptr{UInt8}, code_len::Int, offset::Int, filename::String, first_line::Int, rule::Symbol) + code = ccall(:jl_pchar_to_string, Any, (Ptr{UInt8}, UInt #=Csize_t=#), p, len)::String + parse_result, index = parsecode(_compiler_frontend, rule, code, offset+1; filename, first_line) + ex = syntaxtree(Expr, parse_result) + return svec(ex, index-1) +end #------------------------------------------------------------------------------- +# eval implementation + eval(mod::Module, ex; opts...) = eval(_compiler_frontend, mod, ex; opts...) -function eval(frontend::CompilerFrontend, mod::Module, ex; mapexpr=nothing, opts...) +function eval(frontend::AbstractCompilerFrontend, mod::Module, ex; mapexpr=nothing, opts...) iter = lower_init(_compiler_frontend, mod, ex, mapexpr; opts...) simple_eval(mod, iter) end @@ -109,6 +132,9 @@ end # Shim in case we want extend the allowed types of newmod.location _module_loc(loc::LineNumberNode) = (loc.file, loc.line) +# Need to figure these types out for bootstrap +import Base: Base, VERSION, @v_str, >=, Cint, Cstring, Cvoid, @ccall, @assert, push!, pop! + if VERSION >= v"1.13.0-DEV.1199" # https://github.com/JuliaLang/julia/pull/59604 function simple_eval(mod::Module, newmod::BeginModule) @@ -129,8 +155,8 @@ function simple_eval(mod::Module, iter::TopLevelCodeIterator) result = nothing while true thunk = lower_step(iter, Base.get_world_counter(), new_mod) - if isnothing(thunk) - @assert isempty(modules) + if thunk === nothing + # @assert isempty(modules) return result end result = simple_eval(mod, thunk) @@ -187,11 +213,12 @@ end end - #------------------------------------------------------------------------------- -function include_string(frontend::CompilerFrontend, mod::Module, code::AbstractString; +function include_string(frontend::AbstractCompilerFrontend, mod::Module, code::AbstractString; filename::AbstractString="string", mapexpr=nothing, opts...) - ex = parseall(frontend, code; filename=filename) + parse_result, _ = parsecode(frontend, :all, code, 1; filename, first_line=1) + checkparse(frontend, parse_result) + ex = syntaxtree(frontend, parse_result) eval(mod, ex; mapexpr=mapexpr, opts...) end @@ -199,8 +226,6 @@ function include_string(mod::Module, code::AbstractString; opts...) include_string(_compiler_frontend, mod, code; opts...) end -# TODO: Simple include() implementation would also be hooked up here. - end # module _Core @@ -219,35 +244,44 @@ using .._Core # end #------------------------------------------------------------------------------- +using ._Core.CompilerFrontend + # Julia's builtin flisp-based compiler frontend -struct FlispCompilerFrontend <: _Core.CompilerFrontend +struct FlispCompilerFrontend <: AbstractCompilerFrontend end -function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none", - lineno=0, world::Unsigned=typemax(Csize_t), warn::Bool=false) - warn = warn ? 1 : 0 - ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), - ex, mod, filename, lineno, world, warn) +function fl_parse(code::String, + filename::String, first_line, offset, options) + ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Csize_t, Any), + code, sizeof(code), filename, first_line, offset, options) end -function fl_parse(text::Union{Core.SimpleVector,String}, - filename::String, lineno, offset, options) - if text isa Core.SimpleVector - # Will be generated by C entry points jl_parse_string etc - text, text_len = text - else - text_len = sizeof(text) - end - ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Csize_t, Any), - text, text_len, filename, lineno, offset, options) +function CompilerFrontend.parsecode(frontend::FlispCompilerFrontend, rule::Symbol, code, first_index; + filename::String="none", first_line::Int=1) + fl_parse(code, filename, first_line, first_index-1, rule) end -function _Core.parseall(frontend::FlispCompilerFrontend, code::AbstractString; - filename="none") - ex, _ = fl_parse(code, filename, 1, 0, :all) +function CompilerFrontend.syntaxtree(frontend::FlispCompilerFrontend, ex) + syntaxtree(frontend, Expr, ex) +end + +function CompilerFrontend.syntaxtree(frontend::FlispCompilerFrontend, ::Type{Expr}, ex) return ex end +function CompilerFrontend.checkparse(frontend::FlispCompilerFrontend, ex) + if !(ex isa Expr) + return + end + h = ex.head + if h === :toplevel && !isempty(ex.args) + checkparse(frontend, last(ex.args)) + elseif h === :error || h === :incomplete + throw(Meta.ParseError(ex)) + end + return +end + mutable struct FlispLoweringIterator <: _Core.TopLevelCodeIterator current_loc::LineNumberNode do_warn::Bool @@ -255,16 +289,19 @@ mutable struct FlispLoweringIterator <: _Core.TopLevelCodeIterator mapexpr::Any end -function _Core.lower_init(::FlispCompilerFrontend, mod::Module, ex, mapexpr; filename="none", - lineno=0, warn=false, opts...) - FlispLoweringIterator(LineNumberNode(lineno, filename), warn, [(mod, ex, false, 0)], mapexpr) +function CompilerFrontend.lower_init(::FlispCompilerFrontend, mod::Module, ex, mapexpr; + filename="none", first_line=0, warn=false, opts...) + FlispLoweringIterator(LineNumberNode(first_line, filename), warn, [(mod, ex, false, 0)], mapexpr) end -using ._Core: lower_step, BeginModule, EndModule, LoweredValue - -_expr_head(ex) = ex isa Expr ? ex.head : :none +function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none", + first_line=0, world::Unsigned=typemax(Csize_t), warn::Bool=false) + warn = warn ? 1 : 0 + ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), + ex, mod, filename, first_line, world, warn) +end -function _Core.lower_step(iter::FlispLoweringIterator, macro_world, push_mod=nothing) +function CompilerFrontend.lower_step(iter::FlispLoweringIterator, macro_world, push_mod=nothing) if isempty(iter.todo) return nothing end @@ -296,20 +333,20 @@ function _Core.lower_step(iter::FlispLoweringIterator, macro_world, push_mod=not end end - k = _expr_head(ex) - if !(k in (:toplevel, :module)) + h = ex isa Expr ? ex.head : :none + if !(h in (:toplevel, :module)) if !is_module_body && !isnothing(iter.mapexpr) ex = iter.mapexpr(ex) end # Expand macros so that we may consume :toplevel from macro expansions # used at top level. ex = macroexpand(mod, ex) - k = _expr_head(ex) + h = ex isa Expr ? ex.head : :none end - if k == :toplevel + if h == :toplevel push!(iter.todo, (mod, ex, false, 1)) return lower_step(iter, macro_world) - elseif k == :module + elseif h == :module if length(ex.args) != 3 error("syntax: malformed module expression") end @@ -350,20 +387,65 @@ function _Core.lower_step(iter::FlispLoweringIterator, macro_world, push_mod=not end #------------------------------------------------------------------------------- -# Current default frontend: Core._parse hook for parsing plus flisp lowering -# implementation +# Current default frontend: JuliaSyntax for parsing plus flisp lowering +# implementation. (Possibly can go into JuliaSyntax?) + +using JuliaSyntax + +struct DefaultCompilerFrontend <: AbstractCompilerFrontend +end + +struct JuliaSyntaxParseResult + stream::JuliaSyntax.ParseStream + rule::Symbol + filename::String + first_line::Int +end + +function CompilerFrontend.parsecode(::DefaultCompilerFrontend, rule::Symbol, code::AbstractString, + first_index::Integer; filename="none", first_line=1) + stream = JuliaSyntax.ParseStream(code, first_index, version=VERSION) + JuliaSyntax.parse!(stream; rule=rule, incremental=true) + next_byte = JuliaSyntax.last_byte(stream) + 1 + return (JuliaSyntaxParseResult(stream, rule, filename, first_line), next_byte) +end -struct DefaultCompilerFrontend <: _Core.CompilerFrontend +function CompilerFrontend.syntaxtree(frontend::DefaultCompilerFrontend, + res::JuliaSyntaxParseResult) + syntaxtree(frontend, Expr, res) end -function _Core.parseall(::DefaultCompilerFrontend, code::AbstractString; filename="none") - ex, _ = Core._parse(code, filename, 1, 0, :all) +function CompilerFrontend.syntaxtree(::AbstractCompilerFrontend, ::Type{Expr}, + res::JuliaSyntaxParseResult) + stream = res.stream + ex = JuliaSyntax.all_trivia(stream) ? nothing : + JuliaSyntax.build_base_compat_expr(stream, res.rule; + filename=res.filename, first_line=res.first_line) return ex end -function _Core.lower_init(::DefaultCompilerFrontend, mod::Module, ex, mapexpr; - filename="none", lineno=0, warn=false, opts...) - FlispLoweringIterator(LineNumberNode(lineno, filename), warn, [(mod, ex, false, 0)], mapexpr) +function CompilerFrontend.syntaxtree(::AbstractCompilerFrontend, ::Type{T}, + res::JuliaSyntaxParseResult) where {T} + stream = res.stream + ex = JuliaSyntax.all_trivia(stream) ? nothing : + JuliaSyntax.build_tree(T, stream; filename=res.filename, first_line=res.first_line) + return ex +end + +function CompilerFrontend.checkparse(::AbstractCompilerFrontend, res::JuliaSyntaxParseResult; + do_warn=false) + stream = res.stream + if JuliaSyntax.any_error(stream) + throw(JuliaSyntax.ParseError(stream; filename=res.filename, first_line=res.first_line)) + end + # TODO: Show warnings to logger instead of stdout + JuliaSyntax.show_diagnostics(stdout, stream) + nothing +end + +function CompilerFrontend.lower_init(::DefaultCompilerFrontend, mod::Module, ex, mapexpr; + filename="none", first_line=0, warn=false, opts...) + FlispLoweringIterator(LineNumberNode(first_line, filename), warn, [(mod, ex, false, 0)], mapexpr) end #------------------------------------------------------------------------------- @@ -420,23 +502,35 @@ end # module _Base #------------------------------------------------------------------------------- -struct JuliaLoweringFrontend <: _Core.CompilerFrontend +using ._Core.CompilerFrontend +using ._Base: JuliaSyntaxParseResult + +struct JuliaLoweringFrontend <: AbstractCompilerFrontend + expr_compat_mode::Bool # Default compat mode # world::UInt # TODO: fixed world age for frontend end -function _Core.parseall(::JuliaLoweringFrontend, code::AbstractString; filename="none") - JuliaSyntax.parseall(SyntaxTree, code; filename=filename, ignore_warnings=true) +function CompilerFrontend.parsecode(::JuliaLoweringFrontend, rule::Symbol, code::AbstractString, + first_index::Integer; filename="none", first_line=1) + stream = JuliaSyntax.ParseStream(code, first_index, version=VERSION) + JuliaSyntax.parse!(stream; rule=rule, incremental=true) + next_byte = JuliaSyntax.last_byte(stream) + 1 + return (JuliaSyntaxParseResult(stream, rule, filename, first_line), next_byte) end -struct LoweringIterator <: _Core.TopLevelCodeIterator +function CompilerFrontend.syntaxtree(frontend::JuliaLoweringFrontend, parse_result::JuliaSyntaxParseResult) + syntaxtree(frontend, SyntaxTree, parse_result) +end + +struct LoweringIterator <: TopLevelCodeIterator # frontend::JuliaLoweringFrontend # TODO: world age? ctx::MacroExpansionContext todo::Vector{Tuple{SyntaxTree, Bool, Int}} mapexpr::Any end -function _Core.lower_init(::JuliaLoweringFrontend, mod::Module, ex, mapexpr; - expr_compat_mode::Bool=false) +function CompilerFrontend.lower_init(frontend::JuliaLoweringFrontend, mod::Module, ex, mapexpr; + expr_compat_mode::Bool=frontend.expr_compat_mode) if !(ex isa SyntaxTree) ex = expr_to_syntaxtree(ex) else @@ -449,9 +543,7 @@ function _Core.lower_init(::JuliaLoweringFrontend, mod::Module, ex, mapexpr; LoweringIterator(ctx, [(ex, false, 0)], mapexpr) end -using ._Core: lower_step, BeginModule, EndModule - -function _Core.lower_step(iter::LoweringIterator, macro_world, push_mod=nothing) +function CompilerFrontend.lower_step(iter::LoweringIterator, macro_world, push_mod=nothing) if !isnothing(push_mod) push_layer!(iter.ctx, push_mod, false) end @@ -480,8 +572,8 @@ function _Core.lower_step(iter::LoweringIterator, macro_world, push_mod=nothing) if !(k in KSet"toplevel module") if !is_module_body && !isnothing(iter.mapexpr) # TODO: `mapexpr` is a pretty niche tool and in principle could be - # generically implemented on top of expression iteration if we had - # an option to do that without macro expansion. + # implemented more generally on top of expression iteration if we + # had an option to do that without macro expansion. ex = iter.ctx.expr_compat_mode ? expr_to_syntaxtree(iter.ctx, iter.mapexpr(Expr(ex))) : iter.mapexpr(ex) @@ -522,6 +614,7 @@ function _Core.lower_step(iter::LoweringIterator, macro_world, push_mod=nothing) end +#------------------------------------------------------------------------------- #------------------------------------------------------------------------------- # Simple lowering hook. Can be removed once we complete the frontend API above. """ @@ -579,7 +672,7 @@ end #------------------------------------------------------------------------------- -_Core._set_compiler_frontend!(JuliaLoweringFrontend()) +_Core._set_compiler_frontend!(JuliaLoweringFrontend(false)) # Pull implementations from _Base/_Core into JuliaLowering for now # (Assumes frontend is in _Core not Core.) diff --git a/test/hooks.jl b/test/hooks.jl index 1696bb0e..785967e5 100644 --- a/test/hooks.jl +++ b/test/hooks.jl @@ -62,7 +62,7 @@ const JL = JuliaLowering @testset "CompilerFrontend" begin _Core = JuliaLowering._Core - old_fe = _Core._set_compiler_frontend!(JuliaLowering.JuliaLoweringFrontend()) + old_fe = _Core._set_compiler_frontend!(JuliaLowering.JuliaLoweringFrontend(false)) try # Expr works with eval() _Core.eval(test_mod, :(xxx = 6)) diff --git a/test/macros.jl b/test/macros.jl index 5fd42faa..6208c168 100644 --- a/test/macros.jl +++ b/test/macros.jl @@ -340,11 +340,11 @@ end # with K"Placeholder"s @test JuliaLowering.include_string(test_mod, """ __ = 1 - function isglobal_chk(___) + function isglobal_chk_2(___) local ____ = 1 (@isglobal(_), @isglobal(__), @isglobal(___), @isglobal(____)) end - isglobal_chk(1) + isglobal_chk_2(1) """) === (false, false, false, false) # @test appears to be the only macro in base to use :inert