diff --git a/src/compat.jl b/src/compat.jl index bb2edc3e..72967e3e 100644 --- a/src/compat.jl +++ b/src/compat.jl @@ -29,7 +29,7 @@ function expr_to_syntaxtree(@nospecialize(e), lnn::Union{LineNumberNode, Nothing expr_to_syntaxtree(graph, e, lnn) end -@fzone "JL: expr_to_syntaxtree" function expr_to_syntaxtree(ctx, @nospecialize(e), lnn::Union{LineNumberNode, Nothing}) +@fzone "JL: expr_to_syntaxtree" function expr_to_syntaxtree(ctx, @nospecialize(e), lnn::Union{LineNumberNode, Nothing}=nothing) graph = syntax_graph(ctx) toplevel_src = if isnothing(lnn) # Provenance sinkhole for all nodes until we hit a linenode diff --git a/src/eval.jl b/src/eval.jl index fe859ae0..b9c3e5fb 100644 --- a/src/eval.jl +++ b/src/eval.jl @@ -15,94 +15,6 @@ function macroexpand(mod::Module, ex; expr_compat_mode=false, world=Base.get_wor ex1 end -# Incremental lowering API which can manage toplevel and module expressions. -# -# This iteration API is oddly bespoke and arguably somewhat non-Julian for two -# reasons: -# -# * Lowering knows when new modules are required, and may request them with -# `:begin_module`. However `eval()` generates those modules so they need to -# be passed back into lowering. So we can't just use `Base.iterate()`. (Put a -# different way, we have a situation which is suited to coroutines but we -# don't want to use full Julia `Task`s for this.) -# * We might want to implement this `eval()` in Julia's C runtime code or early -# in bootstrap. Hence using SimpleVector and Symbol as the return values of -# `lower_step()` -# -# We might consider changing at least the second of these choices, depending on -# how we end up putting this into Base. - -struct LoweringIterator{GraphType} - ctx::MacroExpansionContext{GraphType} - todo::Vector{Tuple{SyntaxTree{GraphType}, Bool, Int}} -end - -function lower_init(ex::SyntaxTree, mod::Module, macro_world::UInt; expr_compat_mode::Bool=false) - graph = ensure_macro_attributes(syntax_graph(ex)) - ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world) - ex = reparent(ctx, ex) - LoweringIterator{typeof(graph)}(ctx, [(ex, false, 0)]) -end - -function lower_step(iter, push_mod=nothing) - if !isnothing(push_mod) - push_layer!(iter.ctx, push_mod, false) - end - - if isempty(iter.todo) - return Core.svec(:done) - end - - ex, is_module_body, child_idx = pop!(iter.todo) - if child_idx > 0 - next_child = child_idx + 1 - if child_idx <= numchildren(ex) - push!(iter.todo, (ex, is_module_body, next_child)) - ex = ex[child_idx] - else - if is_module_body - pop_layer!(iter.ctx) - return Core.svec(:end_module) - else - return lower_step(iter) - end - end - end - - k = kind(ex) - if !(k in KSet"toplevel module") - ex = expand_forms_1(iter.ctx, ex) - k = kind(ex) - end - if k == K"toplevel" - push!(iter.todo, (ex, false, 1)) - return lower_step(iter) - elseif k == K"module" - name = ex[1] - if kind(name) != K"Identifier" - throw(LoweringError(name, "Expected module name")) - end - newmod_name = Symbol(name.name_val) - body = ex[2] - if kind(body) != K"block" - throw(LoweringError(body, "Expected block in module body")) - end - std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG) - loc = source_location(LineNumberNode, ex) - push!(iter.todo, (body, true, 1)) - return Core.svec(:begin_module, newmod_name, std_defs, loc) - else - # Non macro expansion parts of lowering - ctx2, ex2 = expand_forms_2(iter.ctx, ex) - ctx3, ex3 = resolve_scopes(ctx2, ex2) - ctx4, ex4 = convert_closures(ctx3, ex3) - ctx5, ex5 = linearize_ir(ctx4, ex4) - thunk = to_lowered_expr(ex5) - return Core.svec(:thunk, thunk) - end -end - - #------------------------------------------------------------------------------- function codeinfo_has_image_globalref(@nospecialize(e)) @@ -449,111 +361,3 @@ function _to_lowered_expr(ex::SyntaxTree, stmt_offset::Int) end end -#------------------------------------------------------------------------------- -# Our version of eval - should be upstreamed though? -@fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree; - macro_world::UInt=Base.get_world_counter(), - opts...) - iter = lower_init(ex, mod, macro_world; opts...) - _eval(mod, iter) -end - -if VERSION >= v"1.13.0-DEV.1199" # https://github.com/JuliaLang/julia/pull/59604 - -function _eval(mod, iter) - modules = Module[] - new_mod = nothing - result = nothing - while true - thunk = lower_step(iter, new_mod)::Core.SimpleVector - new_mod = nothing - type = thunk[1]::Symbol - if type == :done - break - elseif type == :begin_module - push!(modules, mod) - mod = @ccall jl_begin_new_module(mod::Any, thunk[2]::Symbol, thunk[3]::Cint, - thunk[4].file::Cstring, thunk[4].line::Cint)::Module - new_mod = mod - elseif type == :end_module - @ccall jl_end_new_module(mod::Module)::Cvoid - result = mod - mod = pop!(modules) - else - @assert type == :thunk - result = Core.eval(mod, thunk[2]) - end - end - @assert isempty(modules) - return result -end - -else - -function _eval(mod, iter, new_mod=nothing) - in_new_mod = !isnothing(new_mod) - result = nothing - while true - thunk = lower_step(iter, new_mod)::Core.SimpleVector - new_mod = nothing - type = thunk[1]::Symbol - if type == :done - @assert !in_new_mod - break - elseif type == :begin_module - name = thunk[2]::Symbol - std_defs = thunk[3] - result = Core.eval(mod, - Expr(:module, std_defs, name, - Expr(:block, thunk[4], Expr(:call, m->_eval(m, iter, m), name))) - ) - elseif type == :end_module - @assert in_new_mod - return mod - else - @assert type == :thunk - result = Core.eval(mod, thunk[2]) - end - end - return result -end - -end - -""" - include(mod::Module, path::AbstractString) - -Evaluate the contents of the input source file in the global scope of module -`mod`. Every module (except those defined with baremodule) has its own -definition of `include()` omitting the `mod` argument, which evaluates the file -in that module. Returns the result of the last evaluated expression of the -input file. During including, a task-local include path is set to the directory -containing the file. Nested calls to include will search relative to that path. -This function is typically used to load source interactively, or to combine -files in packages that are broken into multiple source files. -""" -function include(mod::Module, path::AbstractString) - path, prev = Base._include_dependency(mod, path) - code = read(path, String) - tls = task_local_storage() - tls[:SOURCE_PATH] = path - try - return include_string(mod, code, path) - finally - if prev === nothing - delete!(tls, :SOURCE_PATH) - else - tls[:SOURCE_PATH] = prev - end - end -end - -""" - include_string(mod::Module, code::AbstractString, filename::AbstractString="string") - -Like `include`, except reads code from the given string rather than from a file. -""" -function include_string(mod::Module, code::AbstractString, filename::AbstractString="string"; - expr_compat_mode=false) - eval(mod, parseall(SyntaxTree, code; filename=filename); expr_compat_mode) -end diff --git a/src/hooks.jl b/src/hooks.jl index 31157632..4bfa3872 100644 --- a/src/hooks.jl +++ b/src/hooks.jl @@ -1,3 +1,622 @@ +#------------------------------------------------------------------------------- +# Experimental functionality that should be moved into Base / Core when it +# seems ready + +baremodule _Core + +module CompilerFrontend + +# Parsing +export AbstractCompilerFrontend, parsecode, syntaxtree, checkparse +# Lowering +export TopLevelCodeIterator, BeginModule, EndModule, LoweredValue, lower_init, lower_step + + +using Base + +abstract type AbstractCompilerFrontend +end + +""" + (parse_result, next_index) = parsecode(frontend, rule, code, first_index; + filename, first_line) + +Parse Julia code with the provided Julia compiler `frontend`. `parse_result` is +a container for the "result of parsing" from which a syntax tree and compiler +errors / diagnostics may be extracted. +""" +function parsecode +end + +""" +tree = syntaxtree(frontend, [tree_type, ] parse_result) + +Return a syntax `tree` of type `tree_type` based on the `parse_result` coming +from `parse_code`. If `tree_type` is absent, return the preferred tree type for +the frontend as may be passed to `lower_init()`. +""" +function syntaxtree +end + +""" +checkparse(frontend, parse_result) + +Check that parsing was successful, throwing an error if not. + +TODO: Add support for compiler warnings and other diagnostics here? +""" +function checkparse +end + +# Incremental lowering API which can manage toplevel and module expressions via +# iteration. The iterator API here is oddly bespoke because `eval()` and +# lowering must pass parameters back and forth. (Lowering to inform eval about +# modules begin/end and eval to create the actual module object and pass it back.) +abstract type TopLevelCodeIterator +end + +struct BeginModule + name::Symbol + standard_defs::Bool # If true, use `Base` and define `eval` and `include` + location # Currently LineNumberNode but might be something else in future +end + +struct EndModule +end + +# In simple cases flisp lowering may want to produce `CodeInfo` with a trivial +# return inside (for example, `return 1`). In that case we just return a +# `LoweredValue`. +struct LoweredValue + val + location # Currently LineNumberNode +end + +# Return a subtype of `TopLevelCodeIterator` which may be passed to `lower_step` +function lower_init +end + +# Returns one of +# * `CodeInfo` - top level thunk to evaluate +# * `BeginModule` - start a new module (must be paired with EndModule later in iteration) +# * `EndModule` - finish current module +# * `Nothing` - iteration is finished +function lower_step +end + +end # module CompilerFrontend + +using .CompilerFrontend + +using Core: CodeInfo, svec + +#------------------------------------------------------------------------------- +# Default compiler frontend +_compiler_frontend = nothing + +function _set_compiler_frontend!(frontend) + global _compiler_frontend + old = _compiler_frontend + _compiler_frontend = frontend + return old +end + +# Parsing entry point for Julia C runtime code +function _parsecode(code_ptr::Ptr{UInt8}, code_len::Int, offset::Int, filename::String, first_line::Int, rule::Symbol) + code = ccall(:jl_pchar_to_string, Any, (Ptr{UInt8}, UInt #=Csize_t=#), p, len)::String + parse_result, index = parsecode(_compiler_frontend, rule, code, offset+1; filename, first_line) + ex = syntaxtree(Expr, parse_result) + return svec(ex, index-1) +end + +#------------------------------------------------------------------------------- +# eval implementation + +eval(mod::Module, ex; opts...) = eval(_compiler_frontend, mod, ex; opts...) + +function eval(frontend::AbstractCompilerFrontend, mod::Module, ex; mapexpr=nothing, opts...) + iter = lower_init(_compiler_frontend, mod, ex, mapexpr; opts...) + simple_eval(mod, iter) +end + +# Barebones `eval()` based on top level iteration API +function simple_eval(mod::Module, thunk::CodeInfo) + # TODO: @ccall jl_eval_thunk instead? + Core.eval(mod, Expr(:thunk, thunk)) +end + +function simple_eval(mod::Module, v::LoweredValue) + return v.val +end + +# Shim in case we want extend the allowed types of newmod.location +_module_loc(loc::LineNumberNode) = (loc.file, loc.line) + +# Need to figure these types out for bootstrap +import Base: Base, VERSION, @v_str, >=, Cint, Cstring, Cvoid, @ccall, @assert, push!, pop! + +if VERSION >= v"1.13.0-DEV.1199" # https://github.com/JuliaLang/julia/pull/59604 + +function simple_eval(mod::Module, newmod::BeginModule) + file, line = _module_loc(newmod.location) + @ccall jl_begin_new_module(mod::Module, newmod.name::Symbol, + newmod.standard_defs::Cint, + file::Cstring, line::Cint)::Module +end + +function simple_eval(mod::Module, ::EndModule) + @ccall jl_end_new_module(mod::Module)::Cvoid + return mod +end + +function simple_eval(mod::Module, iter::TopLevelCodeIterator) + modules = Module[] + new_mod = nothing + result = nothing + while true + thunk = lower_step(iter, Base.get_world_counter(), new_mod) + if thunk === nothing + # @assert isempty(modules) + return result + end + result = simple_eval(mod, thunk) + if thunk isa BeginModule + push!(modules, mod) + mod = result + new_mod = mod + elseif thunk isa EndModule + result = mod + mod = pop!(modules) + new_mod = nothing + else + new_mod = nothing + end + end +end + +else + +# 1.12 compat +# +# We can't easily implement the following in pre 1.13 Julia. Possibly we could +# do something using a Task to manage the stack frame of Core.eval, but we'd +# need something other than `Module` as the first agument of `simple_eval` in +# order to hold the associated channels and it would be terribly heavy weight. +# +# simple_eval(mod::Module, newmod::BeginModule) +# simple_eval(mod::Module, ::EndModule) + +function simple_eval(mod::Module, iter::TopLevelCodeIterator, new_mod=nothing) + in_new_mod = !isnothing(new_mod) + result = nothing + while true + thunk = lower_step(iter, Base.get_world_counter(), new_mod) + new_mod = nothing + if isnothing(thunk) + @assert !in_new_mod + return result + elseif thunk isa BeginModule + file, line = _module_loc(thunk.location) + result = Core.eval(mod, + Expr(:module, thunk.standard_defs, thunk.name, + Expr(:block, LineNumberNode(line, file), + Expr(:call, m->simple_eval(m, iter, m), thunk.name))) + ) + elseif thunk isa EndModule + @assert in_new_mod + return mod + else + result = simple_eval(mod, thunk) + end + end +end + +end + +#------------------------------------------------------------------------------- +function include_string(frontend::AbstractCompilerFrontend, mod::Module, code::AbstractString; + filename::AbstractString="string", mapexpr=nothing, opts...) + parse_result, _ = parsecode(frontend, :all, code, 1; filename, first_line=1) + checkparse(frontend, parse_result) + ex = syntaxtree(frontend, parse_result) + eval(mod, ex; mapexpr=mapexpr, opts...) +end + +function include_string(mod::Module, code::AbstractString; opts...) + include_string(_compiler_frontend, mod, code; opts...) +end + +end # module _Core + + +baremodule _Base + +using Base + +using .._Core + +# TODO: Meta integration... +# +# module _Meta +# +# function parseall +# ... +# end + +#------------------------------------------------------------------------------- +using ._Core.CompilerFrontend + +# Julia's builtin flisp-based compiler frontend +struct FlispCompilerFrontend <: AbstractCompilerFrontend +end + +function fl_parse(code::String, + filename::String, first_line, offset, options) + ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Csize_t, Any), + code, sizeof(code), filename, first_line, offset, options) +end + +function CompilerFrontend.parsecode(frontend::FlispCompilerFrontend, rule::Symbol, code, first_index; + filename::String="none", first_line::Int=1) + fl_parse(code, filename, first_line, first_index-1, rule) +end + +function CompilerFrontend.syntaxtree(frontend::FlispCompilerFrontend, ex) + syntaxtree(frontend, Expr, ex) +end + +function CompilerFrontend.syntaxtree(frontend::FlispCompilerFrontend, ::Type{Expr}, ex) + return ex +end + +function CompilerFrontend.checkparse(frontend::FlispCompilerFrontend, ex) + if !(ex isa Expr) + return + end + h = ex.head + if h === :toplevel && !isempty(ex.args) + checkparse(frontend, last(ex.args)) + elseif h === :error || h === :incomplete + throw(Meta.ParseError(ex)) + end + return +end + +mutable struct FlispLoweringIterator <: _Core.TopLevelCodeIterator + current_loc::LineNumberNode + do_warn::Bool + todo::Vector{Tuple{Module, Any, Bool, Int}} + mapexpr::Any +end + +function CompilerFrontend.lower_init(::FlispCompilerFrontend, mod::Module, ex, mapexpr; + filename="none", first_line=0, warn=false, opts...) + FlispLoweringIterator(LineNumberNode(first_line, filename), warn, [(mod, ex, false, 0)], mapexpr) +end + +function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none", + first_line=0, world::Unsigned=typemax(Csize_t), warn::Bool=false) + warn = warn ? 1 : 0 + ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), + ex, mod, filename, first_line, world, warn) +end + +function CompilerFrontend.lower_step(iter::FlispLoweringIterator, macro_world, push_mod=nothing) + if isempty(iter.todo) + return nothing + end + + mod, ex, is_module_body, child_idx = pop!(iter.todo) + + if is_module_body && child_idx == 1 + @assert !isnothing(push_mod) + mod = push_mod + end + + if child_idx > 0 + next_child = child_idx + 1 + if child_idx <= length(ex.args) + push!(iter.todo, (mod, ex, is_module_body, next_child)) + c = ex.args[child_idx] + if c isa LineNumberNode && next_child <= length(ex.args) + iter.current_loc = c + return lower_step(iter, macro_world) + else + ex = c + end + else + if is_module_body + return EndModule() + else + return lower_step(iter, macro_world) + end + end + end + + h = ex isa Expr ? ex.head : :none + if !(h in (:toplevel, :module)) + if !is_module_body && !isnothing(iter.mapexpr) + ex = iter.mapexpr(ex) + end + # Expand macros so that we may consume :toplevel from macro expansions + # used at top level. + ex = macroexpand(mod, ex) + h = ex isa Expr ? ex.head : :none + end + if h == :toplevel + push!(iter.todo, (mod, ex, false, 1)) + return lower_step(iter, macro_world) + elseif h == :module + if length(ex.args) != 3 + error("syntax: malformed module expression") + end + std_defs = ex.args[1] === true + newmod_name = ex.args[2] + if !(newmod_name isa Symbol) + throw(TypeError(:module, "", Symbol, newmod_name)) + end + body = ex.args[3] + if !(body isa Expr && body.head == :block) + error("syntax: malformed module expression") + end + loc = length(body.args) > 1 && body.args[1] isa LineNumberNode ? + body.args[1] : iter.current_loc + push!(iter.todo, (mod, body, true, 1)) + return BeginModule(newmod_name, std_defs, loc) + elseif ex isa LineNumberNode + # LineNumberNode in value position lowers to `nothing`. (fl_lower + # produces a line number node which then `eval()`'s to nothing but this + # seems incorrect) + return LoweredValue(nothing, ex) + else + # Non macro expansion parts of lowering + do_warn = true # TODO? + thunk = fl_lower(ex, mod, string(iter.current_loc.file), + iter.current_loc.line, macro_world, do_warn)[1] + if thunk isa Expr && thunk.head == :thunk + return thunk.args[1] + else + # For trivial expressions, flisp lowering decides a CodeInfo is not + # required and returns the value itself. + # + # It's unclear whether this is a good idea in the future (what + # about provenance), but for now we use `LoweredValue` for this case. + return LoweredValue(thunk, iter.current_loc) + end + end +end + +#------------------------------------------------------------------------------- +# Current default frontend: JuliaSyntax for parsing plus flisp lowering +# implementation. (Possibly can go into JuliaSyntax?) + +using JuliaSyntax + +struct DefaultCompilerFrontend <: AbstractCompilerFrontend +end + +struct JuliaSyntaxParseResult + stream::JuliaSyntax.ParseStream + rule::Symbol + filename::String + first_line::Int +end + +function CompilerFrontend.parsecode(::DefaultCompilerFrontend, rule::Symbol, code::AbstractString, + first_index::Integer; filename="none", first_line=1) + stream = JuliaSyntax.ParseStream(code, first_index, version=VERSION) + JuliaSyntax.parse!(stream; rule=rule, incremental=true) + next_byte = JuliaSyntax.last_byte(stream) + 1 + return (JuliaSyntaxParseResult(stream, rule, filename, first_line), next_byte) +end + +function CompilerFrontend.syntaxtree(frontend::DefaultCompilerFrontend, + res::JuliaSyntaxParseResult) + syntaxtree(frontend, Expr, res) +end + +function CompilerFrontend.syntaxtree(::AbstractCompilerFrontend, ::Type{Expr}, + res::JuliaSyntaxParseResult) + stream = res.stream + ex = JuliaSyntax.all_trivia(stream) ? nothing : + JuliaSyntax.build_base_compat_expr(stream, res.rule; + filename=res.filename, first_line=res.first_line) + return ex +end + +function CompilerFrontend.syntaxtree(::AbstractCompilerFrontend, ::Type{T}, + res::JuliaSyntaxParseResult) where {T} + stream = res.stream + ex = JuliaSyntax.all_trivia(stream) ? nothing : + JuliaSyntax.build_tree(T, stream; filename=res.filename, first_line=res.first_line) + return ex +end + +function CompilerFrontend.checkparse(::AbstractCompilerFrontend, res::JuliaSyntaxParseResult; + do_warn=false) + stream = res.stream + if JuliaSyntax.any_error(stream) + throw(JuliaSyntax.ParseError(stream; filename=res.filename, first_line=res.first_line)) + end + # TODO: Show warnings to logger instead of stdout + JuliaSyntax.show_diagnostics(stdout, stream) + nothing +end + +function CompilerFrontend.lower_init(::DefaultCompilerFrontend, mod::Module, ex, mapexpr; + filename="none", first_line=0, warn=false, opts...) + FlispLoweringIterator(LineNumberNode(first_line, filename), warn, [(mod, ex, false, 0)], mapexpr) +end + +#------------------------------------------------------------------------------- + +function include_string(mapexpr::Function, mod::Module, + code::AbstractString, filename::AbstractString="string"; + opts...) + # `identity` is not defined in Core - need to special case it here if we + # want to elide Expr conversion in some cases. + _Core.include_string(mod, code; + mapexpr=(mapexpr===identity ? nothing : mapexpr), + filename=filename, + opts...) +end + +function include_string(mod::Module, code::AbstractString, filename::AbstractString="string"; + opts...) + include_string(identity, mod, code, filename; opts...) +end + +""" + include([mapexpr::Function], mod::Module, path::AbstractString) + +Evaluate the contents of the input source file in the global scope of module +`mod`. Every module (except those defined with baremodule) has its own +definition of `include()` omitting the `mod` argument, which evaluates the file +in that module. Returns the result of the last evaluated expression of the +input file. During including, a task-local include path is set to the directory +containing the file. Nested calls to include will search relative to that path. +This function is typically used to load source interactively, or to combine +files in packages that are broken into multiple source files. +""" +function include(mapexpr::Function, mod::Module, path::AbstractString) + path, prev = Base._include_dependency(mod, path) + code = read(path, String) + tls = task_local_storage() + tls[:SOURCE_PATH] = path + try + return include_string(mapexpr, mod, code, path) + finally + if prev === nothing + delete!(tls, :SOURCE_PATH) + else + tls[:SOURCE_PATH] = prev + end + end +end + +function include(mod::Module, path::AbstractString) + include(identity, mod, path) +end + +end # module _Base + + +#------------------------------------------------------------------------------- +using ._Core.CompilerFrontend +using ._Base: JuliaSyntaxParseResult + +struct JuliaLoweringFrontend <: AbstractCompilerFrontend + expr_compat_mode::Bool # Default compat mode + # world::UInt # TODO: fixed world age for frontend +end + +function CompilerFrontend.parsecode(::JuliaLoweringFrontend, rule::Symbol, code::AbstractString, + first_index::Integer; filename="none", first_line=1) + stream = JuliaSyntax.ParseStream(code, first_index, version=VERSION) + JuliaSyntax.parse!(stream; rule=rule, incremental=true) + next_byte = JuliaSyntax.last_byte(stream) + 1 + return (JuliaSyntaxParseResult(stream, rule, filename, first_line), next_byte) +end + +function CompilerFrontend.syntaxtree(frontend::JuliaLoweringFrontend, parse_result::JuliaSyntaxParseResult) + syntaxtree(frontend, SyntaxTree, parse_result) +end + +struct LoweringIterator <: TopLevelCodeIterator + # frontend::JuliaLoweringFrontend # TODO: world age? + ctx::MacroExpansionContext + todo::Vector{Tuple{SyntaxTree, Bool, Int}} + mapexpr::Any +end + +function CompilerFrontend.lower_init(frontend::JuliaLoweringFrontend, mod::Module, ex, mapexpr; + expr_compat_mode::Bool=frontend.expr_compat_mode) + if !(ex isa SyntaxTree) + ex = expr_to_syntaxtree(ex) + else + # TODO: Copy `ex`? We don't want the underlying graph mutated :-( + end + graph = ensure_macro_attributes(syntax_graph(ex)) + dummy_world = zero(UInt) + ctx = MacroExpansionContext(graph, mod, expr_compat_mode, dummy_world) + ex = reparent(ctx, ex) + LoweringIterator(ctx, [(ex, false, 0)], mapexpr) +end + +function CompilerFrontend.lower_step(iter::LoweringIterator, macro_world, push_mod=nothing) + if !isnothing(push_mod) + push_layer!(iter.ctx, push_mod, false) + end + + if isempty(iter.todo) + return nothing + end + + ex, is_module_body, child_idx = pop!(iter.todo) + if child_idx > 0 + next_child = child_idx + 1 + if child_idx <= numchildren(ex) + push!(iter.todo, (ex, is_module_body, next_child)) + ex = ex[child_idx] + else + if is_module_body + pop_layer!(iter.ctx) + return EndModule() + else + return lower_step(iter, macro_world) + end + end + end + + k = kind(ex) + if !(k in KSet"toplevel module") + if !is_module_body && !isnothing(iter.mapexpr) + # TODO: `mapexpr` is a pretty niche tool and in principle could be + # implemented more generally on top of expression iteration if we + # had an option to do that without macro expansion. + ex = iter.ctx.expr_compat_mode ? + expr_to_syntaxtree(iter.ctx, iter.mapexpr(Expr(ex))) : + iter.mapexpr(ex) + end + c = iter.ctx + # Copy context in order to update macro_world + ctx = MacroExpansionContext(c.graph, c.bindings, c.scope_layers, + c.scope_layer_stack, c.expr_compat_mode, macro_world) + ex = expand_forms_1(ctx, ex) + k = kind(ex) + end + if k == K"toplevel" + push!(iter.todo, (ex, false, 1)) + return lower_step(iter, macro_world) + elseif k == K"module" + name = ex[1] + if kind(name) != K"Identifier" + throw(LoweringError(name, "Expected module name")) + end + newmod_name = Symbol(name.name_val) + body = ex[2] + if kind(body) != K"block" + throw(LoweringError(body, "Expected block in module body")) + end + std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG) + loc = source_location(LineNumberNode, ex) + push!(iter.todo, (body, true, 1)) + return BeginModule(newmod_name, std_defs, loc) + else + # Non macro expansion parts of lowering + ctx2, ex2 = expand_forms_2(iter.ctx, ex) + ctx3, ex3 = resolve_scopes(ctx2, ex2) + ctx4, ex4 = convert_closures(ctx3, ex3) + ctx5, ex5 = linearize_ir(ctx4, ex4) + thunk = to_lowered_expr(ex5) + return thunk.args[1] # TODO: Remove wrapping elsewhere? + end +end + + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- +# Simple lowering hook. Can be removed once we complete the frontend API above. """ Becomes `Core._lower()` upon activating JuliaLowering. @@ -37,10 +656,6 @@ function core_lowering_hook(@nospecialize(code), mod::Module, end end -# TODO: Write a parser hook here. The input to `core_lowering_hook` should -# eventually be a (convertible to) SyntaxTree, but we need to make updates to -# the parsing API to include a parameter for AST type. - const _has_v1_13_hooks = isdefined(Core, :_lower) function activate!(enable=true) @@ -54,3 +669,16 @@ function activate!(enable=true) Core._setlowerer!(Base.fl_lower) end end + +#------------------------------------------------------------------------------- + +_Core._set_compiler_frontend!(JuliaLoweringFrontend(false)) + +# Pull implementations from _Base/_Core into JuliaLowering for now +# (Assumes frontend is in _Core not Core.) +@fzone "JL: eval" function eval(mod::Module, ex; opts...) + _Core.eval(mod, ex; opts...) +end + +const include = _Base.include +const include_string = _Base.include_string diff --git a/test/hooks.jl b/test/hooks.jl index d83c2b14..785967e5 100644 --- a/test/hooks.jl +++ b/test/hooks.jl @@ -59,4 +59,118 @@ const JL = JuliaLowering # @test jeval("Base.@propagate_inbounds @inline meta_double_quote_issue(x) = x") isa Function end end + + @testset "CompilerFrontend" begin + _Core = JuliaLowering._Core + old_fe = _Core._set_compiler_frontend!(JuliaLowering.JuliaLoweringFrontend(false)) + try + # Expr works with eval() + _Core.eval(test_mod, :(xxx = 6)) + @test _Core.eval(test_mod, :(xxx / 2)) == 3 + + # SyntaxTree works with eval() + _Core.eval(test_mod, JuliaLowering.@SyntaxTree :(xxx = 8)) + @test _Core.eval(test_mod, JuliaLowering.@SyntaxTree :(xxx / 2)) == 4 + + finally + _Core._set_compiler_frontend!(old_fe) + end + end +end + +@testset "Compiler frontend $frontend" for frontend in [JuliaLowering._Base.FlispCompilerFrontend(), + JuliaLowering._Base.DefaultCompilerFrontend()] + test_mod = Module() + + _Core = JuliaLowering._Core + _Base = JuliaLowering._Base + old_fe = _Core._set_compiler_frontend!(frontend) + try + # Basic top level expressions + @test _Base.include_string(test_mod, + """ + x = 1 + y = 2 + + (x,y) + """) === (1,2) + + # Nested modules and module init order + Amod = _Base.include_string(test_mod, """ + module A + init_order = [] + __init__() = push!(init_order, "A") + module B + using ..A + __init__() = push!(A.init_order, "B") + end + module C + using ..A + __init__() = push!(A.init_order, "C") + module D + using ...A + __init__() = push!(A.init_order, "D") + end + module E + using ...A + __init__() = push!(A.init_order, "E") + end + end + end + """) + Core.@latestworld + @test nameof(Amod) == :A + @test Amod.C.D isa Module + @test Amod.init_order == ["B", "D", "E", "C", "A"] + + # Macro expansion world age + @test _Base.include_string(test_mod, + """ + module ModuleTopLevelEvalWorldTest + macro mac2() + :(101) + end + + xx = @mac2 + end + + ModuleTopLevelEvalWorldTest.xx + """) === 101 + + # Macros expanding to `Expr(:toplevel)` + @test _Base.include_string(test_mod, + """ + macro expand_to_toplevel() + esc(Expr(:toplevel, + :(a = 1), + :(b = 2), + :((a, b)), + )) + end + + @expand_to_toplevel + """) == (1,2) + + # Test that `mapexpr` argument to `include()` is applied + function test_mapexpr(ex) + if ex isa Expr && ex.head != :module + 10101 + else + ex + end + end + @test JuliaLowering.include_string(test_mapexpr, test_mod, """ + module ContentNotMapped + x = 1 # This line won't have mapexpr applied even though it's at + # top level in the module. + end + + this + expression + will + be + replaced + """) === 10101 + Core.@latestworld + @test test_mod.ContentNotMapped.x == 1 + + finally + _Core._set_compiler_frontend!(old_fe) + end end diff --git a/test/macros.jl b/test/macros.jl index 5fd42faa..6208c168 100644 --- a/test/macros.jl +++ b/test/macros.jl @@ -340,11 +340,11 @@ end # with K"Placeholder"s @test JuliaLowering.include_string(test_mod, """ __ = 1 - function isglobal_chk(___) + function isglobal_chk_2(___) local ____ = 1 (@isglobal(_), @isglobal(__), @isglobal(___), @isglobal(____)) end - isglobal_chk(1) + isglobal_chk_2(1) """) === (false, false, false, false) # @test appears to be the only macro in base to use :inert diff --git a/test/modules.jl b/test/modules.jl index a68c5f8a..e32e1165 100644 --- a/test/modules.jl +++ b/test/modules.jl @@ -1,4 +1,4 @@ -@testset "modules" begin +@testset "modules and top level code" begin test_mod = Module() @@ -51,4 +51,42 @@ end """) @test Amod.init_order == ["B", "D", "E", "C", "A"] +# Macros in top level and module expressions may be used immediately in the +# next top level statement after their definition +@test JuliaLowering.include_string(test_mod, """ +macro mac1() + :(42) +end + +@mac1 +""") === 42 + +@test JuliaLowering.include_string(test_mod, """ +module ModuleTopLevelEvalWorldTest + macro mac2() + :(101) + end + + xx = @mac2 +end + +ModuleTopLevelEvalWorldTest.xx +""") === 101 + +function test_mapexpr_1(ex::JuliaLowering.SyntaxTree) + JuliaLowering.@ast ex ex 10101::K"Integer" +end + +@test JuliaLowering.include_string(test_mapexpr_1, test_mod, """ +this + expression + will + be + replaced +""") === 10101 + +function test_mapexpr_2(ex::Expr) + 20202 +end + +@test JuliaLowering.include_string(test_mapexpr_2, test_mod, """ +this + expression + will + be + replaced +"""; expr_compat_mode=true) === 20202 + end