Skip to content

Compiler frontend API and Core integration sketch #85

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Compiler frontend API and Core integration sketch #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4e8ec55
`mapexpr` support + refine incremental lowering API
c42f Sep 26, 2025
f889a14
Compiler frontend API and sketch for Core integration
c42f Sep 26, 2025
1807b47
fixup! Compiler frontend API and sketch for Core integration
c42f Sep 26, 2025
29b3aea
Flisp-based lowering iterator
c42f Oct 4, 2025
ad82a19
Factor compiler frontend interface into a module
c42f Oct 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/compat.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ function expr_to_syntaxtree(@nospecialize(e), lnn::Union{LineNumberNode, Nothing
expr_to_syntaxtree(graph, e, lnn)
end

@fzone "JL: expr_to_syntaxtree" function expr_to_syntaxtree(ctx, @nospecialize(e), lnn::Union{LineNumberNode, Nothing})
@fzone "JL: expr_to_syntaxtree" function expr_to_syntaxtree(ctx, @nospecialize(e), lnn::Union{LineNumberNode, Nothing}=nothing)
graph = syntax_graph(ctx)
toplevel_src = if isnothing(lnn)
# Provenance sinkhole for all nodes until we hit a linenode
Expand Down
196 changes: 0 additions & 196 deletions src/eval.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,94 +15,6 @@ function macroexpand(mod::Module, ex; expr_compat_mode=false, world=Base.get_wor
ex1
end

# Incremental lowering API which can manage toplevel and module expressions.
#
# This iteration API is oddly bespoke and arguably somewhat non-Julian for two
# reasons:
#
# * Lowering knows when new modules are required, and may request them with
# `:begin_module`. However `eval()` generates those modules so they need to
# be passed back into lowering. So we can't just use `Base.iterate()`. (Put a
# different way, we have a situation which is suited to coroutines but we
# don't want to use full Julia `Task`s for this.)
# * We might want to implement this `eval()` in Julia's C runtime code or early
# in bootstrap. Hence using SimpleVector and Symbol as the return values of
# `lower_step()`
#
# We might consider changing at least the second of these choices, depending on
# how we end up putting this into Base.

struct LoweringIterator{GraphType}
ctx::MacroExpansionContext{GraphType}
todo::Vector{Tuple{SyntaxTree{GraphType}, Bool, Int}}
end

function lower_init(ex::SyntaxTree, mod::Module, macro_world::UInt; expr_compat_mode::Bool=false)
graph = ensure_macro_attributes(syntax_graph(ex))
ctx = MacroExpansionContext(graph, mod, expr_compat_mode, macro_world)
ex = reparent(ctx, ex)
LoweringIterator{typeof(graph)}(ctx, [(ex, false, 0)])
end

function lower_step(iter, push_mod=nothing)
if !isnothing(push_mod)
push_layer!(iter.ctx, push_mod, false)
end

if isempty(iter.todo)
return Core.svec(:done)
end

ex, is_module_body, child_idx = pop!(iter.todo)
if child_idx > 0
next_child = child_idx + 1
if child_idx <= numchildren(ex)
push!(iter.todo, (ex, is_module_body, next_child))
ex = ex[child_idx]
else
if is_module_body
pop_layer!(iter.ctx)
return Core.svec(:end_module)
else
return lower_step(iter)
end
end
end

k = kind(ex)
if !(k in KSet"toplevel module")
ex = expand_forms_1(iter.ctx, ex)
k = kind(ex)
end
if k == K"toplevel"
push!(iter.todo, (ex, false, 1))
return lower_step(iter)
elseif k == K"module"
name = ex[1]
if kind(name) != K"Identifier"
throw(LoweringError(name, "Expected module name"))
end
newmod_name = Symbol(name.name_val)
body = ex[2]
if kind(body) != K"block"
throw(LoweringError(body, "Expected block in module body"))
end
std_defs = !has_flags(ex, JuliaSyntax.BARE_MODULE_FLAG)
loc = source_location(LineNumberNode, ex)
push!(iter.todo, (body, true, 1))
return Core.svec(:begin_module, newmod_name, std_defs, loc)
else
# Non macro expansion parts of lowering
ctx2, ex2 = expand_forms_2(iter.ctx, ex)
ctx3, ex3 = resolve_scopes(ctx2, ex2)
ctx4, ex4 = convert_closures(ctx3, ex3)
ctx5, ex5 = linearize_ir(ctx4, ex4)
thunk = to_lowered_expr(ex5)
return Core.svec(:thunk, thunk)
end
end


#-------------------------------------------------------------------------------

function codeinfo_has_image_globalref(@nospecialize(e))
Expand Down Expand Up @@ -449,111 +361,3 @@ function _to_lowered_expr(ex::SyntaxTree, stmt_offset::Int)
end
end

#-------------------------------------------------------------------------------
# Our version of eval - should be upstreamed though?
@fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree;
macro_world::UInt=Base.get_world_counter(),
opts...)
iter = lower_init(ex, mod, macro_world; opts...)
_eval(mod, iter)
end

if VERSION >= v"1.13.0-DEV.1199" # https://github.com/JuliaLang/julia/pull/59604

function _eval(mod, iter)
modules = Module[]
new_mod = nothing
result = nothing
while true
thunk = lower_step(iter, new_mod)::Core.SimpleVector
new_mod = nothing
type = thunk[1]::Symbol
if type == :done
break
elseif type == :begin_module
push!(modules, mod)
mod = @ccall jl_begin_new_module(mod::Any, thunk[2]::Symbol, thunk[3]::Cint,
thunk[4].file::Cstring, thunk[4].line::Cint)::Module
new_mod = mod
elseif type == :end_module
@ccall jl_end_new_module(mod::Module)::Cvoid
result = mod
mod = pop!(modules)
else
@assert type == :thunk
result = Core.eval(mod, thunk[2])
end
end
@assert isempty(modules)
return result
end

else

function _eval(mod, iter, new_mod=nothing)
in_new_mod = !isnothing(new_mod)
result = nothing
while true
thunk = lower_step(iter, new_mod)::Core.SimpleVector
new_mod = nothing
type = thunk[1]::Symbol
if type == :done
@assert !in_new_mod
break
elseif type == :begin_module
name = thunk[2]::Symbol
std_defs = thunk[3]
result = Core.eval(mod,
Expr(:module, std_defs, name,
Expr(:block, thunk[4], Expr(:call, m->_eval(m, iter, m), name)))
)
elseif type == :end_module
@assert in_new_mod
return mod
else
@assert type == :thunk
result = Core.eval(mod, thunk[2])
end
end
return result
end

end

"""
include(mod::Module, path::AbstractString)

Evaluate the contents of the input source file in the global scope of module
`mod`. Every module (except those defined with baremodule) has its own
definition of `include()` omitting the `mod` argument, which evaluates the file
in that module. Returns the result of the last evaluated expression of the
input file. During including, a task-local include path is set to the directory
containing the file. Nested calls to include will search relative to that path.
This function is typically used to load source interactively, or to combine
files in packages that are broken into multiple source files.
"""
function include(mod::Module, path::AbstractString)
path, prev = Base._include_dependency(mod, path)
code = read(path, String)
tls = task_local_storage()
tls[:SOURCE_PATH] = path
try
return include_string(mod, code, path)
finally
if prev === nothing
delete!(tls, :SOURCE_PATH)
else
tls[:SOURCE_PATH] = prev
end
end
end

"""
include_string(mod::Module, code::AbstractString, filename::AbstractString="string")

Like `include`, except reads code from the given string rather than from a file.
"""
function include_string(mod::Module, code::AbstractString, filename::AbstractString="string";
expr_compat_mode=false)
eval(mod, parseall(SyntaxTree, code; filename=filename); expr_compat_mode)
end
Loading
Loading