diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index e22a7d980f06c..5652b9738936c 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -375,8 +375,9 @@ const _return_type = Compiler.return_type # Enable compiler Compiler.bootstrap!() -include("flparse.jl") +include("flfrontend.jl") Core._setparser!(fl_parse) +Core._setlowerer!(fl_lower) # Further definition of Base will happen in Base.jl if loaded. diff --git a/base/boot.jl b/base/boot.jl index f9208c3874229..e97d3dea7927d 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -998,8 +998,9 @@ function struct_name_shim(@nospecialize(x), name::Symbol, mod::Module, @nospecia return x === mod ? t : getfield(x, name) end -# Binding for the julia parser, called as -# +# Bindings for the julia frontend. The internal jl_parse and jl_lower will call +# Core._parse and Core._lower respectively (if they are not `nothing`.) + # Core._parse(text, filename, lineno, offset, options) # # Parse Julia code from the buffer `text`, starting at `offset` and attributing @@ -1009,11 +1010,17 @@ end # # `_parse` must return an `svec` containing an `Expr` and the new offset as an # `Int`. -# -# The internal jl_parse will call into Core._parse if not `nothing`. _parse = nothing +# Core._lower(code, module, filename="none", linenum=0, world=0xfff..., warn=false) +# +# Lower `code` (usually Expr), returning `svec(e::Any xs::Any...)` where `e` is +# the lowered code, and `xs` is possible additional information from +# JuliaLowering (TBD). +_lower = nothing + _setparser!(parser) = setglobal!(Core, :_parse, parser) +_setlowerer!(lowerer) = setglobal!(Core, :_lower, lowerer) # support for deprecated uses of builtin functions _apply(x...) = _apply_iterate(Main.Base.iterate, x...) diff --git a/base/expr.jl b/base/expr.jl index dd357736bc529..9b46795fc7ff9 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -1669,7 +1669,8 @@ end # Implementation of generated functions function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool) - ci = ccall(:jl_lower_expr_mod, Any, (Any, Any), ex, defmod) + ci = ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), + ex, defmod, "none", 0, typemax(Csize_t), 0)[1] if !isa(ci, CodeInfo) if isa(ci, Expr) && ci.head === :error msg = ci.args[1] diff --git a/base/flparse.jl b/base/flfrontend.jl similarity index 71% rename from base/flparse.jl rename to base/flfrontend.jl index 8b474cf148fb2..86b291cf7328b 100644 --- a/base/flparse.jl +++ b/base/flfrontend.jl @@ -17,3 +17,10 @@ end function fl_parse(text::AbstractString, filename::AbstractString, lineno, offset, options) fl_parse(String(text), String(filename), lineno, offset, options) end + +function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none", + lineno=0, world::Unsigned=typemax(Csize_t), warn::Bool=false) + warn = warn ? 1 : 0 + ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), + ex, mod, filename, lineno, world, warn) +end diff --git a/base/loading.jl b/base/loading.jl index b26a063247169..31faa86027b90 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2808,7 +2808,11 @@ function include_string(mapexpr::Function, mod::Module, code::AbstractString, loc = LineNumberNode(1, Symbol(filename)) try ast = Meta.parseall(code, filename=filename) - @assert Meta.isexpr(ast, :toplevel) + if !Meta.isexpr(ast, :toplevel) + @assert Core._lower != fl_lower + # Only reached when JuliaLowering and alternate parse functions are activated + return Core.eval(mod, ast) + end result = nothing line_and_ex = Expr(:toplevel, loc, nothing) for ex in ast.args diff --git a/base/meta.jl b/base/meta.jl index 32c78ce0639f8..5d880a7442b3e 100644 --- a/base/meta.jl +++ b/base/meta.jl @@ -160,7 +160,7 @@ Takes the expression `x` and returns an equivalent expression in lowered form for executing in module `m`. See also [`code_lowered`](@ref). """ -lower(m::Module, @nospecialize(x)) = ccall(:jl_lower_expr_mod, Any, (Any, Any), x, m) +lower(m::Module, @nospecialize(x)) = Core._lower(x, m, "none", 0, typemax(Csize_t), false)[1] """ @lower [m] x diff --git a/doc/src/devdocs/eval.md b/doc/src/devdocs/eval.md index e2c4adb741fcc..82bedc80116fe 100644 --- a/doc/src/devdocs/eval.md +++ b/doc/src/devdocs/eval.md @@ -89,7 +89,7 @@ the expression. Macro expansion involves a handoff from [`eval()`](@ref) (in Jul function `jl_macroexpand()` (written in `flisp`) to the Julia macro itself (written in - what else - Julia) via `fl_invoke_julia_macro()`, and back. -Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`jl_lower_expr_mod()`, +Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`Core._lower()`, although it can also be invoked directly by a call to [`macroexpand()`](@ref)/`jl_macroexpand()`. ## [Type Inference](@id dev-type-inference) diff --git a/src/ast.c b/src/ast.c index b7edaac43f134..0e941f1d1fae8 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1275,13 +1275,12 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule return expr; } -// Main entry point to flisp lowering. Most arguments are optional; see `jl_lower_expr_mod`. // warn: Print any lowering warnings returned; otherwise ignore JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line, size_t world, bool_t warn) + const char *filename, int line, size_t world, bool_t warn) { JL_TIMING(LOWERING, LOWERING); - jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); + jl_timing_show_location(filename, line, inmodule, JL_TIMING_DEFAULT_BLOCK); jl_array_t *kwargs = NULL; JL_GC_PUSH3(&expr, &kwargs, &inmodule); expr = jl_copy_ast(expr); @@ -1290,7 +1289,7 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, fl_context_t *fl_ctx = &ctx->fl; value_t arg = julia_to_scm(fl_ctx, expr); value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg, - symbol(fl_ctx, file), fixnum(line)); + symbol(fl_ctx, filename), fixnum(line)); value_t lwr = car_(e); value_t warnings = car_(cdr_(e)); expr = scm_to_julia(fl_ctx, lwr, inmodule); @@ -1306,6 +1305,7 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, jl_error("julia-logmsg: bad argument list - expected " ":warn level (symbol) group (symbol) id file line msg . kwargs"); } + JL_GC_PUSH1(&warning); jl_value_t *level = jl_exprarg(warning, 0); jl_value_t *group = jl_exprarg(warning, 1); jl_value_t *id = jl_exprarg(warning, 2); @@ -1318,22 +1318,45 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, } JL_TYPECHK(logmsg, long, level); jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg); + JL_GC_POP(); } + jl_value_t *result = (jl_value_t *)jl_svec1(expr); JL_GC_POP(); - return expr; + return result; } -// Lower an expression tree into Julia's intermediate-representation. +// Main C entry point to lowering. Calls jl_fl_lower during bootstrap, and +// Core._lower otherwise (this is also jl_fl_lower unless we have JuliaLowering) JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line, size_t world, bool_t warn) + const char *filename, int line, size_t world, bool_t warn) { - // TODO: Allow change of lowerer - return jl_fl_lower(expr, inmodule, file, line, world, warn); -} - -JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule) -{ - return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0); + jl_value_t *core_lower = NULL; + if (jl_core_module) { + core_lower = jl_get_global(jl_core_module, jl_symbol("_lower")); + } + if (!core_lower || core_lower == jl_nothing) { + return jl_fl_lower(expr, inmodule, filename, line, world, warn); + } + jl_value_t **args; + JL_GC_PUSHARGS(args, 7); + args[0] = core_lower; + args[1] = expr; + args[2] = (jl_value_t*)inmodule; + args[3] = jl_cstr_to_string(filename); + args[4] = jl_box_ulong(line); + args[5] = jl_box_ulong(world); + args[6] = warn ? jl_true : jl_false; + jl_task_t *ct = jl_current_task; + size_t last_age = ct->world_age; + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + jl_value_t *result = jl_apply(args, 7); + ct->world_age = last_age; + args[0] = result; // root during error check below + JL_TYPECHK(parse, simplevector, result); + if (jl_svec_len(result) < 1) + jl_error("Result from lowering should be `svec(a::Any, x::Any...)`"); + JL_GC_POP(); + return result; } jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 6cb9ab630a4b1..9f57de6420c52 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -285,7 +285,6 @@ XX(jl_load_file_string) \ XX(jl_lookup_code_address) \ XX(jl_lower) \ - XX(jl_lower_expr_mod) \ XX(jl_lseek) \ XX(jl_lstat) \ XX(jl_macroexpand) \ diff --git a/src/julia.h b/src/julia.h index 195c23f20662f..b1a4c583329f6 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2260,7 +2260,6 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len, JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, const char *file, int line, size_t world, bool_t warn); -JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule); // deprecated; use jl_parse_all JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *text, size_t text_len, const char *filename, size_t filename_len); diff --git a/src/toplevel.c b/src/toplevel.c index 64a559ac774e6..7d2d209059f1c 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -195,8 +195,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex for (int i = 0; i < jl_array_nrows(exprs); i++) { // process toplevel form - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - form = jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0); + form = jl_svecref(jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0), 0); ct->world_age = jl_atomic_load_acquire(&jl_world_counter); (void)jl_toplevel_eval_flex(newm, form, 1, 1, &filename, &lineno); } @@ -806,10 +805,8 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val JL_GC_PUSH4(&mfunc, &thk, &ex, &root); size_t last_age = ct->world_age; - if (!expanded && jl_needs_lowering(e)) { - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - ex = (jl_expr_t*)jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1); - ct->world_age = last_age; + if (!expanded && (jl_needs_lowering(e))) { + ex = (jl_expr_t*)jl_svecref(jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1), 0); } jl_sym_t *head = jl_is_expr(ex) ? ex->head : NULL; @@ -971,8 +968,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val for (i = 0; i < jl_array_nrows(ex->args); i++) { root = jl_array_ptr_ref(ex->args, i); if (jl_needs_lowering(root)) { - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - root = jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1); + root = jl_svecref(jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1), 0); } ct->world_age = jl_atomic_load_acquire(&jl_world_counter); res = jl_toplevel_eval_flex(m, root, fast, 1, toplevel_filename, toplevel_lineno); @@ -1150,8 +1146,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, jl_lineno = lineno; continue; } - ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); - expression = jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1); + expression = jl_svecref(jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1), 0); ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); result = jl_toplevel_eval_flex(module, expression, 1, 1, &filename_str, &lineno); } diff --git a/stdlib/InteractiveUtils/src/macros.jl b/stdlib/InteractiveUtils/src/macros.jl index 9c59dea132664..aa4092014361e 100644 --- a/stdlib/InteractiveUtils/src/macros.jl +++ b/stdlib/InteractiveUtils/src/macros.jl @@ -558,8 +558,8 @@ in the current environment. When using `@activate`, additional options for a component may be specified in square brackets `@activate Compiler[:option1, :option]` -Currently `@activate Compiler` is the only available component that may be -activatived. +Currently `Compiler` and `JuliaLowering` are the only available components that +may be activatived. For `@activate Compiler`, the following options are available: 1. `:reflection` - Activate the compiler for reflection purposes only. @@ -593,7 +593,7 @@ macro activate(what) if !isa(Component, Symbol) error("Usage Error: Component $Component is not a symbol") end - allowed_components = (:Compiler,) + allowed_components = (:Compiler, :JuliaLowering) if !(Component in allowed_components) error("Usage Error: Component $Component is not recognized. Expected one of $allowed_components") end diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index f6314ed92ee53..637ac1ef6de7d 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -298,12 +298,12 @@ const install_packages_hooks = Any[] # N.B.: Any functions starting with __repl_entry cut off backtraces when printing in the REPL. # We need to do this for both the actual eval and macroexpand, since the latter can cause custom macro # code to run (and error). -__repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) = - ccall(:jl_lower, Any, (Any, Any, Ptr{UInt8}, Cint, Csize_t, Cint), ast, mod, toplevel_file[], toplevel_line[], typemax(Csize_t), 0) -__repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) = - ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Cint}), mod, ast, 1, 1, toplevel_file, toplevel_line) +__repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Csize_t}) = + Core._lower(ast, mod, toplevel_file[], toplevel_line[])[1] +__repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Csize_t}) = + ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Csize_t}), mod, ast, 1, 1, toplevel_file, toplevel_line) -function toplevel_eval_with_hooks(mod::Module, @nospecialize(ast), toplevel_file=Ref{Ptr{UInt8}}(Base.unsafe_convert(Ptr{UInt8}, :REPL)), toplevel_line=Ref{Cint}(1)) +function toplevel_eval_with_hooks(mod::Module, @nospecialize(ast), toplevel_file=Ref{Ptr{UInt8}}(Base.unsafe_convert(Ptr{UInt8}, :REPL)), toplevel_line=Ref{Csize_t}(1)) if !isexpr(ast, :toplevel) ast = invokelatest(__repl_entry_lower_with_loc, mod, ast, toplevel_file, toplevel_line) check_for_missing_packages_and_run_hooks(ast) diff --git a/test/meta.jl b/test/meta.jl index f8707bc36791c..3d5fc08ee24e9 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -234,7 +234,7 @@ let ex = Meta.parseall("@foo", filename=:bar) @test isa(arg2arg2, LineNumberNode) && arg2arg2.file === :bar end -_lower(m::Module, ex, world::UInt) = ccall(:jl_lower, Any, (Any, Ref{Module}, Cstring, Cint, Csize_t, Cint), ex, m, "none", 0, world, 0) +_lower(m::Module, ex, world::UInt) = Base.fl_lower(ex, m, "none", 0, world, false)[1] module TestExpandInWorldModule macro m() 1 end