Lowering API: Allow for switching to JuliaLowering

base/Base_compiler.jl

@@ -375,8 +375,9 @@ const _return_type = Compiler.return_type
 # Enable compiler
 Compiler.bootstrap!()
 
-include("flparse.jl")
+include("flfrontend.jl")
 Core._setparser!(fl_parse)
+Core._setlowerer!(fl_lower)
 
 # Further definition of Base will happen in Base.jl if loaded.
 

base/boot.jl

@@ -998,8 +998,9 @@ function struct_name_shim(@nospecialize(x), name::Symbol, mod::Module, @nospecia
     return x === mod ? t : getfield(x, name)
 end
 
-# Binding for the julia parser, called as
-#
+# Bindings for the julia frontend.  The internal jl_parse and jl_lower will call
+# Core._parse and Core._lower respectively (if they are not `nothing`.)
+
 #    Core._parse(text, filename, lineno, offset, options)
 #
 # Parse Julia code from the buffer `text`, starting at `offset` and attributing
@@ -1009,11 +1010,17 @@ end
 #
 # `_parse` must return an `svec` containing an `Expr` and the new offset as an
 # `Int`.
-#
-# The internal jl_parse will call into Core._parse if not `nothing`.
 _parse = nothing
 
+#    Core._lower(code, module, filename="none", linenum=0, world=0xfff..., warn=false)
+#
+# Lower `code` (usually Expr), returning `svec(e::Any xs::Any...)` where `e` is
+# the lowered code, and `xs` is possible additional information from
+# JuliaLowering (TBD).
+_lower = nothing
+
 _setparser!(parser) = setglobal!(Core, :_parse, parser)
+_setlowerer!(lowerer) = setglobal!(Core, :_lower, lowerer)
 
 # support for deprecated uses of builtin functions
 _apply(x...) = _apply_iterate(Main.Base.iterate, x...)

base/expr.jl

@@ -1669,7 +1669,8 @@ end
 
 # Implementation of generated functions
 function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool)
-    ci = ccall(:jl_lower_expr_mod, Any, (Any, Any), ex, defmod)
+    ci = ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint),
+               ex, defmod, "none", 0, typemax(Csize_t), 0)[1]
     if !isa(ci, CodeInfo)
         if isa(ci, Expr) && ci.head === :error
             msg = ci.args[1]

base/flparse.jl → base/flfrontend.jl

@@ -17,3 +17,10 @@ end
 function fl_parse(text::AbstractString, filename::AbstractString, lineno, offset, options)
     fl_parse(String(text), String(filename), lineno, offset, options)
 end
+
+function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none",
+                  lineno=0, world::Unsigned=typemax(Csize_t), warn::Bool=false)
+    warn = warn ? 1 : 0
+    ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint),
+          ex, mod, filename, lineno, world, warn)
+end

base/loading.jl

@@ -2808,7 +2808,11 @@ function include_string(mapexpr::Function, mod::Module, code::AbstractString,
     loc = LineNumberNode(1, Symbol(filename))
     try
         ast = Meta.parseall(code, filename=filename)
-        @assert Meta.isexpr(ast, :toplevel)
+        if !Meta.isexpr(ast, :toplevel)
+            @assert Core._lower != fl_lower
+            # Only reached when JuliaLowering and alternate parse functions are activated
+            return Core.eval(mod, ast)
+        end
         result = nothing
         line_and_ex = Expr(:toplevel, loc, nothing)
         for ex in ast.args

base/meta.jl

@@ -160,7 +160,7 @@ Takes the expression `x` and returns an equivalent expression in lowered form
 for executing in module `m`.
 See also [`code_lowered`](@ref).
 """
-lower(m::Module, @nospecialize(x)) = ccall(:jl_lower_expr_mod, Any, (Any, Any), x, m)
+lower(m::Module, @nospecialize(x)) = Core._lower(x, m, "none", 0, typemax(Csize_t), false)[1]
 
 """
     @lower [m] x

doc/src/devdocs/eval.md

@@ -89,7 +89,7 @@ the expression. Macro expansion involves a handoff from [`eval()`](@ref) (in Jul
 function `jl_macroexpand()` (written in `flisp`) to the Julia macro itself (written in - what
 else - Julia) via `fl_invoke_julia_macro()`, and back.
 
-Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`jl_lower_expr_mod()`,
+Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`Core._lower()`,
 although it can also be invoked directly by a call to [`macroexpand()`](@ref)/`jl_macroexpand()`.
 
 ## [Type Inference](@id dev-type-inference)

src/ast.c

@@ -1275,13 +1275,12 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule
     return expr;
 }
 
-// Main entry point to flisp lowering.  Most arguments are optional; see `jl_lower_expr_mod`.
 // warn: Print any lowering warnings returned; otherwise ignore
 JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule,
-                                     const char *file, int line, size_t world, bool_t warn)
+                                     const char *filename, int line, size_t world, bool_t warn)
 {
     JL_TIMING(LOWERING, LOWERING);
-    jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK);
+    jl_timing_show_location(filename, line, inmodule, JL_TIMING_DEFAULT_BLOCK);
     jl_array_t *kwargs = NULL;
     JL_GC_PUSH3(&expr, &kwargs, &inmodule);
     expr = jl_copy_ast(expr);
@@ -1290,7 +1289,7 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule,
     fl_context_t *fl_ctx = &ctx->fl;
     value_t arg = julia_to_scm(fl_ctx, expr);
     value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg,
-                          symbol(fl_ctx, file), fixnum(line));
+                          symbol(fl_ctx, filename), fixnum(line));
     value_t lwr = car_(e);
     value_t warnings = car_(cdr_(e));
     expr = scm_to_julia(fl_ctx, lwr, inmodule);
@@ -1306,6 +1305,7 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule,
             jl_error("julia-logmsg: bad argument list - expected "
                      ":warn level (symbol) group (symbol) id file line msg . kwargs");
         }
+        JL_GC_PUSH1(&warning);
         jl_value_t *level = jl_exprarg(warning, 0);
         jl_value_t *group = jl_exprarg(warning, 1);
         jl_value_t *id = jl_exprarg(warning, 2);
@@ -1318,22 +1318,45 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule,
         }
         JL_TYPECHK(logmsg, long, level);
         jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg);
+        JL_GC_POP();
     }
+    jl_value_t *result = (jl_value_t *)jl_svec1(expr);
     JL_GC_POP();
-    return expr;
+    return result;
 }
 
-// Lower an expression tree into Julia's intermediate-representation.
+// Main C entry point to lowering.  Calls jl_fl_lower during bootstrap, and
+// Core._lower otherwise (this is also jl_fl_lower unless we have JuliaLowering)
 JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule,
-                                  const char *file, int line, size_t world, bool_t warn)
+                                  const char *filename, int line, size_t world, bool_t warn)
 {
-    // TODO: Allow change of lowerer
-    return jl_fl_lower(expr, inmodule, file, line, world, warn);
-}
-
-JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule)
-{
-    return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0);
+    jl_value_t *core_lower = NULL;
+    if (jl_core_module) {
+        core_lower = jl_get_global(jl_core_module, jl_symbol("_lower"));
+    }
+    if (!core_lower || core_lower == jl_nothing) {
+        return jl_fl_lower(expr, inmodule, filename, line, world, warn);
+    }
+    jl_value_t **args;
+    JL_GC_PUSHARGS(args, 7);
+    args[0] = core_lower;
+    args[1] = expr;
+    args[2] = (jl_value_t*)inmodule;
+    args[3] = jl_cstr_to_string(filename);
+    args[4] = jl_box_ulong(line);
+    args[5] = jl_box_ulong(world);
+    args[6] = warn ? jl_true : jl_false;
+    jl_task_t *ct = jl_current_task;
+    size_t last_age = ct->world_age;
+    ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
+    jl_value_t *result = jl_apply(args, 7);
+    ct->world_age = last_age;
+    args[0] = result; // root during error check below
+    JL_TYPECHK(parse, simplevector, result);
+    if (jl_svec_len(result) < 1)
+        jl_error("Result from lowering should be `svec(a::Any, x::Any...)`");
+    JL_GC_POP();
+    return result;
 }
 
 jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line)

src/jl_exported_funcs.inc

@@ -285,7 +285,6 @@
     XX(jl_load_file_string) \
     XX(jl_lookup_code_address) \
     XX(jl_lower) \
-    XX(jl_lower_expr_mod) \
     XX(jl_lseek) \
     XX(jl_lstat) \
     XX(jl_macroexpand) \

src/julia.h

@@ -2260,7 +2260,6 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len,
 JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule,
                                   const char *file, int line, size_t world,
                                   bool_t warn);
-JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule);
 // deprecated; use jl_parse_all
 JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *text, size_t text_len,
                                              const char *filename, size_t filename_len);

src/toplevel.c

@@ -195,8 +195,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex
 
     for (int i = 0; i < jl_array_nrows(exprs); i++) {
         // process toplevel form
-        ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
-        form = jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0);
+        form = jl_svecref(jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0), 0);
         ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
         (void)jl_toplevel_eval_flex(newm, form, 1, 1, &filename, &lineno);
     }
@@ -806,10 +805,8 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val
     JL_GC_PUSH4(&mfunc, &thk, &ex, &root);
 
     size_t last_age = ct->world_age;
-    if (!expanded && jl_needs_lowering(e)) {
-        ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
-        ex = (jl_expr_t*)jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1);
-        ct->world_age = last_age;
+    if (!expanded && (jl_needs_lowering(e))) {
+        ex = (jl_expr_t*)jl_svecref(jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1), 0);
     }
     jl_sym_t *head = jl_is_expr(ex) ? ex->head : NULL;
 
@@ -971,8 +968,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val
         for (i = 0; i < jl_array_nrows(ex->args); i++) {
             root = jl_array_ptr_ref(ex->args, i);
             if (jl_needs_lowering(root)) {
-                ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
-                root = jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1);
+                root = jl_svecref(jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1), 0);
             }
             ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
             res = jl_toplevel_eval_flex(m, root, fast, 1, toplevel_filename, toplevel_lineno);
@@ -1150,8 +1146,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text,
                 jl_lineno = lineno;
                 continue;
             }
-            ct->world_age = jl_atomic_load_relaxed(&jl_world_counter);
-            expression = jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1);
+            expression = jl_svecref(jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1), 0);
             ct->world_age = jl_atomic_load_relaxed(&jl_world_counter);
             result = jl_toplevel_eval_flex(module, expression, 1, 1, &filename_str, &lineno);
         }

stdlib/InteractiveUtils/src/macros.jl

@@ -558,8 +558,8 @@ in the current environment.
 When using `@activate`, additional options for a component may be specified in
 square brackets `@activate Compiler[:option1, :option]`
 
-Currently `@activate Compiler` is the only available component that may be
-activatived.
+Currently `Compiler` and `JuliaLowering` are the only available components that
+may be activatived.
 
 For `@activate Compiler`, the following options are available:
 1. `:reflection` - Activate the compiler for reflection purposes only.
@@ -593,7 +593,7 @@ macro activate(what)
     if !isa(Component, Symbol)
         error("Usage Error: Component $Component is not a symbol")
     end
-    allowed_components = (:Compiler,)
+    allowed_components = (:Compiler, :JuliaLowering)
     if !(Component in allowed_components)
         error("Usage Error: Component $Component is not recognized. Expected one of $allowed_components")
     end

stdlib/REPL/src/REPL.jl

@@ -298,12 +298,12 @@ const install_packages_hooks = Any[]
 # N.B.: Any functions starting with __repl_entry cut off backtraces when printing in the REPL.
 # We need to do this for both the actual eval and macroexpand, since the latter can cause custom macro
 # code to run (and error).
-__repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) =
-    ccall(:jl_lower, Any, (Any, Any, Ptr{UInt8}, Cint, Csize_t, Cint), ast, mod, toplevel_file[], toplevel_line[], typemax(Csize_t), 0)
-__repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) =
-    ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Cint}), mod, ast, 1, 1, toplevel_file, toplevel_line)
+__repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Csize_t}) =
+    Core._lower(ast, mod, toplevel_file[], toplevel_line[])[1]
+__repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Csize_t}) =
+    ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Csize_t}), mod, ast, 1, 1, toplevel_file, toplevel_line)
 
-function toplevel_eval_with_hooks(mod::Module, @nospecialize(ast), toplevel_file=Ref{Ptr{UInt8}}(Base.unsafe_convert(Ptr{UInt8}, :REPL)), toplevel_line=Ref{Cint}(1))
+function toplevel_eval_with_hooks(mod::Module, @nospecialize(ast), toplevel_file=Ref{Ptr{UInt8}}(Base.unsafe_convert(Ptr{UInt8}, :REPL)), toplevel_line=Ref{Csize_t}(1))
     if !isexpr(ast, :toplevel)
         ast = invokelatest(__repl_entry_lower_with_loc, mod, ast, toplevel_file, toplevel_line)
         check_for_missing_packages_and_run_hooks(ast)

test/meta.jl

@@ -234,7 +234,7 @@ let ex = Meta.parseall("@foo", filename=:bar)
     @test isa(arg2arg2, LineNumberNode) && arg2arg2.file === :bar
 end
 
-_lower(m::Module, ex, world::UInt) = ccall(:jl_lower, Any, (Any, Ref{Module}, Cstring, Cint, Csize_t, Cint), ex, m, "none", 0, world, 0)
+_lower(m::Module, ex, world::UInt) = Base.fl_lower(ex, m, "none", 0, world, false)[1]
 
 module TestExpandInWorldModule
 macro m() 1 end