Lowering API: Allow for switching to JuliaLowering (#58207)

This PR along with [the associated JuliaLowering
PR](c42f/JuliaLowering.jl#10) allow us to
experiment
with JuliaLowering as the core lowering implementation, mainly for JETLS
development at the moment (cc @aviatesk, @c42f).

## Changes

JuliaLowering works with `JuliaLowering.SyntaxTree` input instead of
Expr. This
change allows`SyntaxTree`s out of parsing and into lowering, hopefully
without
disturbing existing code expecting Expr.  The current design wraps
`x::SyntaxTree` like `Expr(:syntaxtree, x)` because otherwise
`toplevel_eval_flex` wouldn't know what `x` is, but once we're past the
experimental phase, it would only make sense for the compiler to
actually know
about the types it's operating on.

### New: `Core._lower`

It (a julia entrypoint), `jl_lower` (C entrypoint), `jl_fl_lower`, and
`fl_lower` all mirror the existing parsing API which was added in
#35243.


<details>

<summary>
Click for pre-PR call graph
</summary>


![graph(6)](https://github.com/user-attachments/assets/a993eb92-5668-4ee7-837f-801feb686cb4)

</details>

After this change:


![graph(7)](https://github.com/user-attachments/assets/697b93cf-b3df-47b2-bc0e-c358fb0a6dca)

Like `Core._parse`, `Core._lower` returns an `svec` with the first
element as
the expected result, and the remaining elements can hold extra
information like
the lowered SyntaxTree with provenance info (I've left the specification
of
extra return values for later, when JuliaLowering is more stable and
JETLS knows
what it needs, because we may opt to pack provenance into the lowered
CodeInfo
object instead). Parsing only uses the svec to return an offset, but it
could
actually make use of the the flexibility to return diagnostics in the
future.

It's unfortunately not clear what types we can expect to go into and
come out of
lowering (I got Expr, LineNumberNode, String, Symbol, and Nothing in
testing.)

### Remove `jl_lower_expr_mod`

I realize this isn't being called from C, and Core._lower is a more
sensible
entry point from julia. We could probably also remove some uncalled
parse and
jl_load functions.

## Limitations

These belong to JuliaLowering, but are listed here as known issues for
anyone
who wants to try it out.

- `(module ...)` forms don't go through JuliaLowering yet. Modules are
lowered
to a call to `JuliaLowering.eval_module(parentmod, modname,
syntaxtree)`,
which is currently a stub function that converts to Expr for lowering
and
  evaluation, which JuliaLowering can't use.
- Macros work differently.  Defining and calling macros should work, but
  invoking normal macros defined pre-JuliaLowering will not.
- Compilation is currently slow (~30 seconds for me).

Author: topolarity

base/Base_compiler.jl

@@ -383,8 +383,9 @@ const _return_type = Compiler.return_type
 # Enable compiler
 Compiler.bootstrap!()
 
-include("flparse.jl")
+include("flfrontend.jl")
 Core._setparser!(fl_parse)
+Core._setlowerer!(fl_lower)
 
 # Further definition of Base will happen in Base.jl if loaded.
 

base/boot.jl

@@ -1046,8 +1046,9 @@ function struct_name_shim(@nospecialize(x), name::Symbol, mod::Module, @nospecia
     return x === mod ? t : getfield(x, name)
 end
 
-# Binding for the julia parser, called as
-#
+# Bindings for the julia frontend.  The internal jl_parse and jl_lower will call
+# Core._parse and Core._lower respectively (if they are not `nothing`.)
+
 #    Core._parse(text, filename, lineno, offset, options)
 #
 # Parse Julia code from the buffer `text`, starting at `offset` and attributing
@@ -1057,11 +1058,17 @@ end
 #
 # `_parse` must return an `svec` containing an `Expr` and the new offset as an
 # `Int`.
-#
-# The internal jl_parse will call into Core._parse if not `nothing`.
 _parse = nothing
 
+#    Core._lower(code, module, filename="none", linenum=0, world=0xfff..., warn=false)
+#
+# Lower `code` (usually Expr), returning `svec(e::Any xs::Any...)` where `e` is
+# the lowered code, and `xs` is possible additional information from
+# JuliaLowering (TBD).
+_lower = nothing
+
 _setparser!(parser) = setglobal!(Core, :_parse, parser)
+_setlowerer!(lowerer) = setglobal!(Core, :_lower, lowerer)
 
 # support for deprecated uses of builtin functions
 _apply(x...) = _apply_iterate(Main.Base.iterate, x...)

base/expr.jl

@@ -1673,7 +1673,8 @@ end
 
 # Implementation of generated functions
 function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool)
-    ci = ccall(:jl_lower_expr_mod, Any, (Any, Any), ex, defmod)
+    ci = ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint),
+               ex, defmod, "none", 0, typemax(Csize_t), 0)[1]
     if !isa(ci, CodeInfo)
         if isa(ci, Expr) && ci.head === :error
             msg = ci.args[1]

base/flparse.jl renamed to base/flfrontend.jl

@@ -17,3 +17,10 @@ end
 function fl_parse(text::AbstractString, filename::AbstractString, lineno, offset, options)
     fl_parse(String(text), String(filename), lineno, offset, options)
 end
+
+function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none",
+                  lineno=0, world::Unsigned=typemax(Csize_t), warn::Bool=false)
+    warn = warn ? 1 : 0
+    ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint),
+          ex, mod, filename, lineno, world, warn)
+end

base/loading.jl

@@ -2809,7 +2809,11 @@ function include_string(mapexpr::Function, mod::Module, code::AbstractString,
     loc = LineNumberNode(1, Symbol(filename))
     try
         ast = Meta.parseall(code, filename=filename)
-        @assert Meta.isexpr(ast, :toplevel)
+        if !Meta.isexpr(ast, :toplevel)
+            @assert Core._lower != fl_lower
+            # Only reached when JuliaLowering and alternate parse functions are activated
+            return Core.eval(mod, ast)
+        end
         result = nothing
         line_and_ex = Expr(:toplevel, loc, nothing)
         for ex in ast.args

base/meta.jl

@@ -160,7 +160,7 @@ Takes the expression `x` and returns an equivalent expression in lowered form
 for executing in module `m`.
 See also [`code_lowered`](@ref).
 """
-lower(m::Module, @nospecialize(x)) = ccall(:jl_lower_expr_mod, Any, (Any, Any), x, m)
+lower(m::Module, @nospecialize(x)) = Core._lower(x, m, "none", 0, typemax(Csize_t), false)[1]
 
 """
     @lower [m] x

doc/src/devdocs/eval.md

@@ -89,7 +89,7 @@ the expression. Macro expansion involves a handoff from [`eval()`](@ref) (in Jul
 function `jl_macroexpand()` (written in `flisp`) to the Julia macro itself (written in - what
 else - Julia) via `fl_invoke_julia_macro()`, and back.
 
-Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`jl_lower_expr_mod()`,
+Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`Core._lower()`,
 although it can also be invoked directly by a call to [`macroexpand()`](@ref)/`jl_macroexpand()`.
 
 ## [Type Inference](@id dev-type-inference)

src/ast.c

@@ -1271,13 +1271,12 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule
     return expr;
 }
 
-// Main entry point to flisp lowering.  Most arguments are optional; see `jl_lower_expr_mod`.
 // warn: Print any lowering warnings returned; otherwise ignore
 JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule,
-                                     const char *file, int line, size_t world, bool_t warn)
+                                     const char *filename, int line, size_t world, bool_t warn)
 {
     JL_TIMING(LOWERING, LOWERING);
-    jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK);
+    jl_timing_show_location(filename, line, inmodule, JL_TIMING_DEFAULT_BLOCK);
     jl_array_t *kwargs = NULL;
     JL_GC_PUSH3(&expr, &kwargs, &inmodule);
     expr = jl_copy_ast(expr);
@@ -1286,7 +1285,7 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule,
     fl_context_t *fl_ctx = &ctx->fl;
     value_t arg = julia_to_scm(fl_ctx, expr);
     value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg,
-                          symbol(fl_ctx, file), fixnum(line));
+                          symbol(fl_ctx, filename), fixnum(line));
     value_t lwr = car_(e);
     value_t warnings = car_(cdr_(e));
     expr = scm_to_julia(fl_ctx, lwr, inmodule);
@@ -1302,6 +1301,7 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule,
             jl_error("julia-logmsg: bad argument list - expected "
                      ":warn level (symbol) group (symbol) id file line msg . kwargs");
         }
+        JL_GC_PUSH1(&warning);
         jl_value_t *level = jl_exprarg(warning, 0);
         jl_value_t *group = jl_exprarg(warning, 1);
         jl_value_t *id = jl_exprarg(warning, 2);
@@ -1314,22 +1314,45 @@ JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule,
         }
         JL_TYPECHK(logmsg, long, level);
         jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg);
+        JL_GC_POP();
     }
+    jl_value_t *result = (jl_value_t *)jl_svec1(expr);
     JL_GC_POP();
-    return expr;
+    return result;
 }
 
-// Lower an expression tree into Julia's intermediate-representation.
+// Main C entry point to lowering.  Calls jl_fl_lower during bootstrap, and
+// Core._lower otherwise (this is also jl_fl_lower unless we have JuliaLowering)
 JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule,
-                                  const char *file, int line, size_t world, bool_t warn)
+                                  const char *filename, int line, size_t world, bool_t warn)
 {
-    // TODO: Allow change of lowerer
-    return jl_fl_lower(expr, inmodule, file, line, world, warn);
-}
-
-JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule)
-{
-    return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0);
+    jl_value_t *core_lower = NULL;
+    if (jl_core_module) {
+        core_lower = jl_get_global(jl_core_module, jl_symbol("_lower"));
+    }
+    if (!core_lower || core_lower == jl_nothing) {
+        return jl_fl_lower(expr, inmodule, filename, line, world, warn);
+    }
+    jl_value_t **args;
+    JL_GC_PUSHARGS(args, 7);
+    args[0] = core_lower;
+    args[1] = expr;
+    args[2] = (jl_value_t*)inmodule;
+    args[3] = jl_cstr_to_string(filename);
+    args[4] = jl_box_ulong(line);
+    args[5] = jl_box_ulong(world);
+    args[6] = warn ? jl_true : jl_false;
+    jl_task_t *ct = jl_current_task;
+    size_t last_age = ct->world_age;
+    ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
+    jl_value_t *result = jl_apply(args, 7);
+    ct->world_age = last_age;
+    args[0] = result; // root during error check below
+    JL_TYPECHK(parse, simplevector, result);
+    if (jl_svec_len(result) < 1)
+        jl_error("Result from lowering should be `svec(a::Any, x::Any...)`");
+    JL_GC_POP();
+    return result;
 }
 
 jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line)

src/jl_exported_funcs.inc

@@ -287,7 +287,6 @@
     XX(jl_load_file_string) \
     XX(jl_lookup_code_address) \
     XX(jl_lower) \
-    XX(jl_lower_expr_mod) \
     XX(jl_lseek) \
     XX(jl_lstat) \
     XX(jl_macroexpand) \

src/julia.h

@@ -2258,7 +2258,6 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len,
 JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule,
                                   const char *file, int line, size_t world,
                                   bool_t warn);
-JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule);
 // deprecated; use jl_parse_all
 JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *text, size_t text_len,
                                              const char *filename, size_t filename_len);