Clean up lowering API (#58147)

This is preparatory work for experimental integration with
JuliaLowering, which
happened to be separable into its own PR.

We currently have three lowering entry points doing slightly different
things:
`jl-expand-to-thunk`, `jl-expand-to-thunk-warn` (for complaining about
ambiguous
soft scope assignments during non-interactive use), and
`jl-expand-to-thunk-stmt` (which wraps the expression in a block
returning
nothing before lowering it) and a bunch of C wrappers on top of those
(see red
nodes in the call graphs below).

In this PR:
- Make all lowering calls go through `jl_lower`, which just calls out to
lisp
for now, but will probably function like `jl_parse` does in a future PR.
  - Handle `warn` with an extra parameter
- Delete most of the lowering wrappers, which were mainly a result of
the
    three entry points
- While we're breaking things in ast.c, take the opportunity to rename
  "expand"-prefixed functions to "lower"-prefixed ones (excluding macro
  expansion functions, which are called as the first part of lowering).

Here's a call graph before this change, made by looking for callers
of each lowering entry point and tracing back one or more steps (expect
mistakes...). Macro expansion functions are mostly omitted for clarity.
Blue is scheme, red is ast.c, and green is toplevel.c.


![graph](https://github.com/user-attachments/assets/b84a8662-06b4-45f6-937f-2d5ab5325ff0)

After this change:

![graph(3)](https://github.com/user-attachments/assets/dd3765e2-1cd4-4ab8-bf85-6fa6eb7f9a74)


#### todo?
- ~~I'd like to see if we can eliminate the `stmt` boolean option from
`jl_lower`
and handle that another way; I'm just not sure it's worth the effort at
the
moment. We only use it in one place in `jl_eval_module_expr`. The
original
  code path was added in #26304.~~

Author: topolarity

base/expr.jl

@@ -1669,7 +1669,7 @@ end
 
 # Implementation of generated functions
 function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool)
-    ci = ccall(:jl_expand, Any, (Any, Any), ex, defmod)
+    ci = ccall(:jl_lower_expr_mod, Any, (Any, Any), ex, defmod)
     if !isa(ci, CodeInfo)
         if isa(ci, Expr) && ci.head === :error
             msg = ci.args[1]

base/meta.jl

@@ -160,7 +160,7 @@ Takes the expression `x` and returns an equivalent expression in lowered form
 for executing in module `m`.
 See also [`code_lowered`](@ref).
 """
-lower(m::Module, @nospecialize(x)) = ccall(:jl_expand, Any, (Any, Any), x, m)
+lower(m::Module, @nospecialize(x)) = ccall(:jl_lower_expr_mod, Any, (Any, Any), x, m)
 
 """
     @lower [m] x

doc/src/devdocs/eval.md

@@ -89,7 +89,7 @@ the expression. Macro expansion involves a handoff from [`eval()`](@ref) (in Jul
 function `jl_macroexpand()` (written in `flisp`) to the Julia macro itself (written in - what
 else - Julia) via `fl_invoke_julia_macro()`, and back.
 
-Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`jl_expand()`,
+Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`jl_lower_expr_mod()`,
 although it can also be invoked directly by a call to [`macroexpand()`](@ref)/`jl_macroexpand()`.
 
 ## [Type Inference](@id dev-type-inference)

src/ast.c

@@ -1275,98 +1275,65 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule
     return expr;
 }
 
-// Lower an expression tree into Julia's intermediate-representation.
-JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule)
-{
-    return jl_expand_with_loc(expr, inmodule, "none", 0);
-}
-
-// Lowering, with starting program location specified
-JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule,
-                                            const char *file, int line)
-{
-    return jl_expand_in_world(expr, inmodule, file, line, ~(size_t)0);
-}
-
-// Lowering, with starting program location and worldage specified
-JL_DLLEXPORT jl_value_t *jl_expand_in_world(jl_value_t *expr, jl_module_t *inmodule,
-                                            const char *file, int line, size_t world)
-{
-    JL_TIMING(LOWERING, LOWERING);
-    jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK);
-    JL_GC_PUSH1(&expr);
-    expr = jl_copy_ast(expr);
-    expr = jl_expand_macros(expr, inmodule, NULL, 0, world, 1);
-    expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk", expr, inmodule, file, line);
-    JL_GC_POP();
-    return expr;
-}
-
-// Same as the above, but printing warnings when applicable
-JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t *inmodule,
-                                                 const char *file, int line)
+// Main entry point to flisp lowering.  Most arguments are optional; see `jl_lower_expr_mod`.
+// warn: Print any lowering warnings returned; otherwise ignore
+JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule,
+                                     const char *file, int line, size_t world, bool_t warn)
 {
     JL_TIMING(LOWERING, LOWERING);
     jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK);
     jl_array_t *kwargs = NULL;
-    JL_GC_PUSH2(&expr, &kwargs);
+    JL_GC_PUSH3(&expr, &kwargs, &inmodule);
     expr = jl_copy_ast(expr);
-    expr = jl_expand_macros(expr, inmodule, NULL, 0, ~(size_t)0, 1);
+    expr = jl_expand_macros(expr, inmodule, NULL, 0, world, 1);
     jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule);
     fl_context_t *fl_ctx = &ctx->fl;
     value_t arg = julia_to_scm(fl_ctx, expr);
-    value_t e = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-expand-to-thunk-warn")), arg,
-                          symbol(fl_ctx, file), fixnum(line), fl_ctx->F);
-    expr = scm_to_julia(fl_ctx, e, inmodule);
+    value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg,
+                          symbol(fl_ctx, file), fixnum(line));
+    value_t lwr = car_(e);
+    value_t warnings = car_(cdr_(e));
+    expr = scm_to_julia(fl_ctx, lwr, inmodule);
     jl_ast_ctx_leave(ctx);
     jl_sym_t *warn_sym = jl_symbol("warn");
-    if (jl_is_expr(expr) && ((jl_expr_t*)expr)->head == warn_sym) {
-        size_t nargs = jl_expr_nargs(expr);
-        for (int i = 0; i < nargs - 1; i++) {
-            jl_value_t *warning = jl_exprarg(expr, i);
-            size_t nargs = 0;
-            if (jl_is_expr(warning) && ((jl_expr_t*)warning)->head == warn_sym)
-                 nargs = jl_expr_nargs(warning);
-            int kwargs_len = (int)nargs - 6;
-            if (nargs < 6 || kwargs_len % 2 != 0) {
-                jl_error("julia-logmsg: bad argument list - expected "
-                         ":warn level (symbol) group (symbol) id file line msg . kwargs");
-            }
-            jl_value_t *level = jl_exprarg(warning, 0);
-            jl_value_t *group = jl_exprarg(warning, 1);
-            jl_value_t *id = jl_exprarg(warning, 2);
-            jl_value_t *file = jl_exprarg(warning, 3);
-            jl_value_t *line = jl_exprarg(warning, 4);
-            jl_value_t *msg = jl_exprarg(warning, 5);
-            kwargs = jl_alloc_vec_any(kwargs_len);
-            for (int i = 0; i < kwargs_len; ++i) {
-                jl_array_ptr_set(kwargs, i, jl_exprarg(warning, i + 6));
-            }
-            JL_TYPECHK(logmsg, long, level);
-            jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg);
+    for (; warn && iscons(warnings); warnings = cdr_(warnings)) {
+        jl_value_t *warning = scm_to_julia(fl_ctx, car_(warnings), inmodule);
+        size_t nargs = 0;
+        if (jl_is_expr(warning) && ((jl_expr_t*)warning)->head == warn_sym)
+            nargs = jl_expr_nargs(warning);
+        int kwargs_len = (int)nargs - 6;
+        if (nargs < 6 || kwargs_len % 2 != 0) {
+            jl_error("julia-logmsg: bad argument list - expected "
+                     ":warn level (symbol) group (symbol) id file line msg . kwargs");
         }
-        expr = jl_exprarg(expr, nargs - 1);
+        jl_value_t *level = jl_exprarg(warning, 0);
+        jl_value_t *group = jl_exprarg(warning, 1);
+        jl_value_t *id = jl_exprarg(warning, 2);
+        jl_value_t *file = jl_exprarg(warning, 3);
+        jl_value_t *line = jl_exprarg(warning, 4);
+        jl_value_t *msg = jl_exprarg(warning, 5);
+        kwargs = jl_alloc_vec_any(kwargs_len);
+        for (int i = 0; i < kwargs_len; ++i) {
+            jl_array_ptr_set(kwargs, i, jl_exprarg(warning, i + 6));
+        }
+        JL_TYPECHK(logmsg, long, level);
+        jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg);
     }
     JL_GC_POP();
     return expr;
 }
 
-// expand in a context where the expression value is unused
-JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule,
-                                                 const char *file, int line)
+// Lower an expression tree into Julia's intermediate-representation.
+JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule,
+                                  const char *file, int line, size_t world, bool_t warn)
 {
-    JL_TIMING(LOWERING, LOWERING);
-    JL_GC_PUSH1(&expr);
-    expr = jl_copy_ast(expr);
-    expr = jl_expand_macros(expr, inmodule, NULL, 0, ~(size_t)0, 1);
-    expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk-stmt", expr, inmodule, file, line);
-    JL_GC_POP();
-    return expr;
+    // TODO: Allow change of lowerer
+    return jl_fl_lower(expr, inmodule, file, line, world, warn);
 }
 
-JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule)
+JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule)
 {
-    return jl_expand_stmt_with_loc(expr, inmodule, "none", 0);
+    return jl_lower(expr, inmodule, "none", 0, ~(size_t)0, 0);
 }
 
 jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line)

src/jl_exported_funcs.inc

@@ -125,11 +125,6 @@
     XX(jl_exit) \
     XX(jl_exit_on_sigint) \
     XX(jl_exit_threaded_region) \
-    XX(jl_expand) \
-    XX(jl_expand_stmt) \
-    XX(jl_expand_stmt_with_loc) \
-    XX(jl_expand_with_loc) \
-    XX(jl_expand_with_loc_warn) \
     XX(jl_field_index) \
     XX(jl_field_isdefined) \
     XX(jl_gc_add_finalizer) \
@@ -289,6 +284,8 @@
     XX(jl_load_dynamic_library) \
     XX(jl_load_file_string) \
     XX(jl_lookup_code_address) \
+    XX(jl_lower) \
+    XX(jl_lower_expr_mod) \
     XX(jl_lseek) \
     XX(jl_lstat) \
     XX(jl_macroexpand) \

src/jlfrontend.scm

@@ -109,7 +109,7 @@
 ;; return a lambda expression representing a thunk for a top-level expression
 ;; note: expansion of stuff inside module is delayed, so the contents obey
 ;; toplevel expansion order (don't expand until stuff before is evaluated).
-(define (expand-toplevel-expr-- e file line)
+(define (lower-toplevel-expr-- e file line)
   (let ((lno (first-lineno e))
         (ex0 (julia-expand-macroscope e)))
     (if (and lno (or (not (length= lno 3)) (not (atom? (caddr lno))))) (set! lno #f))
@@ -118,8 +118,8 @@
             ex0
             (if lno `(toplevel ,lno ,ex0) ex0))
         (let* ((linenode (if (and lno (or (= line 0) (eq? file 'none))) lno `(line ,line ,file)))
-               (ex (julia-expand0 ex0 linenode))
-               (th (julia-expand1
+               (ex (julia-lower0 ex0 linenode))
+               (th (julia-lower1
                     `(lambda () ()
                              (scope-block
                               ,(blockify ex lno)))
@@ -142,38 +142,36 @@
                                     error incomplete))
            (and (memq (car e) '(global const)) (every symbol? (cdr e))))))
 
-(define *in-expand* #f)
+(define *in-lowering* #f)
 
-(define (expand-toplevel-expr e file line)
+(define (lower-toplevel-expr e file line)
   (cond ((or (atom? e) (toplevel-only-expr? e))
          (if (underscore-symbol? e)
              (error "all-underscore identifiers are write-only and their values cannot be used in expressions"))
          e)
         (else
-         (let ((last *in-expand*))
+         (let ((last *in-lowering*))
            (if (not last)
                (begin (reset-gensyms)
-                      (set! *in-expand* #t)))
-           (begin0 (expand-toplevel-expr-- e file line)
-                   (set! *in-expand* last))))))
+                      (set! *in-lowering* #t)))
+           (begin0 (lower-toplevel-expr-- e file line)
+                   (set! *in-lowering* last))))))
 
 ;; used to collect warnings during lowering, which are usually discarded
 ;; unless logging is requested
 (define lowering-warning (lambda lst (void)))
 
 ;; expand a piece of raw surface syntax to an executable thunk
 
-(define (expand-to-thunk- expr file line)
+(define (lower-to-thunk- expr file line)
   (error-wrap (lambda ()
-                (expand-toplevel-expr expr file line))))
+                (lower-toplevel-expr expr file line))))
 
-(define (expand-to-thunk-stmt- expr file line)
-  (expand-to-thunk- (if (toplevel-only-expr? expr)
-                        expr
-                        `(block ,expr (null)))
-                    file line))
-
-(define (jl-expand-to-thunk-warn expr file line stmt)
+;; Returns a list `(,lowered-code ,warnings) where
+;; - warnings (currently only ambiguous soft scope assignments) may be ignored,
+;;   e.g. when running interactively
+;; - more items may be added to the list later
+(define (jl-lower-to-thunk expr file line)
   (let ((warnings '()))
     (with-bindings
      ;; Abuse scm_to_julia here to convert arguments to warn. This is meant for
@@ -183,30 +181,22 @@
         (let ((line (if (= warn_line 0) line warn_line))
               (file (if (eq? warn_file 'none) file warn_file)))
           (set! warnings (cons (list* 'warn level group (symbol (string file line)) file line lst) warnings))))))
-     (let ((thunk (if stmt
-                      (expand-to-thunk-stmt- expr file line)
-                      (expand-to-thunk- expr file line))))
-       (if (pair? warnings) `(warn ,@(reverse warnings) ,thunk) thunk)))))
-
-(define (jl-expand-to-thunk expr file line)
-  (expand-to-thunk- expr file line))
-
-(define (jl-expand-to-thunk-stmt expr file line)
-  (expand-to-thunk-stmt- expr file line))
+     `(,(lower-to-thunk- expr file line)
+       ,(reverse warnings)))))
 
 (define (jl-expand-macroscope expr)
   (error-wrap (lambda ()
                 (julia-expand-macroscope expr))))
 
 (define (jl-default-inner-ctor-body field-kinds file line)
-  (expand-to-thunk- (default-inner-ctor-body (cdr field-kinds) file line) file line))
+  (lower-to-thunk- (default-inner-ctor-body (cdr field-kinds) file line) file line))
 
 (define (jl-default-outer-ctor-body args file line)
-  (expand-to-thunk- (default-outer-ctor-body (cadr args) (caddr args) (cadddr args) file line) file line))
+  (lower-to-thunk- (default-outer-ctor-body (cadr args) (caddr args) (cadddr args) file line) file line))
 
 ; run whole frontend on a string. useful for testing.
 (define (fe str)
-  (expand-toplevel-expr (julia-parse str) 'none 0))
+  (lower-toplevel-expr (julia-parse str) 'none 0))
 
 (define (profile-e s)
   (with-exception-catcher

src/julia-syntax.scm

@@ -5331,7 +5331,7 @@ f(x) = yt(x)
 
 ;; expander entry point
 
-(define (julia-expand1 ex file line)
+(define (julia-lower1 ex file line)
   (compact-and-renumber
    (linearize
     (closure-convert
@@ -5340,7 +5340,7 @@ f(x) = yt(x)
 
 (define *current-desugar-loc* #f)
 
-(define (julia-expand0 ex lno)
+(define (julia-lower0 ex lno)
   (with-bindings ((*current-desugar-loc* lno))
    (trycatch (expand-forms ex)
              (lambda (e)
@@ -5353,7 +5353,7 @@ f(x) = yt(x)
                    (error (string (cadr e) (format-loc *current-desugar-loc*))))
                    (raise e)))))
 
-(define (julia-expand ex (file 'none) (line 0))
-  (julia-expand1
-   (julia-expand0
+(define (julia-lower ex (file 'none) (line 0))
+  (julia-lower1
+   (julia-lower0
     (julia-expand-macroscope ex) `(line ,line ,file)) file line))

src/julia.h

@@ -2257,16 +2257,10 @@ JL_DLLEXPORT jl_value_t *jl_parse_all(const char *text, size_t text_len,
 JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len,
                                          int offset, int greedy);
 // lowering
-JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule);
-JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule,
-                                            const char *file, int line);
-JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t *inmodule,
-                                                 const char *file, int line);
-JL_DLLEXPORT jl_value_t *jl_expand_in_world(jl_value_t *expr, jl_module_t *inmodule,
-                                            const char *file, int line, size_t world);
-JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule);
-JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule,
-                                                 const char *file, int line);
+JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule,
+                                  const char *file, int line, size_t world,
+                                  bool_t warn);
+JL_DLLEXPORT jl_value_t *jl_lower_expr_mod(jl_value_t *expr, jl_module_t *inmodule);
 // deprecated; use jl_parse_all
 JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *text, size_t text_len,
                                              const char *filename, size_t filename_len);

src/toplevel.c

@@ -196,7 +196,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex
     for (int i = 0; i < jl_array_nrows(exprs); i++) {
         // process toplevel form
         ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
-        form = jl_expand_stmt_with_loc(jl_array_ptr_ref(exprs, i), newm, filename, lineno);
+        form = jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0);
         ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
         (void)jl_toplevel_eval_flex(newm, form, 1, 1, &filename, &lineno);
     }
@@ -808,7 +808,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val
     size_t last_age = ct->world_age;
     if (!expanded && jl_needs_lowering(e)) {
         ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
-        ex = (jl_expr_t*)jl_expand_with_loc_warn(e, m, *toplevel_filename, *toplevel_lineno);
+        ex = (jl_expr_t*)jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1);
         ct->world_age = last_age;
     }
     jl_sym_t *head = jl_is_expr(ex) ? ex->head : NULL;
@@ -972,7 +972,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val
             root = jl_array_ptr_ref(ex->args, i);
             if (jl_needs_lowering(root)) {
                 ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
-                root = jl_expand_with_loc_warn(root, m, *toplevel_filename, *toplevel_lineno);
+                root = jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1);
             }
             ct->world_age = jl_atomic_load_acquire(&jl_world_counter);
             res = jl_toplevel_eval_flex(m, root, fast, 1, toplevel_filename, toplevel_lineno);
@@ -1151,8 +1151,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text,
                 continue;
             }
             ct->world_age = jl_atomic_load_relaxed(&jl_world_counter);
-            expression = jl_expand_with_loc_warn(expression, module,
-                                                 jl_string_data(filename), lineno);
+            expression = jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1);
             ct->world_age = jl_atomic_load_relaxed(&jl_world_counter);
             result = jl_toplevel_eval_flex(module, expression, 1, 1, &filename_str, &lineno);
         }

stdlib/REPL/src/REPL.jl

@@ -298,7 +298,7 @@ const install_packages_hooks = Any[]
 # We need to do this for both the actual eval and macroexpand, since the latter can cause custom macro
 # code to run (and error).
 __repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) =
-    ccall(:jl_expand_with_loc, Any, (Any, Any, Ptr{UInt8}, Cint), ast, mod, toplevel_file[], toplevel_line[])
+    ccall(:jl_lower, Any, (Any, Any, Ptr{UInt8}, Cint, Csize_t, Cint), ast, mod, toplevel_file[], toplevel_line[], typemax(Csize_t), 0)
 __repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) =
     ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Cint}), mod, ast, 1, 1, toplevel_file, toplevel_line)