From c85344fb67d9db2625adae493d1b5ca81f38074b Mon Sep 17 00:00:00 2001 From: oscarddssmith Date: Wed, 9 Apr 2025 15:35:14 -0400 Subject: [PATCH 1/7] add activate! support for JuliaSyntax and JuliaLowering --- stdlib/InteractiveUtils/src/macros.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/InteractiveUtils/src/macros.jl b/stdlib/InteractiveUtils/src/macros.jl index 68afc40976275..94ffde0fd0032 100644 --- a/stdlib/InteractiveUtils/src/macros.jl +++ b/stdlib/InteractiveUtils/src/macros.jl @@ -440,7 +440,7 @@ in the current environment. When using `@activate`, additional options for a component may be specified in square brackets `@activate Compiler[:option1, :option]` -Currently `@activate Compiler` is the only available component that may be +Currently `Compiler`, `JuliaSyntax`, and `JuliaLowering` are the only available components that may be activatived. For `@activate Compiler`, the following options are available: @@ -475,7 +475,7 @@ macro activate(what) if !isa(Component, Symbol) error("Usage Error: Component $Component is not a symbol") end - allowed_components = (:Compiler,) + allowed_components = (:Compiler, :JuliaSyntax, :JuliaLowering) if !(Component in allowed_components) error("Usage Error: Component $Component is not recognized. Expected one of $allowed_components") end From a67e1cc927956d325c10402df8bbd54628c86b4f Mon Sep 17 00:00:00 2001 From: oscarddssmith Date: Wed, 9 Apr 2025 16:25:59 -0400 Subject: [PATCH 2/7] check Core._lower in lowering --- src/ast.c | 100 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 38 deletions(-) diff --git a/src/ast.c b/src/ast.c index ab6b04efa526a..fb727b58f533a 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1307,48 +1307,72 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t * const char *file, int line) { JL_TIMING(LOWERING, LOWERING); - jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); - jl_array_t *kwargs = NULL; - JL_GC_PUSH2(&expr, &kwargs); - expr = jl_copy_ast(expr); - expr = jl_expand_macros(expr, inmodule, NULL, 0, ~(size_t)0, 1); - jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule); - fl_context_t *fl_ctx = &ctx->fl; - value_t arg = julia_to_scm(fl_ctx, expr); - value_t e = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-expand-to-thunk-warn")), arg, - symbol(fl_ctx, file), fixnum(line), fl_ctx->F); - expr = scm_to_julia(fl_ctx, e, inmodule); - jl_ast_ctx_leave(ctx); - jl_sym_t *warn_sym = jl_symbol("warn"); - if (jl_is_expr(expr) && ((jl_expr_t*)expr)->head == warn_sym) { - size_t nargs = jl_expr_nargs(expr); - for (int i = 0; i < nargs - 1; i++) { - jl_value_t *warning = jl_exprarg(expr, i); - size_t nargs = 0; - if (jl_is_expr(warning) && ((jl_expr_t*)warning)->head == warn_sym) - nargs = jl_expr_nargs(warning); - int kwargs_len = (int)nargs - 6; - if (nargs < 6 || kwargs_len % 2 != 0) { - jl_error("julia-logmsg: bad argument list - expected " - ":warn level (symbol) group (symbol) id file line msg . kwargs"); - } - jl_value_t *level = jl_exprarg(warning, 0); - jl_value_t *group = jl_exprarg(warning, 1); - jl_value_t *id = jl_exprarg(warning, 2); - jl_value_t *file = jl_exprarg(warning, 3); - jl_value_t *line = jl_exprarg(warning, 4); - jl_value_t *msg = jl_exprarg(warning, 5); - kwargs = jl_alloc_vec_any(kwargs_len); - for (int i = 0; i < kwargs_len; ++i) { - jl_array_ptr_set(kwargs, i, jl_exprarg(warning, i + 6)); + jl_value_t *core_lower = NULL; + if (jl_core_module) { + core_lower = jl_get_global(jl_core_module, jl_symbol("_lower")); + } + if (!core_lower || core_lower == jl_nothing) { + // In bootstrap, directly call the builtin lowerer. + jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); + jl_array_t *kwargs = NULL; + JL_GC_PUSH2(&expr, &kwargs); + expr = jl_copy_ast(expr); + expr = jl_expand_macros(expr, inmodule, NULL, 0, ~(size_t)0, 1); + jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule); + fl_context_t *fl_ctx = &ctx->fl; + value_t arg = julia_to_scm(fl_ctx, expr); + value_t e = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-expand-to-thunk-warn")), arg, + symbol(fl_ctx, file), fixnum(line), fl_ctx->F); + expr = scm_to_julia(fl_ctx, e, inmodule); + jl_ast_ctx_leave(ctx); + jl_sym_t *warn_sym = jl_symbol("warn"); + if (jl_is_expr(expr) && ((jl_expr_t*)expr)->head == warn_sym) { + size_t nargs = jl_expr_nargs(expr); + for (int i = 0; i < nargs - 1; i++) { + jl_value_t *warning = jl_exprarg(expr, i); + size_t nargs = 0; + if (jl_is_expr(warning) && ((jl_expr_t*)warning)->head == warn_sym) + nargs = jl_expr_nargs(warning); + int kwargs_len = (int)nargs - 6; + if (nargs < 6 || kwargs_len % 2 != 0) { + jl_error("julia-logmsg: bad argument list - expected " + ":warn level (symbol) group (symbol) id file line msg . kwargs"); + } + jl_value_t *level = jl_exprarg(warning, 0); + jl_value_t *group = jl_exprarg(warning, 1); + jl_value_t *id = jl_exprarg(warning, 2); + jl_value_t *file = jl_exprarg(warning, 3); + jl_value_t *line = jl_exprarg(warning, 4); + jl_value_t *msg = jl_exprarg(warning, 5); + kwargs = jl_alloc_vec_any(kwargs_len); + for (int i = 0; i < kwargs_len; ++i) { + jl_array_ptr_set(kwargs, i, jl_exprarg(warning, i + 6)); + } + JL_TYPECHK(logmsg, long, level); + jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg); } - JL_TYPECHK(logmsg, long, level); - jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg); + expr = jl_exprarg(expr, nargs - 1); } - expr = jl_exprarg(expr, nargs - 1); + JL_GC_POP(); + return expr; } + + jl_value_t **args; + JL_GC_PUSHARGS(args, 4); + args[0] = core_lower; + args[1] = (jl_value_t*)jl_alloc_svec(2); + jl_svecset(args[1], 0, jl_box_uint8pointer((uint8_t*)text)); + jl_svecset(args[1], 1, jl_box_long(text_len)); + args[2] = filename; + args[3] = jl_box_long(lineno); + jl_task_t *ct = jl_current_task; + size_t last_age = ct->world_age; + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + jl_value_t *result = jl_apply(args, 4); + ct->world_age = last_age; + args[0] = result; // root during error checks below JL_GC_POP(); - return expr; + return result; } // expand in a context where the expression value is unused From 2d2055dd5f6c1b391357c755f1e768a05b421977 Mon Sep 17 00:00:00 2001 From: oscarddssmith Date: Wed, 9 Apr 2025 16:28:37 -0400 Subject: [PATCH 3/7] fix args --- src/ast.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ast.c b/src/ast.c index fb727b58f533a..88539de06105d 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1358,17 +1358,16 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t * } jl_value_t **args; - JL_GC_PUSHARGS(args, 4); + JL_GC_PUSHARGS(args, 5); args[0] = core_lower; - args[1] = (jl_value_t*)jl_alloc_svec(2); - jl_svecset(args[1], 0, jl_box_uint8pointer((uint8_t*)text)); - jl_svecset(args[1], 1, jl_box_long(text_len)); - args[2] = filename; - args[3] = jl_box_long(lineno); + args[1] = expr + args[2] = inmodule + args[3] = filename; + args[4] = jl_box_long(lineno); jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - jl_value_t *result = jl_apply(args, 4); + jl_value_t *result = jl_apply(args, 5); ct->world_age = last_age; args[0] = result; // root during error checks below JL_GC_POP(); From 67916ca7417712b79b7806dfb5f80a9145fc698f Mon Sep 17 00:00:00 2001 From: oscarddssmith Date: Wed, 9 Apr 2025 17:56:53 -0400 Subject: [PATCH 4/7] fixes --- base/boot.jl | 15 +++++++++++++++ src/ast.c | 18 ++++++++---------- src/toplevel.c | 2 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index f9208c3874229..d43b4df0c0c10 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -1015,6 +1015,21 @@ _parse = nothing _setparser!(parser) = setglobal!(Core, :_parse, parser) + +# Binding for the julia lowerer, called as +# +# Core._lower(expr, module, filename, linenum) +# +# Lower Julia code from the buffer `expr`, attributing +# it to `filename`. `expr` must be the result of `_parse` +# +# `_lower` must return an Expr with appropriate internals to be evaled +# +# The internal jl_expand_with_loc_warn will call into Core._lower if not `nothing`. +_lower = nothing + +_setlowerer!(lowerer) = setglobal!(Core, :_lowerer, lowerer) + # support for deprecated uses of builtin functions _apply(x...) = _apply_iterate(Main.Base.iterate, x...) const _apply_pure = _apply diff --git a/src/ast.c b/src/ast.c index 88539de06105d..b4053ab06b09a 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1307,13 +1307,13 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t * const char *file, int line) { JL_TIMING(LOWERING, LOWERING); + jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); jl_value_t *core_lower = NULL; if (jl_core_module) { - core_lower = jl_get_global(jl_core_module, jl_symbol("_lower")); + core_lower = jl_get_global(jl_core_module, jl_symbol("_parse")); } if (!core_lower || core_lower == jl_nothing) { // In bootstrap, directly call the builtin lowerer. - jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); jl_array_t *kwargs = NULL; JL_GC_PUSH2(&expr, &kwargs); expr = jl_copy_ast(expr); @@ -1356,20 +1356,18 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t * JL_GC_POP(); return expr; } - jl_value_t **args; - JL_GC_PUSHARGS(args, 5); + JL_GC_PUSHARGS(args, 4); args[0] = core_lower; - args[1] = expr - args[2] = inmodule - args[3] = filename; - args[4] = jl_box_long(lineno); + args[1] = inmodule; + args[2] = filename; + args[3] = jl_box_long(lineno); jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - jl_value_t *result = jl_apply(args, 5); + jl_value_t *result = jl_apply(args, 6); ct->world_age = last_age; - args[0] = result; // root during error checks below + JL_TYPECHK(parse, expr, result); JL_GC_POP(); return result; } diff --git a/src/toplevel.c b/src/toplevel.c index f1fff694926ba..21e48d904fda0 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -1128,7 +1128,7 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, ast = jl_svecref(jl_parse(jl_string_data(text), jl_string_len(text), filename, 1, 0, (jl_value_t*)jl_all_sym), 0); if (!jl_is_expr(ast) || ((jl_expr_t*)ast)->head != jl_toplevel_sym) { - jl_errorf("jl_parse_all() must generate a top level expression"); + jl_errorf("jl_parse() must generate a top level expression"); } jl_task_t *ct = jl_current_task; From aade3b572b6794a312525436be936e434d4d21c0 Mon Sep 17 00:00:00 2001 From: oscarddssmith Date: Thu, 10 Apr 2025 11:19:46 -0400 Subject: [PATCH 5/7] fixes --- src/ast.c | 9 +++++++-- stdlib/InteractiveUtils/src/macros.jl | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/ast.c b/src/ast.c index b4053ab06b09a..e1625457c3b62 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1356,12 +1356,17 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t * JL_GC_POP(); return expr; } + jl_value_t **args; JL_GC_PUSHARGS(args, 4); args[0] = core_lower; - args[1] = inmodule; + args[1] = (jl_value_t*) inmodule; + jl_value_t *filename = NULL; + JL_GC_PUSH1(&filename); + filename = jl_cstr_to_string(file); args[2] = filename; - args[3] = jl_box_long(lineno); + JL_GC_POP(); + args[3] = jl_box_long(line); jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); diff --git a/stdlib/InteractiveUtils/src/macros.jl b/stdlib/InteractiveUtils/src/macros.jl index 94ffde0fd0032..5538d79c04d1e 100644 --- a/stdlib/InteractiveUtils/src/macros.jl +++ b/stdlib/InteractiveUtils/src/macros.jl @@ -455,6 +455,9 @@ For `@activate Compiler`, the following options are available: 2. `:codegen` - Activate the compiler for internal codegen purposes. The new compiler will be invoked whenever the runtime requests compilation. +For `@activate JuliaSyntax`, the following options are available: +1. `:for_lowering` - Activate JuliaSyntax such that it is compatible with JuliaLowering + `@activate Compiler` without options is equivalent to `@activate Compiler[:reflection]`. """ From 02dbe062bdbb5e04ade40f33145d31465fa5a1c1 Mon Sep 17 00:00:00 2001 From: oscarddssmith Date: Thu, 10 Apr 2025 12:46:04 -0400 Subject: [PATCH 6/7] fixes --- base/boot.jl | 2 +- src/ast.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index d43b4df0c0c10..f34fb84d31ecd 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -1028,7 +1028,7 @@ _setparser!(parser) = setglobal!(Core, :_parse, parser) # The internal jl_expand_with_loc_warn will call into Core._lower if not `nothing`. _lower = nothing -_setlowerer!(lowerer) = setglobal!(Core, :_lowerer, lowerer) +_setlowerer!(lowerer) = setglobal!(Core, :_lower, lowerer) # support for deprecated uses of builtin functions _apply(x...) = _apply_iterate(Main.Base.iterate, x...) diff --git a/src/ast.c b/src/ast.c index e1625457c3b62..868a242e74e70 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1310,7 +1310,7 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t * jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); jl_value_t *core_lower = NULL; if (jl_core_module) { - core_lower = jl_get_global(jl_core_module, jl_symbol("_parse")); + core_lower = jl_get_global(jl_core_module, jl_symbol("_lower")); } if (!core_lower || core_lower == jl_nothing) { // In bootstrap, directly call the builtin lowerer. From 81019704a221c89a1e710e95ff5b6198d5cd45eb Mon Sep 17 00:00:00 2001 From: oscarddssmith Date: Thu, 10 Apr 2025 14:33:25 -0400 Subject: [PATCH 7/7] fixes --- src/ast.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ast.c b/src/ast.c index 868a242e74e70..a2c68fddd22e4 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1358,19 +1358,20 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t * } jl_value_t **args; - JL_GC_PUSHARGS(args, 4); + JL_GC_PUSHARGS(args, 5); args[0] = core_lower; - args[1] = (jl_value_t*) inmodule; + args[1] = expr; + args[2] = (jl_value_t*) inmodule; jl_value_t *filename = NULL; JL_GC_PUSH1(&filename); filename = jl_cstr_to_string(file); - args[2] = filename; + args[3] = filename; JL_GC_POP(); - args[3] = jl_box_long(line); + args[4] = jl_box_long(line); jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - jl_value_t *result = jl_apply(args, 6); + jl_value_t *result = jl_apply(args, 5); ct->world_age = last_age; JL_TYPECHK(parse, expr, result); JL_GC_POP();