From 78a19718f8702205af4feb2fe63f71411b9ce294 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 29 Jul 2025 01:32:13 +0000 Subject: [PATCH] WIP/RFC: Change lowering of GC.preserve to be lexical # Disclaimer This is an attempt to resolve #59124. It is incompletely implemented, but I think the key pieces are implemented in each of the involved subsystems. The idea of this PR is to facilitate discussion of this solution while making sure there aren't any major unknown unknowns (which I don't think there are). # Design This PR adds a new struct defined as: ``` struct GCPreserveDuring f::Any # N.B: This field is opaque - the compiler is allowed to arbitrarily change it # as long as it has the same GC rooting behavior. root::Any GCPreserveDuring(@nospecialize(f), @nospecialize(root)) = new(f, root) end # This has special support in inference and codegen and is only ever actually called # in fallback cases. function (this::GCPreserveDuring)(args...) @noinline r = this.f(args...) # N.B.: This is correct, but stronger than required. If the call to `f` is deleted, # this may be deleted as well. donotdelete(this.root) return r end ``` The idea is that the call method here exists for semantics, but is essentially never actually used. Instead, inference treats it transparently (as if a call to the wrapped function) and codegen codegens it as the original call plus a `jl_roots` operand bundle (which our LLVM passes already supported, because it's used in the gc preserve implementation for foreigncall). # Key notes for relevant subsystems ## Lowering In lowering, the `gc_preserve` syntax form is expanded by first expanding its first argument and then rewriting every call in the expansion from `(call f ,@args)` to `(call (new (top GCPreserveDuring) f preservee) ,@args)`. As lowering is lexical, this will apply to every call in the lexical scope of the macro (and no other calls). Now, of course the preservee itself is not lexical. Lowering itself does not care and simply copies whatever value you put there. However, to preserve the existing semantics of the `@GC.preserve` macro, the implementation of the macro has changed to introduce and intermediate slot (thus capturing the value of the preserved slot at entry to the macro). As a result, the case where the entry to the macro is not evaluated for any reason now errors with an UndefVarError, rather than crashing in the optimizer. ## Inlining Inlining for calls of `GCPreserveDuring` is adjusted to apply the same `GCPreserveDuring` to every statement being inlined. ## :invoke_modify The `:invoke_modify` expr head gains a new case for invokes of `GCPreserveDuring`, which if recognized as such are treated the same as regular `:invoke`s, except that the preservation is applied. I should note though that this causes problems for existing `:invoke_modify`', since there is a semantic ambiguity of whether they are permitted to be decayed to `:invoke`. Currently this is handled by just bailing in this case, but we may want a new expr head instead. ## Codegen Codegen is the least implemented. In the current design, the `jl_cgval_t` struct gains a new `wrapped_typ` field that if set indicates that this is a GCPreserveDuring of the indicated type. `boxed` would reconstitute it as such and the various `emit_` wrappers would turn it back into a non-wrapped `jl_cgval_t` and treat the preservees as appropriate. I'm not sure this'll be the final design. In any case, the preservees are turned into jl_roots operand bundles. --- Compiler/src/abstractinterpretation.jl | 12 +++ Compiler/src/ssair/inlining.jl | 117 +++++++++++++++++++++---- Compiler/src/stmtinfo.jl | 11 +++ base/Base.jl | 5 ++ base/Base_compiler.jl | 3 +- base/boot.jl | 19 ++++ base/gcutils.jl | 11 ++- base/sysimg.jl | 36 ++++---- src/cgutils.cpp | 5 +- src/codegen.cpp | 65 +++++++++----- src/jl_exported_data.inc | 1 + src/jlfrontend.scm | 8 +- src/jltypes.c | 1 + src/julia-syntax.scm | 19 ++-- src/julia.h | 1 + src/staticdata.c | 3 +- 16 files changed, 242 insertions(+), 75 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 100cdd97e511a..9637b7d3287b4 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2883,6 +2883,14 @@ function abstract_call_unknown(interp::AbstractInterpreter, @nospecialize(ft), return Future(CallMeta(rewrap_unionall(uft.parameters[2], wft), Any, Effects(), NoCallInfo())) end return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) + elseif hasintersect(wft, Core.GCPreserveDuring) + wrapped_ft = getfield_tfunc(typeinf_lattice(interp), ft, Const(1)) + newargtypes = copy(arginfo.argtypes) + newargtypes[1] = wrapped_ft + call = abstract_call(interp, ArgInfo(nothing, newargtypes), si, sv, max_methods) + return Future{CallMeta}(call, interp, sv) do call, interp, sv + CallMeta(call.rt, call.exct, call.effects, GCPreserveDuringCallInfo(call.info)) + end end # non-constant function, but the number of arguments is known and the `f` is not a builtin or intrinsic atype = argtypes_to_type(arginfo.argtypes) @@ -3158,6 +3166,10 @@ function abstract_eval_new(interp::AbstractInterpreter, e::Expr, sstate::Stateme end end end + # Special case GCPreserveDuring's root type is not observable - don't try to const prop it + if rt === GCPreserveDuring + ats[2] = Any + end rt = PartialStruct(𝕃ᵢ, rt, undefs, ats) end else diff --git a/Compiler/src/ssair/inlining.jl b/Compiler/src/ssair/inlining.jl index 251767d577157..dfb83fa3655de 100644 --- a/Compiler/src/ssair/inlining.jl +++ b/Compiler/src/ssair/inlining.jl @@ -26,6 +26,16 @@ function InliningTodo(mi::MethodInstance, ir::IRCode, spec_info::SpecInfo, di::D return InliningTodo(mi, ir, spec_info, di, linear_inline_eligible(ir), effects) end +struct GCPreserveRewrite + actual_callee::Any + preservee::Any +end + +struct GCPreserveInliningTodo + gc_preserve::GCPreserveRewrite + inlining::InliningTodo +end + struct ConstantCase val::Any edge::CodeInstance @@ -357,8 +367,31 @@ function adjust_boundscheck!(inline_compact::IncrementalCompact, idx′::Int, st return nothing end +function apply_gc_preserve_inlining!(insert_node!::Inserter, inline_compact::IncrementalCompact, line, @nospecialize(stmt′), @nospecialize(gc_preserve)) + if isexpr(stmt′, :invoke_modify) + # TODO: The combination of an existing invoke_modify and gc_preserve causes trouble. Bail for now. + stmt′.head = :call + popfirst!(stmt′.args) + end + if isexpr(stmt′, :call) || isexpr(stmt′, :invoke) + refarg = stmt′.head === :call ? 1 : 2 + if stmt′.head === :invoke + stmt′.head = :invoke_modify + end + oldarg = stmt′.args[refarg] + stmt′.args[refarg] = insert_node!(NewInstruction( + Expr(:new, GlobalRef(Core, :GCPreserveDuring), oldarg, gc_preserve), + PartialStruct(Core.GCPreserveDuring, + Any[argextype(oldarg, inline_compact), argextype(gc_preserve, inline_compact)]), + NoCallInfo(), line, + IR_FLAG_CONSISTENT | IR_FLAG_EFFECT_FREE | IR_FLAG_NOTHROW)) + elseif isexpr(stmt′, :foreigncall) + push!(stmt′.args, gc_preserve) + end +end + function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector{Any}, - item::InliningTodo, boundscheck::Symbol, todo_bbs::Vector{Tuple{Int, Int}}) + item::InliningTodo, boundscheck::Symbol, todo_bbs::Vector{Tuple{Int, Int}}, @nospecialize(gc_preserve::Any)) # Ok, do the inlining here inlined_at = compact.result[idx][:line] ssa_substitute = ir_prepare_inlining!(InsertHere(compact), compact, item.ir, item.spec_info, item.di, item.mi, inlined_at, argexprs) @@ -380,7 +413,8 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector # something better eventually. inline_compact[idx′] = nothing # alter the line number information for InsertBefore to point to the current instruction in the new linetable - inline_compact[SSAValue(idx′)][:line] = (ssa_substitute.inlined_at[1], ssa_substitute.inlined_at[2], Int32(lineidx)) + line = (ssa_substitute.inlined_at[1], ssa_substitute.inlined_at[2], Int32(lineidx)) + inline_compact[SSAValue(idx′)][:line] = line insert_node! = InsertBefore(inline_compact, SSAValue(idx′)) stmt′ = ssa_substitute_op!(insert_node!, inline_compact[SSAValue(idx′)], stmt′, ssa_substitute) if isa(stmt′, ReturnNode) @@ -394,6 +428,8 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector break elseif isexpr(stmt′, :boundscheck) adjust_boundscheck!(inline_compact, idx′, stmt′, boundscheck) + elseif gc_preserve !== nothing + apply_gc_preserve_inlining!(insert_node!, inline_compact, line, stmt′, gc_preserve) end inline_compact[idx′] = stmt′ end @@ -412,7 +448,8 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector @assert isempty(inline_compact.perm) && isempty(inline_compact.pending_perm) "linetable not in canonical form (missing compact call)" for ((lineidx, idx′), stmt′) in inline_compact inline_compact[idx′] = nothing - inline_compact[SSAValue(idx′)][:line] = (ssa_substitute.inlined_at[1], ssa_substitute.inlined_at[2], Int32(lineidx)) + line = (ssa_substitute.inlined_at[1], ssa_substitute.inlined_at[2], Int32(lineidx)) + inline_compact[SSAValue(idx′)][:line] = line insert_node! = InsertBefore(inline_compact, SSAValue(idx′)) stmt′ = ssa_substitute_op!(insert_node!, inline_compact[SSAValue(idx′)], stmt′, ssa_substitute) if isa(stmt′, ReturnNode) @@ -433,6 +470,8 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector stmt′ = PhiNode(Int32[edge+bb_offset for edge in stmt′.edges], stmt′.values) elseif isexpr(stmt′, :boundscheck) adjust_boundscheck!(inline_compact, idx′, stmt′, boundscheck) + elseif gc_preserve !== nothing + apply_gc_preserve_inlining!(insert_node!, inline_compact, line, stmt′, gc_preserve) end inline_compact[idx′] = stmt′ end @@ -578,7 +617,7 @@ function ir_inline_unionsplit!(compact::IncrementalCompact, idx::Int, argexprs:: end end if isa(case, InliningTodo) - val = ir_inline_item!(compact, idx, argexprs′, case, boundscheck, todo_bbs) + val = ir_inline_item!(compact, idx, argexprs′, case, boundscheck, todo_bbs, nothing) elseif isa(case, InvokeCase) invoke_stmt = Expr(:invoke, case.invoke, argexprs′...) flag = flags_for_effects(case.effects) @@ -624,7 +663,19 @@ function batch_inline!(ir::IRCode, todo::Vector{Pair{Int,Any}}, propagate_inboun if isa(item, UnionSplit) cfg_inline_unionsplit!(ir, idx, item, state, params) else - item = item::InliningTodo + if isa(item, GCPreserveInliningTodo) + item = item::GCPreserveInliningTodo + # Rewrite the call now to drop the GCPreserveDuring, since we're committed to inlining. + # This makes sure that it gets renamed properly. We also need to rename the GC preservee, + # but we don't have a good place to put that, so temporarily append it to the argument list - + # we'll undo this below. + stmt = ir[SSAValue(idx)][:stmt] + stmt.args[1] = item.gc_preserve.actual_callee + push!(stmt.args, item.gc_preserve.preservee) + item = item.inlining + else + item = item::InliningTodo + end # A linear inline does not modify the CFG item.linear_inline_eligible && continue cfg_inline_item!(ir, idx, item, state, false) @@ -660,7 +711,9 @@ function batch_inline!(ir::IRCode, todo::Vector{Pair{Int,Any}}, propagate_inboun refinish = true end if isa(item, InliningTodo) - compact.ssa_rename[old_idx] = ir_inline_item!(compact, idx, argexprs, item, boundscheck, state.todo_bbs) + compact.ssa_rename[old_idx] = ir_inline_item!(compact, idx, argexprs, item, boundscheck, state.todo_bbs, nothing) + elseif isa(item, GCPreserveInliningTodo) + compact.ssa_rename[old_idx] = ir_inline_item!(compact, idx, argexprs, item.inlining, boundscheck, state.todo_bbs, pop!(argexprs)) elseif isa(item, UnionSplit) compact.ssa_rename[old_idx] = ir_inline_unionsplit!(compact, idx, argexprs, item, boundscheck, state.todo_bbs, interp) end @@ -988,7 +1041,7 @@ function retrieve_ir_for_inlining(mi::MethodInstance, opt::OptimizationState, pr end function handle_single_case!(todo::Vector{Pair{Int,Any}}, - ir::IRCode, idx::Int, stmt::Expr, @nospecialize(case), + ir::IRCode, idx::Int, stmt::Expr, @nospecialize(case), gc_preserve::Union{GCPreserveRewrite, Nothing}, isinvoke::Bool = false) if isa(case, ConstantCase) ir[SSAValue(idx)][:stmt] = case.val @@ -996,9 +1049,12 @@ function handle_single_case!(todo::Vector{Pair{Int,Any}}, is_foldable_nothrow(case.effects) && inline_const_if_inlineable!(ir[SSAValue(idx)]) && return nothing isinvoke && rewrite_invoke_exprargs!(stmt) if stmt.head === :invoke + if gc_preserve !== nothing + stmt.head = :invoke_modify + end stmt.args[1] = case.invoke else - stmt.head = :invoke + stmt.head = gc_preserve === nothing ? :invoke : :invoke_modify pushfirst!(stmt.args, case.invoke) end add_flag!(ir[SSAValue(idx)], flags_for_effects(case.effects)) @@ -1006,7 +1062,9 @@ function handle_single_case!(todo::Vector{Pair{Int,Any}}, # Do, well, nothing else isinvoke && rewrite_invoke_exprargs!(stmt) - push!(todo, idx=>(case::InliningTodo)) + case = case::InliningTodo + push!(todo, idx=>gc_preserve === nothing ? case : + GCPreserveInliningTodo(gc_preserve, case)) end return nothing end @@ -1237,6 +1295,7 @@ end # functions. function process_simple!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, flag::UInt32, state::InliningState) + gc_preserve = nothing inst = ir[SSAValue(idx)] stmt = inst[:stmt] if !(stmt isa Expr) @@ -1262,6 +1321,22 @@ function process_simple!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, flag sig = call_sig(ir, stmt) sig === nothing && return nothing + # Handle GCPreserveDuring + if isa(inst[:info], GCPreserveDuringCallInfo) + # See if we can strip the construction of this + wrapped_f = stmt.args[1] + if isa(wrapped_f, SSAValue) + wrapping_stmt = ir[wrapped_f][:stmt] + if isexpr(wrapping_stmt, :new) + ft = argextype(wrapping_stmt.args[2], ir) + gc_preserve = GCPreserveRewrite(wrapping_stmt.args[2], wrapping_stmt.args[3]) + new_argtypes = copy(sig.argtypes) + new_argtypes[1] = ft + sig = Signature(singleton_type(ft), ft, new_argtypes) + end + end + end + # Handle _apply_iterate sig = inline_apply!(todo, ir, idx, stmt, sig, state) sig === nothing && return nothing @@ -1303,7 +1378,7 @@ function process_simple!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, flag return nothing end - return stmt, sig + return stmt, sig, gc_preserve end function handle_any_const_result!(cases::Vector{InliningCase}, @@ -1409,13 +1484,13 @@ end function handle_call!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, @nospecialize(info::CallInfo), flag::UInt32, sig::Signature, - state::InliningState) + state::InliningState, gc_preserve::Union{GCPreserveRewrite, Nothing}) cases = compute_inlining_cases(info, flag, sig, state) cases === nothing && return nothing cases, handled_all_cases, fully_covered, joint_effects = cases atype = argtypes_to_type(sig.argtypes) atype === Union{} && return nothing # accidentally actually unreachable - handle_cases!(todo, ir, idx, stmt, atype, cases, handled_all_cases, fully_covered, joint_effects) + handle_cases!(todo, ir, idx, stmt, atype, cases, handled_all_cases, fully_covered, joint_effects, gc_preserve) end function handle_match!(cases::Vector{InliningCase}, @@ -1499,18 +1574,18 @@ end function handle_cases!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, @nospecialize(atype), cases::Vector{InliningCase}, handled_all_cases::Bool, fully_covered::Bool, - joint_effects::Effects) + joint_effects::Effects, gc_preserve::Union{GCPreserveRewrite, Nothing}) # If we only have one case and that case is fully covered, we may either # be able to do the inlining now (for constant cases), or push it directly # onto the todo list if fully_covered && handled_all_cases && length(cases) == 1 - handle_single_case!(todo, ir, idx, stmt, cases[1].item) + handle_single_case!(todo, ir, idx, stmt, cases[1].item, gc_preserve) elseif length(cases) > 0 || handled_all_cases isa(atype, DataType) || return nothing for case in cases isa(case.sig, DataType) || return nothing end - push!(todo, idx=>UnionSplit(handled_all_cases, fully_covered, atype, cases)) + push!(todo, idx=>UnionSplit(handled_all_cases, fully_covered, atype, cases, gc_preserve)) else add_flag!(ir[SSAValue(idx)], flags_for_effects(joint_effects)) end @@ -1629,13 +1704,17 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState) simpleres = process_simple!(todo, ir, idx, flag, state) simpleres === nothing && continue - stmt, sig = simpleres + stmt, sig, gc_preserve = simpleres info = ir.stmts[idx][:info] + if gc_preserve !== nothing && isa(info, GCPreserveDuringCallInfo) + info = info.info + end # `NativeInterpreter` won't need this, but provide a support for `:invoke` exprs here # for external `AbstractInterpreter`s that may run the inlining pass multiple times if isexpr(stmt, :invoke) + gc_preserve === nothing || continue handle_invoke_expr!(todo, ir, idx, stmt, info, flag, sig, state) continue end @@ -1652,16 +1731,20 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState) # handle special cased builtins if isa(info, OpaqueClosureCallInfo) + gc_preserve === nothing || continue handle_opaque_closure_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, ModifyOpInfo) + gc_preserve === nothing || continue handle_modifyop!_call!(ir, idx, stmt, info, state) elseif sig.f === Core.invoke + gc_preserve === nothing || continue handle_invoke_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, FinalizerInfo) + gc_preserve === nothing || continue handle_finalizer_call!(ir, idx, stmt, info, state) else # cascade to the generic (and extendable) handler - handle_call!(todo, ir, idx, stmt, info, flag, sig, state) + handle_call!(todo, ir, idx, stmt, info, flag, sig, state, gc_preserve) end end diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index d6a63d8f71abf..153257d4f244f 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -502,4 +502,15 @@ function add_edges_impl(edges::Vector{Any}, info::GlobalAccessInfo) push!(edges, info.b) end +""" + info::GCPreserveCallInfo <: CallInfo + +Wraps another CallInfo, indicating that this call came from a looked-through GCPreserveDuring. +""" +struct GCPreserveDuringCallInfo <: CallInfo + info::CallInfo +end +add_edges_impl(edges::Vector{Any}, info::GCPreserveDuringCallInfo) = + add_edges_impl(edges, info.info) + @specialize diff --git a/base/Base.jl b/base/Base.jl index 9d510b5c5d47c..b5de6dfa5d894 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -370,6 +370,11 @@ function start_profile_listener() ccall(:jl_set_peek_cond, Cvoid, (Ptr{Cvoid},), cond.handle) end +function print_padded_time(io, mod, maxlen, t) + print(io, rpad(string(mod) * " ", maxlen + 3, "─")) + Base.time_print(io, t * 10^9); println(io) +end + function __init__() # Base library init global _atexit_hooks_finished = false diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index ef448a02a15e9..3cdf1611c39c5 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -136,7 +136,8 @@ convert(::Type{Any}, Core.@nospecialize x) = x convert(::Type{T}, x::T) where {T} = x include("coreio.jl") -import Core: @doc, @__doc__, WrappedException, @int128_str, @uint128_str, @big_str, @cmd +import Core: @doc, @__doc__, WrappedException, @int128_str, @uint128_str, @big_str, @cmd, + GCPreserveDuring # Export list include("exports.jl") diff --git a/base/boot.jl b/base/boot.jl index d055c47516f91..f934b5da58801 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -1141,4 +1141,23 @@ typename(union::UnionAll) = typename(union.body) include(Core, "optimized_generics.jl") +struct GCPreserveDuring + f::Any + # N.B: This field is opaque - the compiler is allowed to arbitrarily change it + # as long as it has the same GC rooting behavior. + root::Any + GCPreserveDuring(@nospecialize(f), @nospecialize(root)) = new(f, root) +end + +# This has special support in inference and codegen and is only ever actually called +# in fallback cases. +function (this::GCPreserveDuring)(args...) + @noinline + r = this.f(args...) + # N.B.: This is correct, but stronger than required. If the call to `f` is deleted, + # this may be deleted as well. + donotdelete(this.root) + return r +end + ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Core, true) diff --git a/base/gcutils.jl b/base/gcutils.jl index d5e6f4597739f..361d5e2895ca9 100644 --- a/base/gcutils.jl +++ b/base/gcutils.jl @@ -186,8 +186,8 @@ end """ GC.@preserve x1 x2 ... xn expr -Mark the objects `x1, x2, ...` as being *in use* during the evaluation of the -expression `expr`. This is only required in unsafe code where `expr` +Mark the objects `x1, x2, ...` as being *in use* during the evaluation of all +calls lexically in the expression `expr`. This is only required in unsafe code where `expr` *implicitly uses* memory or other resources owned by one of the `x`s. *Implicit use* of `x` covers any indirect use of resources logically owned by @@ -236,7 +236,12 @@ macro preserve(args...) for x in syms isa(x, Symbol) || error("Preserved variable must be a symbol") end - esc(Expr(:gc_preserve, args[end], syms...)) + sym = length(syms) == 1 ? only(syms) : Expr(:tuple, syms...) + g = gensym() + esc(quote + $g = $sym + $(Expr(:gc_preserve, args[end], g)) + end) end """ diff --git a/base/sysimg.jl b/base/sysimg.jl index fd71544c205cc..82a34fc2eec20 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -102,29 +102,23 @@ let maxlen = maximum(textwidth.(string.(stdlibs)); init=0) tot_time_stdlib = 0.0 - # use a temp module to avoid leaving the type of this closure in Main push!(empty!(LOAD_PATH), "@stdlib") - m = Core.Module() - GC.@preserve m begin - print_time = @eval m (mod, t) -> (print(rpad(string(mod) * " ", $maxlen + 3, "─")); - Base.time_print(stdout, t * 10^9); println()) - print_time(Base, (Base.end_base_include - Base.start_base_include) * 10^(-9)) - - Base._track_dependencies[] = true - tot_time_stdlib = @elapsed for stdlib in stdlibs - tt = @elapsed Base.require(Base, stdlib) - print_time(stdlib, tt) - end - for dep in Base._require_dependencies - mod, path, fsize, mtime = dep[1], dep[2], dep[3], dep[5] - (fsize == 0 || mtime == 0.0) && continue - push!(Base._included_files, (mod, path)) - end - empty!(Base._require_dependencies) - Base._track_dependencies[] = false - - print_time("Stdlibs total", tot_time_stdlib) + Base.print_padded_time(stdout, Base, maxlen, (Base.end_base_include - Base.start_base_include) * 10^(-9)) + + Base._track_dependencies[] = true + tot_time_stdlib = @elapsed for stdlib in stdlibs + tt = @elapsed Base.require(Base, stdlib) + Base.print_padded_time(stdout, stdlib, maxlen, tt) + end + for dep in Base._require_dependencies + mod, path, fsize, mtime = dep[1], dep[2], dep[3], dep[5] + (fsize == 0 || mtime == 0.0) && continue + push!(Base._included_files, (mod, path)) end + empty!(Base._require_dependencies) + Base._track_dependencies[] = false + + Base.print_padded_time(stdout, "Stdlibs total", maxlen, tot_time_stdlib) # Clear global state empty!(Core.ARGS) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 97a58f8aa419c..0be3b320cac5d 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -3807,6 +3807,10 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotab if (jt == jl_bottom_type || jt == NULL) // We have an undef value on a (hopefully) dead branch return UndefValue::get(ctx.types().T_prjlvalue); + Value *box; + if (vinfo.wrapped_typ) { + assert(false && "TODO"); + } if (vinfo.constant) return track_pjlvalue(ctx, literal_pointer_val(ctx, vinfo.constant)); // This can happen in early bootstrap for `gc_preserve_begin` return value. @@ -3818,7 +3822,6 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotab return vinfo.V; } - Value *box; if (vinfo.TIndex) { SmallBitVector skip_none; box = box_union(ctx, vinfo, skip_none); diff --git a/src/codegen.cpp b/src/codegen.cpp index 2d9c94baee9b4..14f24aa4207a4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1719,6 +1719,11 @@ struct jl_cgval_t { // handling to account for the possibility that this may be NULL. Value *Vboxed; + // If non-null, this value was an unwrapped GCPreserveDuring, where this value is the preservee. + // Calls of this value need toapply appropriate gc preserve handling. + SmallVector preservees; + jl_value_t *wrapped_typ; + Value *TIndex; // if `V` is an unboxed (tagged) Union described by `typ`, this gives the DataType index (1-based, small int) as an i8 SmallVector inline_roots; // if present, `V` is a pointer, but not in canonical layout jl_value_t *constant; // constant value (rooted in linfo.def.roots) @@ -4914,7 +4919,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // Returns ctx.types().T_prjlvalue static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, - ArrayRef argv, size_t nargs, JuliaFunction<> *trampoline) + ArrayRef argv, size_t nargs, ArrayRef roots, JuliaFunction<> *trampoline) { ++EmittedJLCalls; Function *TheTrampoline = prepare_call(trampoline); @@ -4935,7 +4940,9 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, } theArgs.push_back(arg); } - CallInst *result = ctx.builder.CreateCall(TheTrampoline, theArgs); + + OperandBundleDef OpBundle("jl_roots", roots); + CallInst *result = ctx.builder.CreateCall(TheTrampoline, theArgs, ArrayRef(&OpBundle, roots.empty() ? 0 : 1)); result->setAttributes(TheTrampoline->getAttributes()); // TODO: we could add readonly attributes in many cases to the args return result; @@ -4948,7 +4955,7 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value return emit_jlcall(ctx, prepare_call(theFptr), theF, argv, nargs, trampoline); } -static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, jl_returninfo_t &returninfo, ArrayRef argv, size_t nargs) +static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, jl_returninfo_t &returninfo, ArrayRef argv, size_t nargs, ArrayRef roots) { ++EmittedSpecfunCalls; // emit specialized call site @@ -5047,7 +5054,9 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos idx++; } assert(idx == nfargs); - CallInst *call = ctx.builder.CreateCall(returninfo.decl, argvals); + + OperandBundleDef OpBundle("jl_roots", roots); + CallInst *call = ctx.builder.CreateCall(returninfo.decl, argvals, ArrayRef(&OpBundle, roots.empty() ? 0 : 1)); call->setAttributes(returninfo.attrs); if (gcstack_arg && ctx.emission_context.use_swiftcc) call->setCallingConv(CallingConv::Swift); @@ -5140,7 +5149,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_code_instance_t } static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, ArrayRef roots, jl_value_t *inferred_retty) { Value *theFptr; if (fromexternal) { @@ -5160,7 +5169,7 @@ static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty theFptr = jl_Module->getOrInsertFunction(specFunctionObject, ctx.types().T_jlfunc).getCallee(); addRetAttr(cast(theFptr), Attribute::NonNull); } - Value *ret = emit_jlcall(ctx, theFptr, nullptr, argv, nargs, julia_call); + Value *ret = emit_jlcall(ctx, theFptr, nullptr, argv, nargs, roots, julia_call); return update_julia_type(ctx, mark_julia_type(ctx, ret, true, jlretty), inferred_retty); } @@ -5181,7 +5190,8 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) return emit_invoke(ctx, lival, argv, nargs, rt, false); } -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, bool always_inline) +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, + ArrayRef roots, bool always_inline) { ++EmittedInvokes; bool handled = false; @@ -5205,18 +5215,18 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR Function *f = ctx.f; FunctionType *ft = f->getFunctionType(); if (ft == ctx.types().T_jlfunc) { - Value *ret = emit_jlcall(ctx, f, nullptr, argv, nargs, julia_call); + Value *ret = emit_jlcall(ctx, f, nullptr, argv, nargs, roots, julia_call); result = update_julia_type(ctx, mark_julia_type(ctx, ret, true, ctx.rettype), rt); } else if (ft == ctx.types().T_jlfuncparams) { - Value *ret = emit_jlcall(ctx, f, ctx.spvals_ptr, argv, nargs, julia_call2); + Value *ret = emit_jlcall(ctx, f, ctx.spvals_ptr, argv, nargs, roots, julia_call2); result = update_julia_type(ctx, mark_julia_type(ctx, ret, true, ctx.rettype), rt); } else { unsigned return_roots = 0; jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; StringRef protoname = f->getName(); - result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, &cc, &return_roots, rt); + result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, roots, &cc, &return_roots, rt); } handled = true; } @@ -5232,7 +5242,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR bool specsig, needsparams; std::tie(specsig, needsparams) = uses_specsig(get_ci_abi(codeinst), mi, codeinst->rettype, ctx.params->prefer_specsig); if (needsparams) { - Value *r = emit_jlcall(ctx, jlinvoke_func, track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)mi)), argv, nargs, julia_call2); + Value *r = emit_jlcall(ctx, jlinvoke_func, track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)mi)), argv, nargs, roots, julia_call2); result = mark_julia_type(ctx, r, true, rt); } else { @@ -5288,9 +5298,9 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; unsigned return_roots = 0; if (specsig) - result = emit_call_specfun_other(ctx, codeinst, protoname, external ? codeinst : nullptr, argv, nargs, &cc, &return_roots, rt); + result = emit_call_specfun_other(ctx, codeinst, protoname, external ? codeinst : nullptr, argv, nargs, roots, &cc, &return_roots, rt); else - result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, rt); + result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, roots, rt); if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, nullptr, specsig, !always_inline, always_inline}; @@ -5302,7 +5312,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR } } if (!handled) { - Value *r = emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, julia_call2); + Value *r = emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, roots, julia_call2); result = mark_julia_type(ctx, r, true, rt); } if (result.typ == jl_bottom_type) { @@ -5329,7 +5339,13 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ return jl_cgval_t(); } const jl_cgval_t &f = argv[0]; - if (f.constant) { + if (f.typ == jl_gc_preserve_during_type) { + f.typ = f.wrapped_typ; + SmallVector roots = f.preservees; + f.preservees.clear(); + return emit_invoke(ctx, lival, argv, nargs, roots, rt, false); + } + else if (f.constant) { jl_cgval_t ret; auto it = builtin_func_map().end(); if (f.constant == BUILTIN(modifyfield)) { @@ -6469,11 +6485,20 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ if (jl_is_type_type(ty) && jl_is_datatype(jl_tparam0(ty)) && jl_is_concrete_type(jl_tparam0(ty))) { - assert(nargs <= jl_datatype_nfields(jl_tparam0(ty)) + 1); - jl_cgval_t res = emit_new_struct(ctx, jl_tparam0(ty), nargs - 1, ArrayRef(argv).drop_front(), is_promotable); - if (is_promotable && res.promotion_point && res.promotion_ssa==-1) - res.promotion_ssa = ssaidx_0based; - return res; + jl_value_t *allocated_type = jl_tparam0(ty); + assert(nargs <= jl_datatype_nfields(allocated_type) + 1); + if (allocated_type == jl_gc_preserve_during_type) { + jl_cgval_t res = argv[1]; + res.preservees.append(get_gc_roots_for(ctx, argv[2])); + res.wrapped_typ = res.typ; + res.typ = allocated_type; + return res; + } else { + jl_cgval_t res = emit_new_struct(ctx, allocated_type, nargs - 1, ArrayRef(argv).drop_front(), is_promotable); + if (is_promotable && res.promotion_point && res.promotion_ssa==-1) + res.promotion_ssa = ssaidx_0based; + return res; + } } Value *val = emit_jlcall(ctx, jlnew_func, nullptr, argv, nargs, julia_call); // temporarily mark as `Any`, expecting `emit_ssaval_assign` to update diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 76e8368132424..9f33ef1377c48 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -126,6 +126,7 @@ XX(jl_task_type) \ XX(jl_top_module) \ XX(jl_trimfailure_type) \ + XX(jl_gc_preserve_during_type) \ XX(jl_true) \ XX(jl_tuple_typename) \ XX(jl_tvar_type) \ diff --git a/src/jlfrontend.scm b/src/jlfrontend.scm index a678d481eab1f..a3ebb659a7d27 100644 --- a/src/jlfrontend.scm +++ b/src/jlfrontend.scm @@ -22,10 +22,10 @@ `(incomplete ,msg) (cons 'error (cdr e)))) (begin - ;;(newline) - ;;(display "unexpected error: ") - ;;(prn e) - ;;(print-stack-trace (stacktrace)) + (newline) + (display "unexpected error: ") + (prn e) + (print-stack-trace (stacktrace)) '(error "malformed expression")))) thk)) diff --git a/src/jltypes.c b/src/jltypes.c index b3434b2cbb95b..862d970f22aae 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3956,6 +3956,7 @@ void post_boot_hooks(void) jl_initerror_type = (jl_datatype_t*)core("InitError"); jl_missingcodeerror_type = (jl_datatype_t*)core("MissingCodeError"); jl_trimfailure_type = (jl_datatype_t*)core("TrimFailure"); + jl_gc_preserve_during_type = (jl_datatype_t*)core("GCPreserveDuring"); jl_pair_type = core("Pair"); jl_value_t *kwcall_func = core("kwcall"); diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 052e0000ebe87..d17f868a309d9 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2556,6 +2556,17 @@ (every check-path (cdar e))))) (error (string "malformed \"" what "\" statement")))) +(define (wrap-gc-preserve e x) + (if (pair? e) + (cond ((memq (car e) '(method lambda)) e) + ((eq? (car e) 'foreigncall) + `(,(car e) ,@(list-head (cdr e) 5) ,@(map (lambda (c) (wrap-gc-preserve c x)) (list-tail e 6)))) + ((eq? (car e) 'cfunction) e) + ((eq? (car e) 'call) + `(call (new (top GCPreserveDuring) ,(cadr e) ,x) ,@(map (lambda (c) (wrap-gc-preserve c x)) (cddr e)))) + (else `(,(car e) ,@(map (lambda (c) (wrap-gc-preserve c x)) (cdr e))))) + e)) + ;; table mapping expression head to a function expanding that form (define expand-table (table @@ -2945,13 +2956,7 @@ 'gc_preserve (lambda (e) - (let* ((s (make-ssavalue)) - (r (make-ssavalue))) - `(block - (= ,s (gc_preserve_begin ,@(cddr e))) - (= ,r ,(expand-forms (cadr e))) - (gc_preserve_end ,s) - ,r))) + (wrap-gc-preserve (expand-forms (cadr e)) (caddr e))) 'line (lambda (e) diff --git a/src/julia.h b/src/julia.h index a5564670479db..2dc985cb7861a 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1036,6 +1036,7 @@ extern JL_DLLIMPORT jl_datatype_t *jl_fielderror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_atomicerror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_missingcodeerror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_trimfailure_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_datatype_t *jl_gc_preserve_during_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_lineinfonode_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_abioverride_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_stackovf_exception JL_GLOBALLY_ROOTED; diff --git a/src/staticdata.c b/src/staticdata.c index be1188572f3b0..fa8d7937e22aa 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -116,7 +116,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 151 +#define NUM_TAGS 152 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -282,6 +282,7 @@ static void get_tags(jl_value_t **tags[NUM_TAGS]) INSERT_TAG(jl_opaque_closure_method); INSERT_TAG(jl_nulldebuginfo); INSERT_TAG(jl_method_table); + INSERT_TAG(jl_gc_preserve_during_type); // n.b. must update NUM_TAGS when you add something here #undef INSERT_TAG assert(i == NUM_TAGS - 1);