diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 100cdd97e511a..9637b7d3287b4 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2883,6 +2883,14 @@ function abstract_call_unknown(interp::AbstractInterpreter, @nospecialize(ft), return Future(CallMeta(rewrap_unionall(uft.parameters[2], wft), Any, Effects(), NoCallInfo())) end return Future(CallMeta(Any, Any, Effects(), NoCallInfo())) + elseif hasintersect(wft, Core.GCPreserveDuring) + wrapped_ft = getfield_tfunc(typeinf_lattice(interp), ft, Const(1)) + newargtypes = copy(arginfo.argtypes) + newargtypes[1] = wrapped_ft + call = abstract_call(interp, ArgInfo(nothing, newargtypes), si, sv, max_methods) + return Future{CallMeta}(call, interp, sv) do call, interp, sv + CallMeta(call.rt, call.exct, call.effects, GCPreserveDuringCallInfo(call.info)) + end end # non-constant function, but the number of arguments is known and the `f` is not a builtin or intrinsic atype = argtypes_to_type(arginfo.argtypes) @@ -3158,6 +3166,10 @@ function abstract_eval_new(interp::AbstractInterpreter, e::Expr, sstate::Stateme end end end + # Special case GCPreserveDuring's root type is not observable - don't try to const prop it + if rt === GCPreserveDuring + ats[2] = Any + end rt = PartialStruct(𝕃ᵢ, rt, undefs, ats) end else diff --git a/Compiler/src/ssair/inlining.jl b/Compiler/src/ssair/inlining.jl index 251767d577157..dfb83fa3655de 100644 --- a/Compiler/src/ssair/inlining.jl +++ b/Compiler/src/ssair/inlining.jl @@ -26,6 +26,16 @@ function InliningTodo(mi::MethodInstance, ir::IRCode, spec_info::SpecInfo, di::D return InliningTodo(mi, ir, spec_info, di, linear_inline_eligible(ir), effects) end +struct GCPreserveRewrite + actual_callee::Any + preservee::Any +end + +struct GCPreserveInliningTodo + gc_preserve::GCPreserveRewrite + inlining::InliningTodo +end + struct ConstantCase val::Any edge::CodeInstance @@ -357,8 +367,31 @@ function adjust_boundscheck!(inline_compact::IncrementalCompact, idx′::Int, st return nothing end +function apply_gc_preserve_inlining!(insert_node!::Inserter, inline_compact::IncrementalCompact, line, @nospecialize(stmt′), @nospecialize(gc_preserve)) + if isexpr(stmt′, :invoke_modify) + # TODO: The combination of an existing invoke_modify and gc_preserve causes trouble. Bail for now. + stmt′.head = :call + popfirst!(stmt′.args) + end + if isexpr(stmt′, :call) || isexpr(stmt′, :invoke) + refarg = stmt′.head === :call ? 1 : 2 + if stmt′.head === :invoke + stmt′.head = :invoke_modify + end + oldarg = stmt′.args[refarg] + stmt′.args[refarg] = insert_node!(NewInstruction( + Expr(:new, GlobalRef(Core, :GCPreserveDuring), oldarg, gc_preserve), + PartialStruct(Core.GCPreserveDuring, + Any[argextype(oldarg, inline_compact), argextype(gc_preserve, inline_compact)]), + NoCallInfo(), line, + IR_FLAG_CONSISTENT | IR_FLAG_EFFECT_FREE | IR_FLAG_NOTHROW)) + elseif isexpr(stmt′, :foreigncall) + push!(stmt′.args, gc_preserve) + end +end + function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector{Any}, - item::InliningTodo, boundscheck::Symbol, todo_bbs::Vector{Tuple{Int, Int}}) + item::InliningTodo, boundscheck::Symbol, todo_bbs::Vector{Tuple{Int, Int}}, @nospecialize(gc_preserve::Any)) # Ok, do the inlining here inlined_at = compact.result[idx][:line] ssa_substitute = ir_prepare_inlining!(InsertHere(compact), compact, item.ir, item.spec_info, item.di, item.mi, inlined_at, argexprs) @@ -380,7 +413,8 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector # something better eventually. inline_compact[idx′] = nothing # alter the line number information for InsertBefore to point to the current instruction in the new linetable - inline_compact[SSAValue(idx′)][:line] = (ssa_substitute.inlined_at[1], ssa_substitute.inlined_at[2], Int32(lineidx)) + line = (ssa_substitute.inlined_at[1], ssa_substitute.inlined_at[2], Int32(lineidx)) + inline_compact[SSAValue(idx′)][:line] = line insert_node! = InsertBefore(inline_compact, SSAValue(idx′)) stmt′ = ssa_substitute_op!(insert_node!, inline_compact[SSAValue(idx′)], stmt′, ssa_substitute) if isa(stmt′, ReturnNode) @@ -394,6 +428,8 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector break elseif isexpr(stmt′, :boundscheck) adjust_boundscheck!(inline_compact, idx′, stmt′, boundscheck) + elseif gc_preserve !== nothing + apply_gc_preserve_inlining!(insert_node!, inline_compact, line, stmt′, gc_preserve) end inline_compact[idx′] = stmt′ end @@ -412,7 +448,8 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector @assert isempty(inline_compact.perm) && isempty(inline_compact.pending_perm) "linetable not in canonical form (missing compact call)" for ((lineidx, idx′), stmt′) in inline_compact inline_compact[idx′] = nothing - inline_compact[SSAValue(idx′)][:line] = (ssa_substitute.inlined_at[1], ssa_substitute.inlined_at[2], Int32(lineidx)) + line = (ssa_substitute.inlined_at[1], ssa_substitute.inlined_at[2], Int32(lineidx)) + inline_compact[SSAValue(idx′)][:line] = line insert_node! = InsertBefore(inline_compact, SSAValue(idx′)) stmt′ = ssa_substitute_op!(insert_node!, inline_compact[SSAValue(idx′)], stmt′, ssa_substitute) if isa(stmt′, ReturnNode) @@ -433,6 +470,8 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector stmt′ = PhiNode(Int32[edge+bb_offset for edge in stmt′.edges], stmt′.values) elseif isexpr(stmt′, :boundscheck) adjust_boundscheck!(inline_compact, idx′, stmt′, boundscheck) + elseif gc_preserve !== nothing + apply_gc_preserve_inlining!(insert_node!, inline_compact, line, stmt′, gc_preserve) end inline_compact[idx′] = stmt′ end @@ -578,7 +617,7 @@ function ir_inline_unionsplit!(compact::IncrementalCompact, idx::Int, argexprs:: end end if isa(case, InliningTodo) - val = ir_inline_item!(compact, idx, argexprs′, case, boundscheck, todo_bbs) + val = ir_inline_item!(compact, idx, argexprs′, case, boundscheck, todo_bbs, nothing) elseif isa(case, InvokeCase) invoke_stmt = Expr(:invoke, case.invoke, argexprs′...) flag = flags_for_effects(case.effects) @@ -624,7 +663,19 @@ function batch_inline!(ir::IRCode, todo::Vector{Pair{Int,Any}}, propagate_inboun if isa(item, UnionSplit) cfg_inline_unionsplit!(ir, idx, item, state, params) else - item = item::InliningTodo + if isa(item, GCPreserveInliningTodo) + item = item::GCPreserveInliningTodo + # Rewrite the call now to drop the GCPreserveDuring, since we're committed to inlining. + # This makes sure that it gets renamed properly. We also need to rename the GC preservee, + # but we don't have a good place to put that, so temporarily append it to the argument list - + # we'll undo this below. + stmt = ir[SSAValue(idx)][:stmt] + stmt.args[1] = item.gc_preserve.actual_callee + push!(stmt.args, item.gc_preserve.preservee) + item = item.inlining + else + item = item::InliningTodo + end # A linear inline does not modify the CFG item.linear_inline_eligible && continue cfg_inline_item!(ir, idx, item, state, false) @@ -660,7 +711,9 @@ function batch_inline!(ir::IRCode, todo::Vector{Pair{Int,Any}}, propagate_inboun refinish = true end if isa(item, InliningTodo) - compact.ssa_rename[old_idx] = ir_inline_item!(compact, idx, argexprs, item, boundscheck, state.todo_bbs) + compact.ssa_rename[old_idx] = ir_inline_item!(compact, idx, argexprs, item, boundscheck, state.todo_bbs, nothing) + elseif isa(item, GCPreserveInliningTodo) + compact.ssa_rename[old_idx] = ir_inline_item!(compact, idx, argexprs, item.inlining, boundscheck, state.todo_bbs, pop!(argexprs)) elseif isa(item, UnionSplit) compact.ssa_rename[old_idx] = ir_inline_unionsplit!(compact, idx, argexprs, item, boundscheck, state.todo_bbs, interp) end @@ -988,7 +1041,7 @@ function retrieve_ir_for_inlining(mi::MethodInstance, opt::OptimizationState, pr end function handle_single_case!(todo::Vector{Pair{Int,Any}}, - ir::IRCode, idx::Int, stmt::Expr, @nospecialize(case), + ir::IRCode, idx::Int, stmt::Expr, @nospecialize(case), gc_preserve::Union{GCPreserveRewrite, Nothing}, isinvoke::Bool = false) if isa(case, ConstantCase) ir[SSAValue(idx)][:stmt] = case.val @@ -996,9 +1049,12 @@ function handle_single_case!(todo::Vector{Pair{Int,Any}}, is_foldable_nothrow(case.effects) && inline_const_if_inlineable!(ir[SSAValue(idx)]) && return nothing isinvoke && rewrite_invoke_exprargs!(stmt) if stmt.head === :invoke + if gc_preserve !== nothing + stmt.head = :invoke_modify + end stmt.args[1] = case.invoke else - stmt.head = :invoke + stmt.head = gc_preserve === nothing ? :invoke : :invoke_modify pushfirst!(stmt.args, case.invoke) end add_flag!(ir[SSAValue(idx)], flags_for_effects(case.effects)) @@ -1006,7 +1062,9 @@ function handle_single_case!(todo::Vector{Pair{Int,Any}}, # Do, well, nothing else isinvoke && rewrite_invoke_exprargs!(stmt) - push!(todo, idx=>(case::InliningTodo)) + case = case::InliningTodo + push!(todo, idx=>gc_preserve === nothing ? case : + GCPreserveInliningTodo(gc_preserve, case)) end return nothing end @@ -1237,6 +1295,7 @@ end # functions. function process_simple!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, flag::UInt32, state::InliningState) + gc_preserve = nothing inst = ir[SSAValue(idx)] stmt = inst[:stmt] if !(stmt isa Expr) @@ -1262,6 +1321,22 @@ function process_simple!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, flag sig = call_sig(ir, stmt) sig === nothing && return nothing + # Handle GCPreserveDuring + if isa(inst[:info], GCPreserveDuringCallInfo) + # See if we can strip the construction of this + wrapped_f = stmt.args[1] + if isa(wrapped_f, SSAValue) + wrapping_stmt = ir[wrapped_f][:stmt] + if isexpr(wrapping_stmt, :new) + ft = argextype(wrapping_stmt.args[2], ir) + gc_preserve = GCPreserveRewrite(wrapping_stmt.args[2], wrapping_stmt.args[3]) + new_argtypes = copy(sig.argtypes) + new_argtypes[1] = ft + sig = Signature(singleton_type(ft), ft, new_argtypes) + end + end + end + # Handle _apply_iterate sig = inline_apply!(todo, ir, idx, stmt, sig, state) sig === nothing && return nothing @@ -1303,7 +1378,7 @@ function process_simple!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, flag return nothing end - return stmt, sig + return stmt, sig, gc_preserve end function handle_any_const_result!(cases::Vector{InliningCase}, @@ -1409,13 +1484,13 @@ end function handle_call!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, @nospecialize(info::CallInfo), flag::UInt32, sig::Signature, - state::InliningState) + state::InliningState, gc_preserve::Union{GCPreserveRewrite, Nothing}) cases = compute_inlining_cases(info, flag, sig, state) cases === nothing && return nothing cases, handled_all_cases, fully_covered, joint_effects = cases atype = argtypes_to_type(sig.argtypes) atype === Union{} && return nothing # accidentally actually unreachable - handle_cases!(todo, ir, idx, stmt, atype, cases, handled_all_cases, fully_covered, joint_effects) + handle_cases!(todo, ir, idx, stmt, atype, cases, handled_all_cases, fully_covered, joint_effects, gc_preserve) end function handle_match!(cases::Vector{InliningCase}, @@ -1499,18 +1574,18 @@ end function handle_cases!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr, @nospecialize(atype), cases::Vector{InliningCase}, handled_all_cases::Bool, fully_covered::Bool, - joint_effects::Effects) + joint_effects::Effects, gc_preserve::Union{GCPreserveRewrite, Nothing}) # If we only have one case and that case is fully covered, we may either # be able to do the inlining now (for constant cases), or push it directly # onto the todo list if fully_covered && handled_all_cases && length(cases) == 1 - handle_single_case!(todo, ir, idx, stmt, cases[1].item) + handle_single_case!(todo, ir, idx, stmt, cases[1].item, gc_preserve) elseif length(cases) > 0 || handled_all_cases isa(atype, DataType) || return nothing for case in cases isa(case.sig, DataType) || return nothing end - push!(todo, idx=>UnionSplit(handled_all_cases, fully_covered, atype, cases)) + push!(todo, idx=>UnionSplit(handled_all_cases, fully_covered, atype, cases, gc_preserve)) else add_flag!(ir[SSAValue(idx)], flags_for_effects(joint_effects)) end @@ -1629,13 +1704,17 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState) simpleres = process_simple!(todo, ir, idx, flag, state) simpleres === nothing && continue - stmt, sig = simpleres + stmt, sig, gc_preserve = simpleres info = ir.stmts[idx][:info] + if gc_preserve !== nothing && isa(info, GCPreserveDuringCallInfo) + info = info.info + end # `NativeInterpreter` won't need this, but provide a support for `:invoke` exprs here # for external `AbstractInterpreter`s that may run the inlining pass multiple times if isexpr(stmt, :invoke) + gc_preserve === nothing || continue handle_invoke_expr!(todo, ir, idx, stmt, info, flag, sig, state) continue end @@ -1652,16 +1731,20 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState) # handle special cased builtins if isa(info, OpaqueClosureCallInfo) + gc_preserve === nothing || continue handle_opaque_closure_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, ModifyOpInfo) + gc_preserve === nothing || continue handle_modifyop!_call!(ir, idx, stmt, info, state) elseif sig.f === Core.invoke + gc_preserve === nothing || continue handle_invoke_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, FinalizerInfo) + gc_preserve === nothing || continue handle_finalizer_call!(ir, idx, stmt, info, state) else # cascade to the generic (and extendable) handler - handle_call!(todo, ir, idx, stmt, info, flag, sig, state) + handle_call!(todo, ir, idx, stmt, info, flag, sig, state, gc_preserve) end end diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index d6a63d8f71abf..153257d4f244f 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -502,4 +502,15 @@ function add_edges_impl(edges::Vector{Any}, info::GlobalAccessInfo) push!(edges, info.b) end +""" + info::GCPreserveCallInfo <: CallInfo + +Wraps another CallInfo, indicating that this call came from a looked-through GCPreserveDuring. +""" +struct GCPreserveDuringCallInfo <: CallInfo + info::CallInfo +end +add_edges_impl(edges::Vector{Any}, info::GCPreserveDuringCallInfo) = + add_edges_impl(edges, info.info) + @specialize diff --git a/base/Base.jl b/base/Base.jl index 9d510b5c5d47c..b5de6dfa5d894 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -370,6 +370,11 @@ function start_profile_listener() ccall(:jl_set_peek_cond, Cvoid, (Ptr{Cvoid},), cond.handle) end +function print_padded_time(io, mod, maxlen, t) + print(io, rpad(string(mod) * " ", maxlen + 3, "─")) + Base.time_print(io, t * 10^9); println(io) +end + function __init__() # Base library init global _atexit_hooks_finished = false diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index ef448a02a15e9..3cdf1611c39c5 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -136,7 +136,8 @@ convert(::Type{Any}, Core.@nospecialize x) = x convert(::Type{T}, x::T) where {T} = x include("coreio.jl") -import Core: @doc, @__doc__, WrappedException, @int128_str, @uint128_str, @big_str, @cmd +import Core: @doc, @__doc__, WrappedException, @int128_str, @uint128_str, @big_str, @cmd, + GCPreserveDuring # Export list include("exports.jl") diff --git a/base/boot.jl b/base/boot.jl index d055c47516f91..f934b5da58801 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -1141,4 +1141,23 @@ typename(union::UnionAll) = typename(union.body) include(Core, "optimized_generics.jl") +struct GCPreserveDuring + f::Any + # N.B: This field is opaque - the compiler is allowed to arbitrarily change it + # as long as it has the same GC rooting behavior. + root::Any + GCPreserveDuring(@nospecialize(f), @nospecialize(root)) = new(f, root) +end + +# This has special support in inference and codegen and is only ever actually called +# in fallback cases. +function (this::GCPreserveDuring)(args...) + @noinline + r = this.f(args...) + # N.B.: This is correct, but stronger than required. If the call to `f` is deleted, + # this may be deleted as well. + donotdelete(this.root) + return r +end + ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Core, true) diff --git a/base/gcutils.jl b/base/gcutils.jl index d5e6f4597739f..361d5e2895ca9 100644 --- a/base/gcutils.jl +++ b/base/gcutils.jl @@ -186,8 +186,8 @@ end """ GC.@preserve x1 x2 ... xn expr -Mark the objects `x1, x2, ...` as being *in use* during the evaluation of the -expression `expr`. This is only required in unsafe code where `expr` +Mark the objects `x1, x2, ...` as being *in use* during the evaluation of all +calls lexically in the expression `expr`. This is only required in unsafe code where `expr` *implicitly uses* memory or other resources owned by one of the `x`s. *Implicit use* of `x` covers any indirect use of resources logically owned by @@ -236,7 +236,12 @@ macro preserve(args...) for x in syms isa(x, Symbol) || error("Preserved variable must be a symbol") end - esc(Expr(:gc_preserve, args[end], syms...)) + sym = length(syms) == 1 ? only(syms) : Expr(:tuple, syms...) + g = gensym() + esc(quote + $g = $sym + $(Expr(:gc_preserve, args[end], g)) + end) end """ diff --git a/base/sysimg.jl b/base/sysimg.jl index fd71544c205cc..82a34fc2eec20 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -102,29 +102,23 @@ let maxlen = maximum(textwidth.(string.(stdlibs)); init=0) tot_time_stdlib = 0.0 - # use a temp module to avoid leaving the type of this closure in Main push!(empty!(LOAD_PATH), "@stdlib") - m = Core.Module() - GC.@preserve m begin - print_time = @eval m (mod, t) -> (print(rpad(string(mod) * " ", $maxlen + 3, "─")); - Base.time_print(stdout, t * 10^9); println()) - print_time(Base, (Base.end_base_include - Base.start_base_include) * 10^(-9)) - - Base._track_dependencies[] = true - tot_time_stdlib = @elapsed for stdlib in stdlibs - tt = @elapsed Base.require(Base, stdlib) - print_time(stdlib, tt) - end - for dep in Base._require_dependencies - mod, path, fsize, mtime = dep[1], dep[2], dep[3], dep[5] - (fsize == 0 || mtime == 0.0) && continue - push!(Base._included_files, (mod, path)) - end - empty!(Base._require_dependencies) - Base._track_dependencies[] = false - - print_time("Stdlibs total", tot_time_stdlib) + Base.print_padded_time(stdout, Base, maxlen, (Base.end_base_include - Base.start_base_include) * 10^(-9)) + + Base._track_dependencies[] = true + tot_time_stdlib = @elapsed for stdlib in stdlibs + tt = @elapsed Base.require(Base, stdlib) + Base.print_padded_time(stdout, stdlib, maxlen, tt) + end + for dep in Base._require_dependencies + mod, path, fsize, mtime = dep[1], dep[2], dep[3], dep[5] + (fsize == 0 || mtime == 0.0) && continue + push!(Base._included_files, (mod, path)) end + empty!(Base._require_dependencies) + Base._track_dependencies[] = false + + Base.print_padded_time(stdout, "Stdlibs total", maxlen, tot_time_stdlib) # Clear global state empty!(Core.ARGS) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 97a58f8aa419c..0be3b320cac5d 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -3807,6 +3807,10 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotab if (jt == jl_bottom_type || jt == NULL) // We have an undef value on a (hopefully) dead branch return UndefValue::get(ctx.types().T_prjlvalue); + Value *box; + if (vinfo.wrapped_typ) { + assert(false && "TODO"); + } if (vinfo.constant) return track_pjlvalue(ctx, literal_pointer_val(ctx, vinfo.constant)); // This can happen in early bootstrap for `gc_preserve_begin` return value. @@ -3818,7 +3822,6 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotab return vinfo.V; } - Value *box; if (vinfo.TIndex) { SmallBitVector skip_none; box = box_union(ctx, vinfo, skip_none); diff --git a/src/codegen.cpp b/src/codegen.cpp index 2d9c94baee9b4..14f24aa4207a4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1719,6 +1719,11 @@ struct jl_cgval_t { // handling to account for the possibility that this may be NULL. Value *Vboxed; + // If non-null, this value was an unwrapped GCPreserveDuring, where this value is the preservee. + // Calls of this value need toapply appropriate gc preserve handling. + SmallVector preservees; + jl_value_t *wrapped_typ; + Value *TIndex; // if `V` is an unboxed (tagged) Union described by `typ`, this gives the DataType index (1-based, small int) as an i8 SmallVector inline_roots; // if present, `V` is a pointer, but not in canonical layout jl_value_t *constant; // constant value (rooted in linfo.def.roots) @@ -4914,7 +4919,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // Returns ctx.types().T_prjlvalue static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, - ArrayRef argv, size_t nargs, JuliaFunction<> *trampoline) + ArrayRef argv, size_t nargs, ArrayRef roots, JuliaFunction<> *trampoline) { ++EmittedJLCalls; Function *TheTrampoline = prepare_call(trampoline); @@ -4935,7 +4940,9 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, } theArgs.push_back(arg); } - CallInst *result = ctx.builder.CreateCall(TheTrampoline, theArgs); + + OperandBundleDef OpBundle("jl_roots", roots); + CallInst *result = ctx.builder.CreateCall(TheTrampoline, theArgs, ArrayRef(&OpBundle, roots.empty() ? 0 : 1)); result->setAttributes(TheTrampoline->getAttributes()); // TODO: we could add readonly attributes in many cases to the args return result; @@ -4948,7 +4955,7 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value return emit_jlcall(ctx, prepare_call(theFptr), theF, argv, nargs, trampoline); } -static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, jl_returninfo_t &returninfo, ArrayRef argv, size_t nargs) +static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_closure, jl_value_t *specTypes, jl_value_t *jlretty, jl_returninfo_t &returninfo, ArrayRef argv, size_t nargs, ArrayRef roots) { ++EmittedSpecfunCalls; // emit specialized call site @@ -5047,7 +5054,9 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos idx++; } assert(idx == nfargs); - CallInst *call = ctx.builder.CreateCall(returninfo.decl, argvals); + + OperandBundleDef OpBundle("jl_roots", roots); + CallInst *call = ctx.builder.CreateCall(returninfo.decl, argvals, ArrayRef(&OpBundle, roots.empty() ? 0 : 1)); call->setAttributes(returninfo.attrs); if (gcstack_arg && ctx.emission_context.use_swiftcc) call->setCallingConv(CallingConv::Swift); @@ -5140,7 +5149,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_code_instance_t } static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty, StringRef specFunctionObject, jl_code_instance_t *fromexternal, - ArrayRef argv, size_t nargs, jl_value_t *inferred_retty) + ArrayRef argv, size_t nargs, ArrayRef roots, jl_value_t *inferred_retty) { Value *theFptr; if (fromexternal) { @@ -5160,7 +5169,7 @@ static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty theFptr = jl_Module->getOrInsertFunction(specFunctionObject, ctx.types().T_jlfunc).getCallee(); addRetAttr(cast(theFptr), Attribute::NonNull); } - Value *ret = emit_jlcall(ctx, theFptr, nullptr, argv, nargs, julia_call); + Value *ret = emit_jlcall(ctx, theFptr, nullptr, argv, nargs, roots, julia_call); return update_julia_type(ctx, mark_julia_type(ctx, ret, true, jlretty), inferred_retty); } @@ -5181,7 +5190,8 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) return emit_invoke(ctx, lival, argv, nargs, rt, false); } -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, bool always_inline) +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, + ArrayRef roots, bool always_inline) { ++EmittedInvokes; bool handled = false; @@ -5205,18 +5215,18 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR Function *f = ctx.f; FunctionType *ft = f->getFunctionType(); if (ft == ctx.types().T_jlfunc) { - Value *ret = emit_jlcall(ctx, f, nullptr, argv, nargs, julia_call); + Value *ret = emit_jlcall(ctx, f, nullptr, argv, nargs, roots, julia_call); result = update_julia_type(ctx, mark_julia_type(ctx, ret, true, ctx.rettype), rt); } else if (ft == ctx.types().T_jlfuncparams) { - Value *ret = emit_jlcall(ctx, f, ctx.spvals_ptr, argv, nargs, julia_call2); + Value *ret = emit_jlcall(ctx, f, ctx.spvals_ptr, argv, nargs, roots, julia_call2); result = update_julia_type(ctx, mark_julia_type(ctx, ret, true, ctx.rettype), rt); } else { unsigned return_roots = 0; jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; StringRef protoname = f->getName(); - result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, &cc, &return_roots, rt); + result = emit_call_specfun_other(ctx, mi, ctx.rettype, protoname, nullptr, argv, nargs, roots, &cc, &return_roots, rt); } handled = true; } @@ -5232,7 +5242,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR bool specsig, needsparams; std::tie(specsig, needsparams) = uses_specsig(get_ci_abi(codeinst), mi, codeinst->rettype, ctx.params->prefer_specsig); if (needsparams) { - Value *r = emit_jlcall(ctx, jlinvoke_func, track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)mi)), argv, nargs, julia_call2); + Value *r = emit_jlcall(ctx, jlinvoke_func, track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)mi)), argv, nargs, roots, julia_call2); result = mark_julia_type(ctx, r, true, rt); } else { @@ -5288,9 +5298,9 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; unsigned return_roots = 0; if (specsig) - result = emit_call_specfun_other(ctx, codeinst, protoname, external ? codeinst : nullptr, argv, nargs, &cc, &return_roots, rt); + result = emit_call_specfun_other(ctx, codeinst, protoname, external ? codeinst : nullptr, argv, nargs, roots, &cc, &return_roots, rt); else - result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, rt); + result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, roots, rt); if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, nullptr, specsig, !always_inline, always_inline}; @@ -5302,7 +5312,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR } } if (!handled) { - Value *r = emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, julia_call2); + Value *r = emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, roots, julia_call2); result = mark_julia_type(ctx, r, true, rt); } if (result.typ == jl_bottom_type) { @@ -5329,7 +5339,13 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ return jl_cgval_t(); } const jl_cgval_t &f = argv[0]; - if (f.constant) { + if (f.typ == jl_gc_preserve_during_type) { + f.typ = f.wrapped_typ; + SmallVector roots = f.preservees; + f.preservees.clear(); + return emit_invoke(ctx, lival, argv, nargs, roots, rt, false); + } + else if (f.constant) { jl_cgval_t ret; auto it = builtin_func_map().end(); if (f.constant == BUILTIN(modifyfield)) { @@ -6469,11 +6485,20 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ if (jl_is_type_type(ty) && jl_is_datatype(jl_tparam0(ty)) && jl_is_concrete_type(jl_tparam0(ty))) { - assert(nargs <= jl_datatype_nfields(jl_tparam0(ty)) + 1); - jl_cgval_t res = emit_new_struct(ctx, jl_tparam0(ty), nargs - 1, ArrayRef(argv).drop_front(), is_promotable); - if (is_promotable && res.promotion_point && res.promotion_ssa==-1) - res.promotion_ssa = ssaidx_0based; - return res; + jl_value_t *allocated_type = jl_tparam0(ty); + assert(nargs <= jl_datatype_nfields(allocated_type) + 1); + if (allocated_type == jl_gc_preserve_during_type) { + jl_cgval_t res = argv[1]; + res.preservees.append(get_gc_roots_for(ctx, argv[2])); + res.wrapped_typ = res.typ; + res.typ = allocated_type; + return res; + } else { + jl_cgval_t res = emit_new_struct(ctx, allocated_type, nargs - 1, ArrayRef(argv).drop_front(), is_promotable); + if (is_promotable && res.promotion_point && res.promotion_ssa==-1) + res.promotion_ssa = ssaidx_0based; + return res; + } } Value *val = emit_jlcall(ctx, jlnew_func, nullptr, argv, nargs, julia_call); // temporarily mark as `Any`, expecting `emit_ssaval_assign` to update diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 76e8368132424..9f33ef1377c48 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -126,6 +126,7 @@ XX(jl_task_type) \ XX(jl_top_module) \ XX(jl_trimfailure_type) \ + XX(jl_gc_preserve_during_type) \ XX(jl_true) \ XX(jl_tuple_typename) \ XX(jl_tvar_type) \ diff --git a/src/jlfrontend.scm b/src/jlfrontend.scm index a678d481eab1f..a3ebb659a7d27 100644 --- a/src/jlfrontend.scm +++ b/src/jlfrontend.scm @@ -22,10 +22,10 @@ `(incomplete ,msg) (cons 'error (cdr e)))) (begin - ;;(newline) - ;;(display "unexpected error: ") - ;;(prn e) - ;;(print-stack-trace (stacktrace)) + (newline) + (display "unexpected error: ") + (prn e) + (print-stack-trace (stacktrace)) '(error "malformed expression")))) thk)) diff --git a/src/jltypes.c b/src/jltypes.c index b3434b2cbb95b..862d970f22aae 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3956,6 +3956,7 @@ void post_boot_hooks(void) jl_initerror_type = (jl_datatype_t*)core("InitError"); jl_missingcodeerror_type = (jl_datatype_t*)core("MissingCodeError"); jl_trimfailure_type = (jl_datatype_t*)core("TrimFailure"); + jl_gc_preserve_during_type = (jl_datatype_t*)core("GCPreserveDuring"); jl_pair_type = core("Pair"); jl_value_t *kwcall_func = core("kwcall"); diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 052e0000ebe87..d17f868a309d9 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2556,6 +2556,17 @@ (every check-path (cdar e))))) (error (string "malformed \"" what "\" statement")))) +(define (wrap-gc-preserve e x) + (if (pair? e) + (cond ((memq (car e) '(method lambda)) e) + ((eq? (car e) 'foreigncall) + `(,(car e) ,@(list-head (cdr e) 5) ,@(map (lambda (c) (wrap-gc-preserve c x)) (list-tail e 6)))) + ((eq? (car e) 'cfunction) e) + ((eq? (car e) 'call) + `(call (new (top GCPreserveDuring) ,(cadr e) ,x) ,@(map (lambda (c) (wrap-gc-preserve c x)) (cddr e)))) + (else `(,(car e) ,@(map (lambda (c) (wrap-gc-preserve c x)) (cdr e))))) + e)) + ;; table mapping expression head to a function expanding that form (define expand-table (table @@ -2945,13 +2956,7 @@ 'gc_preserve (lambda (e) - (let* ((s (make-ssavalue)) - (r (make-ssavalue))) - `(block - (= ,s (gc_preserve_begin ,@(cddr e))) - (= ,r ,(expand-forms (cadr e))) - (gc_preserve_end ,s) - ,r))) + (wrap-gc-preserve (expand-forms (cadr e)) (caddr e))) 'line (lambda (e) diff --git a/src/julia.h b/src/julia.h index a5564670479db..2dc985cb7861a 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1036,6 +1036,7 @@ extern JL_DLLIMPORT jl_datatype_t *jl_fielderror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_atomicerror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_missingcodeerror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_trimfailure_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_datatype_t *jl_gc_preserve_during_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_lineinfonode_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_abioverride_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_stackovf_exception JL_GLOBALLY_ROOTED; diff --git a/src/staticdata.c b/src/staticdata.c index be1188572f3b0..fa8d7937e22aa 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -116,7 +116,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 151 +#define NUM_TAGS 152 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -282,6 +282,7 @@ static void get_tags(jl_value_t **tags[NUM_TAGS]) INSERT_TAG(jl_opaque_closure_method); INSERT_TAG(jl_nulldebuginfo); INSERT_TAG(jl_method_table); + INSERT_TAG(jl_gc_preserve_during_type); // n.b. must update NUM_TAGS when you add something here #undef INSERT_TAG assert(i == NUM_TAGS - 1);