compiler: consolidate staticdata.jl and invalidations.jl into Compiler proper [NFCI] (#59238)

This aims to consolidate more of the code that is expected to run in
typeinf_world into Compiler proper. The intent is that these are likely
to gain more interaction (both directions) with inference, as the
compiler should use this mini-compiler and vice versa.

We actually might want a `Base.CompilerUtils` module so that the
`Compiler` package itself is smaller and less reliant on internal APIs,
but for now just move these into a submodule of Compiler.

Written by Claude

Author: vtjnash

Compiler/src/Compiler.jl

@@ -48,8 +48,8 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali
     PARTITION_KIND_GLOBAL, PARTITION_KIND_UNDEF_CONST, PARTITION_KIND_BACKDATED_CONST, PARTITION_KIND_DECLARED,
     PARTITION_FLAG_DEPWARN,
     Base, BitVector, Bottom, Callable, DataTypeFieldDesc,
-    EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES,
-    OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME,
+    EffectsOverride, Filter, Generator, NUM_EFFECTS_OVERRIDES,
+    OneTo, Ordering, RefValue, _NAMEDTUPLE_NAME,
     _array_for, _bits_findnext, _methods_by_ftype, _uniontypes, all, allocatedinline, any,
     argument_datatype, binding_kind, cconvert, copy_exprargs, datatype_arrayelem,
     datatype_fieldcount, datatype_fieldtypes, datatype_layoutsize, datatype_nfields,
@@ -64,9 +64,11 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali
     partition_restriction, quoted, rename_unionall, rewrap_unionall, specialize_method,
     structdiff, tls_world_age, unconstrain_vararg_length, unionlen, uniontype_layout,
     uniontypes, unsafe_convert, unwrap_unionall, unwrapva, vect, widen_diagonal,
-    _uncompressed_ir, maybe_add_binding_backedge!, datatype_min_ninitialized,
+    _uncompressed_ir, datatype_min_ninitialized,
     partialstruct_init_undefs, fieldcount_noerror, _eval_import, _eval_using,
-    get_ci_mi
+    get_ci_mi, get_methodtable, morespecific, specializations, has_image_globalref,
+    PARTITION_MASK_KIND, PARTITION_KIND_GUARD, PARTITION_FLAG_EXPORTED, PARTITION_FLAG_DEPRECATED,
+    BINDING_FLAG_ANY_IMPLICIT_EDGES, is_some_implicit, IteratorSize, SizeUnknown, get_require_world, JLOptions
 
 using Base.Order
 
@@ -188,6 +190,10 @@ include("bootstrap.jl")
 include("reflection_interface.jl")
 include("opaque_closure.jl")
 
+baremodule ReinferUtils end
+include(ReinferUtils, "reinfer.jl")
+include(ReinferUtils, "bindinginvalidations.jl")
+
 macro __SOURCE_FILE__()
     __source__.file === nothing && return nothing
     return QuoteNode(__source__.file::Symbol)

base/invalidation.jl renamed to Compiler/src/bindinginvalidations.jl

@@ -1,26 +1,17 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-struct GlobalRefIterator
-    mod::Module
-end
-IteratorSize(::Type{GlobalRefIterator}) = SizeUnknown()
-globalrefs(mod::Module) = GlobalRefIterator(mod)
-
-function iterate(gri::GlobalRefIterator, i = 1)
-    m = gri.mod
-    table = ccall(:jl_module_get_bindings, Ref{SimpleVector}, (Any,), m)
-    i > length(table) && return nothing
-    b = table[i]
-    b === nothing && return iterate(gri, i+1)
-    return ((b::Core.Binding).globalref, i+1)
-end
+using ..Compiler: _uncompressed_ir, specializations, get_ci_mi, convert, unsafe_load, cglobal, generating_output, has_image_globalref,
+    PARTITION_MASK_KIND, PARTITION_KIND_GUARD, PARTITION_FLAG_EXPORTED, PARTITION_FLAG_DEPRECATED,
+    BINDING_FLAG_ANY_IMPLICIT_EDGES, binding_kind, partition_restriction, is_some_imported,
+    is_some_binding_imported, is_some_implicit, SizeUnknown, maybe_add_binding_backedge!, walk_binding_partition, abstract_eval_partition_load, userefs
+using .Core: SimpleVector, CodeInfo
 
 function foreachgr(visit, src::CodeInfo)
     stmts = src.code
     for i = 1:length(stmts)
         stmt = stmts[i]
         isa(stmt, GlobalRef) && visit(stmt)
-        for ur in Compiler.userefs(stmt)
+        for ur in userefs(stmt)
             arg = ur[]
             isa(arg, GlobalRef) && visit(arg)
         end
@@ -35,7 +26,7 @@ function anygr(visit, src::CodeInfo)
             visit(stmt) && return true
             continue
         end
-        for ur in Compiler.userefs(stmt)
+        for ur in userefs(stmt)
             arg = ur[]
             isa(arg, GlobalRef) && visit(arg) && return true
         end
@@ -69,7 +60,7 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid
         src = _uncompressed_ir(method)
         invalidate_all = should_invalidate_code_for_globalref(gr, src)
     end
-    if invalidate_all && !Base.generating_output()
+    if invalidate_all && !generating_output()
         @atomic method.did_scan_source |= 0x4
     end
     invalidated_any = false
@@ -99,15 +90,15 @@ export_affecting_partition_flags(bpart::Core.BindingPartition) =
 function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt)
     gr = b.globalref
 
-    (_, (ib, ibpart)) = Compiler.walk_binding_partition(b, invalidated_bpart, new_max_world)
-    (_, (nb, nbpart)) = Compiler.walk_binding_partition(b, new_bpart, new_max_world+1)
+    (_, (ib, ibpart)) = walk_binding_partition(b, invalidated_bpart, new_max_world)
+    (_, (nb, nbpart)) = walk_binding_partition(b, new_bpart, new_max_world+1)
 
     # `abstract_eval_partition_load` is the maximum amount of information that inference
     # reads from a binding partition. If this information does not change - we do not need to
     # invalidate any code that inference created, because we know that the result will not change.
     need_to_invalidate_code =
-        Compiler.abstract_eval_partition_load(nothing, ib, ibpart) !==
-        Compiler.abstract_eval_partition_load(nothing, nb, nbpart)
+        abstract_eval_partition_load(nothing, ib, ibpart) !==
+        abstract_eval_partition_load(nothing, nb, nbpart)
 
     need_to_invalidate_export = export_affecting_partition_flags(invalidated_bpart) !==
                                 export_affecting_partition_flags(new_bpart)
@@ -173,12 +164,6 @@ end
 invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) =
     invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world)
 
-function maybe_add_binding_backedge!(b::Core.Binding, edge::Union{Method, CodeInstance})
-    meth = isa(edge, Method) ? edge : get_ci_mi(edge).def
-    ccall(:jl_maybe_add_binding_backedge, Cint, (Any, Any, Any), b, edge, meth)
-    return nothing
-end
-
 function binding_was_invalidated(b::Core.Binding)
     # At least one partition is required for invalidation
     !isdefined(b, :partitions) && return false
@@ -206,7 +191,7 @@ function scan_new_method!(method::Method, image_backedges_only::Bool)
 end
 
 function scan_new_methods!(extext_methods::Vector{Any}, internal_methods::Vector{Any}, image_backedges_only::Bool)
-    if image_backedges_only && Base.generating_output(true)
+    if image_backedges_only && generating_output(true)
         # Replacing image bindings is forbidden during incremental precompilation - skip backedge insertion
         return
     end

base/staticdata.jl renamed to Compiler/src/reinfer.jl

@@ -1,10 +1,13 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
+using ..Compiler.Base
+using ..Compiler: _findsup, store_backedges, JLOptions, get_world_counter,
+    _methods_by_ftype, get_methodtable, get_ci_mi, should_instrument,
+    morespecific, RefValue, get_require_world, Vector, IdDict
 using .Core: CodeInstance, MethodInstance
-using .Base: JLOptions, Compiler, get_world_counter, _methods_by_ftype, get_methodtable, get_ci_mi, morespecific
 
 const WORLD_AGE_REVALIDATION_SENTINEL::UInt = 1
-const _jl_debug_method_invalidation = Ref{Union{Nothing,Vector{Any}}}(nothing)
+const _jl_debug_method_invalidation = RefValue{Union{Nothing,Vector{Any}}}(nothing)
 debug_method_invalidation(onoff::Bool) =
     _jl_debug_method_invalidation[] = onoff ? Any[] : nothing
 
@@ -70,7 +73,7 @@ function insert_backedges(edges::Vector{Any}, ext_ci_list::Union{Nothing,Vector{
     # determine which CodeInstance objects are still valid in our image
     # to enable any applicable new codes
     backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt)
-    Base.scan_new_methods!(extext_methods, internal_methods, backedges_only)
+    scan_new_methods!(extext_methods, internal_methods, backedges_only)
     workspace = VerifyMethodWorkspace()
     _insert_backedges(edges, workspace)
     if ext_ci_list !== nothing
@@ -135,23 +138,23 @@ function needs_instrumentation(codeinst::CodeInstance, mi::MethodInstance, def::
     if JLOptions().code_coverage != 0 || JLOptions().malloc_log != 0
         # test if the code needs to run with instrumentation, in which case we cannot use existing generated code
         if isdefined(def, :debuginfo) ? # generated_only functions do not have debuginfo, so fall back to considering their codeinst debuginfo though this may be slower and less reliable
-            Compiler.should_instrument(def.module, def.debuginfo) :
-            isdefined(codeinst, :debuginfo) && Compiler.should_instrument(def.module, codeinst.debuginfo)
+            should_instrument(def.module, def.debuginfo) :
+            isdefined(codeinst, :debuginfo) && should_instrument(def.module, codeinst.debuginfo)
             return true
         end
         gensig = gen_staged_sig(def, mi)
         if gensig !== nothing
             # if this is defined by a generator, try to consider forcing re-running the generators too, to add coverage for them
-            minworld = Ref{UInt}(1)
-            maxworld = Ref{UInt}(typemax(UInt))
-            has_ambig = Ref{Int32}(0)
+            minworld = RefValue{UInt}(1)
+            maxworld = RefValue{UInt}(typemax(UInt))
+            has_ambig = RefValue{Int32}(0)
             result = _methods_by_ftype(gensig, nothing, -1, validation_world, #=ambig=#false, minworld, maxworld, has_ambig)
             if result !== nothing
                 for k = 1:length(result)
                     match = result[k]::Core.MethodMatch
                     genmethod = match.method
                     # no, I refuse to refuse to recurse into your cursed generated function generators and will only test one level deep here
-                    if isdefined(genmethod, :debuginfo) && Compiler.should_instrument(genmethod.module, genmethod.debuginfo)
+                    if isdefined(genmethod, :debuginfo) && should_instrument(genmethod.module, genmethod.debuginfo)
                         return true
                     end
                 end
@@ -194,7 +197,7 @@ function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace
                 continue
             end
 
-            minworld, maxworld = Base.get_require_world(), validation_world
+            minworld, maxworld = get_require_world(), validation_world
 
             if haskey(workspace.visiting, initial.codeinst)
                 workspace.result_states[current_depth] = VerifyMethodResultState(workspace.visiting[initial.codeinst], minworld, maxworld)
@@ -209,7 +212,7 @@ function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace
             # Check for invalidation of GlobalRef edges
             if (initial.def.did_scan_source & 0x1) == 0x0
                 backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt)
-                Base.scan_new_method!(initial.def, backedges_only)
+                scan_new_method!(initial.def, backedges_only)
             end
             if (initial.def.did_scan_source & 0x4) != 0x0
                 maxworld = 0
@@ -220,7 +223,7 @@ function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace
             end
 
             # Process all non-CodeInstance edges
-            if !isempty(initial.callees) && maxworld != Base.get_require_world()
+            if !isempty(initial.callees) && maxworld != get_require_world()
                 matches = []
                 j = 1
                 while j <= length(initial.callees)
@@ -248,7 +251,7 @@ function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace
                         j += 1
                         min_valid2 = minworld
                         max_valid2 = maxworld
-                        if !Base.binding_was_invalidated(edge)
+                        if !binding_was_invalidated(edge)
                             if isdefined(edge, :partitions)
                                 min_valid2 = edge.partitions.min_world
                                 max_valid2 = edge.partitions.max_world
@@ -333,7 +336,7 @@ function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace
                     end
                     @atomic :monotonic child.max_world = result.result_maxworld
                     if result.result_maxworld == validation_world && validation_world == get_world_counter()
-                        Compiler.store_backedges(child, child.edges)
+                        store_backedges(child, child.edges)
                     end
                     @assert workspace.visiting[child] == length(workspace.stack) + 1
                     delete!(workspace.visiting, child)
@@ -559,9 +562,9 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n
    end
     # next, compare the current result of ml_matches to the old result
     lim = _jl_debug_method_invalidation[] !== nothing ? Int(typemax(Int32)) : n
-    minworld = Ref{UInt}(1)
-    maxworld = Ref{UInt}(typemax(UInt))
-    has_ambig = Ref{Int32}(0)
+    minworld = RefValue{UInt}(1)
+    maxworld = RefValue{UInt}(typemax(UInt))
+    has_ambig = RefValue{Int32}(0)
     result = _methods_by_ftype(sig, nothing, lim, world, #=ambig=#false, minworld, maxworld, has_ambig)
     if result === nothing
         empty!(matches)
@@ -626,7 +629,7 @@ function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UIn
             minworld = 1
             maxworld = 0
         else
-            matched, valid_worlds = Compiler._findsup(invokesig, mt, world)
+            matched, valid_worlds = _findsup(invokesig, mt, world)
             minworld, maxworld = valid_worlds.min_world, valid_worlds.max_world
             if matched === nothing
                 maxworld = 0
@@ -641,3 +644,9 @@ function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UIn
     end
     return minworld, maxworld
 end
+
+# Wrapper to call insert_backedges in typeinf_world for external calls
+function insert_backedges_typeinf(edges::Vector{Any}, ext_ci_list::Union{Nothing,Vector{Any}}, extext_methods::Vector{Any}, internal_methods::Vector{Any})
+    args = Any[insert_backedges, edges, ext_ci_list, extext_methods, internal_methods]
+    return ccall(:jl_call_in_typeinf_world, Any, (Ptr{Any}, Cint), args, length(args))
+end

Compiler/src/typeinfer.jl

@@ -741,6 +741,13 @@ function Base.iterate(it::ForwardToBackedgeIterator, i::Int = 1)
 end
 
 # record the backedges
+
+function maybe_add_binding_backedge!(b::Core.Binding, edge::Union{Method, CodeInstance})
+    meth = isa(edge, Method) ? edge : get_ci_mi(edge).def
+    ccall(:jl_maybe_add_binding_backedge, Cint, (Any, Any, Any), b, edge, meth)
+    return nothing
+end
+
 function store_backedges(caller::CodeInstance, edges::SimpleVector)
     isa(get_ci_mi(caller).def, Method) || return # don't add backedges to toplevel method instance
 

base/Base.jl

@@ -267,9 +267,6 @@ include("uuid.jl")
 include("pkgid.jl")
 include("toml_parser.jl")
 include("linking.jl")
-module StaticData
-include("staticdata.jl")
-end
 include("loading.jl")
 
 # BinaryPlatforms, used by Artifacts.  Needs `Sort`.

base/Base_compiler.jl

@@ -348,7 +348,6 @@ using .Order
 
 include("coreir.jl")
 include("module.jl")
-include("invalidation.jl")
 
 BUILDROOT::String = ""
 DATAROOT::String = ""
@@ -377,6 +376,7 @@ process_sysimg_args!()
 function isready end
 
 include(strcat(DATAROOT, "julia/Compiler/src/Compiler.jl"))
+using .Compiler.ReinferUtils: ReinferUtils, invalidate_code_for_globalref!
 
 const _return_type = Compiler.return_type
 

base/c.jl

@@ -226,16 +226,16 @@ function expand_ccallable(name, rt, def)
             else
                 f = :(typeof($f))
             end
-            at = map(sig.args[2:end]) do a
-                if isa(a,Expr) && a.head === :(::)
-                    a.args[end]
-                else
-                    :Any
-                end
-            end
+            at = Any[let a = sig.args[i]
+                    if isa(a,Expr) && a.head === :(::)
+                        a.args[end]
+                    else
+                        :Any
+                    end
+                end for i in 2:length(sig.args)]
             return quote
                 @__doc__ $(esc(def))
-                _ccallable($name, $(esc(rt)), $(Expr(:curly, :Tuple, esc(f), map(esc, at)...)))
+                _ccallable($name, $(esc(rt)), $(Expr(:curly, :Tuple, esc(f), map!(esc, at, at)...)))
             end
         end
     end
@@ -349,7 +349,7 @@ function ccall_macro_parse(exprs)
     end
     # add any varargs if necessary
     nreq = 0
-    if !isnothing(varargs)
+    if varargs !== nothing
         if length(args) == 0
             throw(ArgumentError("C ABI prohibits vararg without one required argument"))
         end
@@ -389,7 +389,7 @@ function ccall_macro_lower(convention, func, rettype, types, args, gc_safe, nreq
 
     return Expr(:block, statements...,
                 Expr(:call, :ccall, func, cconv, esc(rettype),
-                     Expr(:tuple, map(esc, types)...), map(esc, args)...))
+                     Expr(:tuple, map!(esc, types, types)...), map!(esc, args, args)...))
 end
 
 """

base/loading.jl

@@ -1288,7 +1288,7 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No
         extext_methods = sv[5]::Vector{Any}
         internal_methods = sv[6]::Vector{Any}
         Compiler.@zone "CC: INSERT_BACKEDGES" begin
-            StaticData.insert_backedges(edges, ext_edges, extext_methods, internal_methods)
+            ReinferUtils.insert_backedges_typeinf(edges, ext_edges, extext_methods, internal_methods)
         end
         restored = register_restored_modules(sv, pkg, path)
 

sysimage.mk

@@ -57,7 +57,6 @@ COMPILER_SRCS := $(addprefix $(JULIAHOME)/, \
 		base/int.jl \
 		base/indices.jl \
 		base/iterators.jl \
-		base/invalidation.jl \
 		base/module.jl \
 		base/namedtuple.jl \
 		base/ntuple.jl \

test/precompile.jl

@@ -1055,9 +1055,9 @@ precompile_test_harness("code caching") do dir
         const glbi = LogBindingInvalidation(2.0)
     end)
     @eval using $StaleC
-    invalidations = Base.StaticData.debug_method_invalidation(true)
+    invalidations = Base.ReinferUtils.debug_method_invalidation(true)
     @eval using $StaleB
-    Base.StaticData.debug_method_invalidation(false)
+    Base.ReinferUtils.debug_method_invalidation(false)
     invokelatest() do
         MB = getfield(@__MODULE__, StaleB)
         MC = getfield(@__MODULE__, StaleC)