Merge branch 'master' into teh/juliac_headerlogs

Author: timholy

AGENTS.md

@@ -27,10 +27,11 @@ If you made changes to the runtime (any files in `src/`), you will need to rebui
 julia. Run `make -j` to rebuild julia. This process may take up to 10 minutes
 depending on your changes.
 
-After `make` run these static analysis checks:
-  - `make -C src clang-sa-<filename>` (replace `<filename>` with the basename of the file you modified)
-  - `make -C src clang-sagc-<filename>` which may require adding JL_GC_PUSH arguments, or JL_GC_PROMISE_ROOTED statements., or require fixing locks. Remember arguments are assumed rooted, so check the callers to make sure that is handled. If the value is being temporarily moved around in a struct or arraylist, `JL_GC_PROMISE_ROOTED(struct->field)` may be needed as a statement (it return void) immediately after reloading the struct before any use of struct. Put the promise as early in the code as is legal.
-  - `make -C src clang-tidy-<filename>`
+After making changes, run static analysis checks:
+  - First run `make -C src install-analysis-deps` to initialize dependencies (only needed once the first time).
+  - Run `make -C src analyze-<filename> --output-sync -j8` (replace `<filename>` with the basename of any C or C++ file you modified, excluding headers).
+  - Tests can also be rerun individually with `clang-sa-<filename>`, `clang-sagc-<filename>` or `clang-tidy-<filename>`.
+  - If `clang-sagc-<filename>` fails, it may require adding `JL_GC_PUSH` statements, or `JL_GC_PROMISE_ROOTED` statements., or require fixing locks. Remember arguments are assumed rooted, so check the callers to make sure that is handled. If the value is being temporarily moved around in a struct or arraylist, `JL_GC_PROMISE_ROOTED(struct->field)` may be needed as a statement (it return void) immediately after reloading the struct before any use of struct. Put that promise as early in the code as is legal, near the definition not the use.
 
 ## Using Revise
 
@@ -97,7 +98,7 @@ make sure to take them into account.
 - Do not `ccall` runtime C functions directly if there are existing wrappers for the function.
 - Do not explicitly add a module prefix if the code you're adding is in the same module. E.g. do not use `Base.` for code in Base unless required.
 
-## Commit message formatting
+## Commit messages and pull requests
 
 When writing commit messages, follow the format "component: Brief summary" for
 the title. In the body of the commit message, provide a brief prose summary
@@ -109,7 +110,7 @@ If your change fixes one or more issues, use the syntax "Fixes #" at the end of
 When referencing external GitHub PRs or issues, use proper GitHub interlinking format (e.g., `owner/repo#123` for PRs/issues).
 When fixing CI failures, include the link to the specific CI failure in the commit message.
 
-When creating pull requests, if the pull request consists of one commit only,
-use the body of the commit for the body of the pull request. If there are multiple
-commits in the pull request, follow the same guidelines for the pull request
-as for the commit body.
+When creating pull requests:
+1. If the pull request consists of one commit only, use the body of the commit for the body of the pull request.
+2. If there are multiple commits in the pull request, follow the same guidelines for the pull request as for the commit body.
+3. Make sure that the base commit of the pull request is recent (within the past two days) - if not rebase your changes first.

Compiler/src/Compiler.jl

@@ -48,12 +48,12 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali
     PARTITION_KIND_GLOBAL, PARTITION_KIND_UNDEF_CONST, PARTITION_KIND_BACKDATED_CONST, PARTITION_KIND_DECLARED,
     PARTITION_FLAG_DEPWARN,
     Base, BitVector, Bottom, Callable, DataTypeFieldDesc,
-    EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES,
-    OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME,
+    EffectsOverride, Filter, Generator, NUM_EFFECTS_OVERRIDES,
+    OneTo, Ordering, RefValue, _NAMEDTUPLE_NAME,
     _array_for, _bits_findnext, _methods_by_ftype, _uniontypes, all, allocatedinline, any,
     argument_datatype, binding_kind, cconvert, copy_exprargs, datatype_arrayelem,
     datatype_fieldcount, datatype_fieldtypes, datatype_layoutsize, datatype_nfields,
-    datatype_pointerfree, decode_effects_override, diff_names, fieldindex,
+    datatype_pointerfree, decode_effects_override, diff_names, fieldindex, visit,
     generating_output, get_nospecializeinfer_sig, get_world_counter, has_free_typevars,
     hasgenerator, hasintersect, indexed_iterate, isType, is_file_tracked, is_function_def,
     is_meta_expr, is_meta_expr_head, is_nospecialized, is_nospecializeinfer, is_defined_const_binding,
@@ -64,9 +64,11 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali
     partition_restriction, quoted, rename_unionall, rewrap_unionall, specialize_method,
     structdiff, tls_world_age, unconstrain_vararg_length, unionlen, uniontype_layout,
     uniontypes, unsafe_convert, unwrap_unionall, unwrapva, vect, widen_diagonal,
-    _uncompressed_ir, maybe_add_binding_backedge!, datatype_min_ninitialized,
+    _uncompressed_ir, datatype_min_ninitialized,
     partialstruct_init_undefs, fieldcount_noerror, _eval_import, _eval_using,
-    get_ci_mi
+    get_ci_mi, get_methodtable, morespecific, specializations, has_image_globalref,
+    PARTITION_MASK_KIND, PARTITION_KIND_GUARD, PARTITION_FLAG_EXPORTED, PARTITION_FLAG_DEPRECATED,
+    BINDING_FLAG_ANY_IMPLICIT_EDGES, is_some_implicit, IteratorSize, SizeUnknown, get_require_world, JLOptions
 
 using Base.Order
 
@@ -185,9 +187,14 @@ include("typeinfer.jl")
 include("optimize.jl")
 
 include("bootstrap.jl")
+include("precompile.jl")
 include("reflection_interface.jl")
 include("opaque_closure.jl")
 
+baremodule ReinferUtils end
+include(ReinferUtils, "reinfer.jl")
+include(ReinferUtils, "bindinginvalidations.jl")
+
 macro __SOURCE_FILE__()
     __source__.file === nothing && return nothing
     return QuoteNode(__source__.file::Symbol)

Compiler/src/abstractinterpretation.jl

@@ -1245,7 +1245,7 @@ function const_prop_methodinstance_heuristic(interp::AbstractInterpreter,
         if isa(code, CodeInstance)
             inferred = @atomic :monotonic code.inferred
             # TODO propagate a specific `CallInfo` that conveys information about this call
-            if src_inlining_policy(interp, inferred, NoCallInfo(), IR_FLAG_NULL)
+            if src_inlining_policy(interp, mi, inferred, NoCallInfo(), IR_FLAG_NULL)
                 return true
             end
         end
@@ -3485,7 +3485,59 @@ function refine_partial_type(@nospecialize t)
     return t
 end
 
+abstract_eval_nonlinearized_foreigncall_name(interp::AbstractInterpreter, e, sstate::StatementState, sv::IRInterpretationState) = nothing
+
+function abstract_eval_nonlinearized_foreigncall_name(interp::AbstractInterpreter, e, sstate::StatementState, sv::AbsIntState)
+    if isexpr(e, :call)
+        n = length(e.args)
+        argtypes = Vector{Any}(undef, n)
+        callresult = Future{CallMeta}()
+        i::Int = 1
+        nextstate::UInt8 = 0x0
+        local ai, res
+        function evalargs(interp, sv)
+            if nextstate === 0x1
+                @goto state1
+            elseif nextstate === 0x2
+                @goto state2
+            end
+            while i <= n
+                ai = abstract_eval_nonlinearized_foreigncall_name(interp, e.args[i], sstate, sv)
+                if !isready(ai)
+                    nextstate = 0x1
+                    return false
+                    @label state1
+                end
+                argtypes[i] = ai[].rt
+                i += 1
+            end
+            res = abstract_call(interp, ArgInfo(e.args, argtypes), sstate, sv)
+            if !isready(res)
+                nextstate = 0x2
+                return false
+                @label state2
+            end
+            callresult[] = res[]
+            return true
+        end
+        evalargs(interp, sv) || push!(sv.tasks, evalargs)
+        return callresult
+    else
+        return Future(abstract_eval_basic_statement(interp, e, sstate, sv))
+    end
+end
+
 function abstract_eval_foreigncall(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState)
+    callee = e.args[1]
+    if isexpr(callee, :call) && length(callee.args) > 1 && callee.args[1] == GlobalRef(Core, :tuple)
+        # NOTE these expressions are not properly linearized
+        abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[2], sstate, sv)
+        if length(callee.args) > 2
+            abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[3], sstate, sv)
+        end
+    else
+        abstract_eval_value(interp, callee, sstate, sv)
+    end
     mi = frame_instance(sv)
     t = sp_type_rewrap(e.args[2], mi, true)
     for i = 3:length(e.args)

base/invalidation.jl renamed to Compiler/src/bindinginvalidations.jl

@@ -1,26 +1,17 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-struct GlobalRefIterator
-    mod::Module
-end
-IteratorSize(::Type{GlobalRefIterator}) = SizeUnknown()
-globalrefs(mod::Module) = GlobalRefIterator(mod)
-
-function iterate(gri::GlobalRefIterator, i = 1)
-    m = gri.mod
-    table = ccall(:jl_module_get_bindings, Ref{SimpleVector}, (Any,), m)
-    i > length(table) && return nothing
-    b = table[i]
-    b === nothing && return iterate(gri, i+1)
-    return ((b::Core.Binding).globalref, i+1)
-end
+using ..Compiler: _uncompressed_ir, specializations, get_ci_mi, convert, unsafe_load, cglobal, generating_output, has_image_globalref,
+    PARTITION_MASK_KIND, PARTITION_KIND_GUARD, PARTITION_FLAG_EXPORTED, PARTITION_FLAG_DEPRECATED,
+    BINDING_FLAG_ANY_IMPLICIT_EDGES, binding_kind, partition_restriction, is_some_imported,
+    is_some_binding_imported, is_some_implicit, SizeUnknown, maybe_add_binding_backedge!, walk_binding_partition, abstract_eval_partition_load, userefs
+using .Core: SimpleVector, CodeInfo
 
 function foreachgr(visit, src::CodeInfo)
     stmts = src.code
     for i = 1:length(stmts)
         stmt = stmts[i]
         isa(stmt, GlobalRef) && visit(stmt)
-        for ur in Compiler.userefs(stmt)
+        for ur in userefs(stmt)
             arg = ur[]
             isa(arg, GlobalRef) && visit(arg)
         end
@@ -35,7 +26,7 @@ function anygr(visit, src::CodeInfo)
             visit(stmt) && return true
             continue
         end
-        for ur in Compiler.userefs(stmt)
+        for ur in userefs(stmt)
             arg = ur[]
             isa(arg, GlobalRef) && visit(arg) && return true
         end
@@ -69,7 +60,7 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid
         src = _uncompressed_ir(method)
         invalidate_all = should_invalidate_code_for_globalref(gr, src)
     end
-    if invalidate_all && !Base.generating_output()
+    if invalidate_all && !generating_output()
         @atomic method.did_scan_source |= 0x4
     end
     invalidated_any = false
@@ -99,15 +90,15 @@ export_affecting_partition_flags(bpart::Core.BindingPartition) =
 function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt)
     gr = b.globalref
 
-    (_, (ib, ibpart)) = Compiler.walk_binding_partition(b, invalidated_bpart, new_max_world)
-    (_, (nb, nbpart)) = Compiler.walk_binding_partition(b, new_bpart, new_max_world+1)
+    (_, (ib, ibpart)) = walk_binding_partition(b, invalidated_bpart, new_max_world)
+    (_, (nb, nbpart)) = walk_binding_partition(b, new_bpart, new_max_world+1)
 
     # `abstract_eval_partition_load` is the maximum amount of information that inference
     # reads from a binding partition. If this information does not change - we do not need to
     # invalidate any code that inference created, because we know that the result will not change.
     need_to_invalidate_code =
-        Compiler.abstract_eval_partition_load(nothing, ib, ibpart) !==
-        Compiler.abstract_eval_partition_load(nothing, nb, nbpart)
+        abstract_eval_partition_load(nothing, ib, ibpart) !==
+        abstract_eval_partition_load(nothing, nb, nbpart)
 
     need_to_invalidate_export = export_affecting_partition_flags(invalidated_bpart) !==
                                 export_affecting_partition_flags(new_bpart)
@@ -155,9 +146,7 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core
                 latest_bpart = user_binding.partitions
                 latest_bpart.max_world == typemax(UInt) || continue
                 is_some_implicit(binding_kind(latest_bpart)) || continue
-                new_bpart = need_to_invalidate_export ?
-                    ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world) :
-                    latest_bpart
+                new_bpart = ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world)
                 if need_to_invalidate_code || new_bpart !== latest_bpart
                     push!(queued_bindings, (convert(Core.Binding, user_binding), latest_bpart, new_bpart))
                 end
@@ -173,12 +162,6 @@ end
 invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) =
     invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world)
 
-function maybe_add_binding_backedge!(b::Core.Binding, edge::Union{Method, CodeInstance})
-    meth = isa(edge, Method) ? edge : get_ci_mi(edge).def
-    ccall(:jl_maybe_add_binding_backedge, Cint, (Any, Any, Any), b, edge, meth)
-    return nothing
-end
-
 function binding_was_invalidated(b::Core.Binding)
     # At least one partition is required for invalidation
     !isdefined(b, :partitions) && return false
@@ -206,7 +189,7 @@ function scan_new_method!(method::Method, image_backedges_only::Bool)
 end
 
 function scan_new_methods!(extext_methods::Vector{Any}, internal_methods::Vector{Any}, image_backedges_only::Bool)
-    if image_backedges_only && Base.generating_output(true)
+    if image_backedges_only && generating_output(true)
         # Replacing image bindings is forbidden during incremental precompilation - skip backedge insertion
         return
     end

Compiler/src/bootstrap.jl

@@ -7,6 +7,8 @@
 
 function activate_codegen!()
     ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
+    # Register the new unified compile and emit function
+    ccall(:jl_set_compile_and_emit_func, Cvoid, (Any,), compile_and_emit_native)
     Core.eval(Compiler, quote
         let typeinf_world_age = Base.tls_world_age()
             @eval Core.OptimizedGenerics.CompilerPlugins.typeinf(::Nothing, mi::MethodInstance, source_mode::UInt8) =

Compiler/src/inferencestate.jl

@@ -587,7 +587,8 @@ end
 _should_instrument(loc::Symbol) = is_file_tracked(loc)
 _should_instrument(loc::Method) = _should_instrument(loc.file)
 _should_instrument(loc::MethodInstance) = _should_instrument(loc.def)
-_should_instrument(loc::Module) = false
+_should_instrument(::Module) = false
+_should_instrument(::Nothing) = false
 function _should_instrument(info::DebugInfo)
     linetable = info.linetable
     linetable === nothing || (_should_instrument(linetable) && return true)

Compiler/src/optimize.jl

@@ -132,6 +132,17 @@ is_declared_noinline(@nospecialize src::MaybeCompressed) =
 #####################
 
 # return whether this src should be inlined. If so, retrieve_ir_for_inlining must return an IRCode from it
+
+function src_inlining_policy(interp::AbstractInterpreter, mi::MethodInstance,
+    @nospecialize(src), @nospecialize(info::CallInfo), stmt_flag::UInt32)
+    # If we have a generator, but we can't invoke it (because argument type information is lacking),
+    # don't inline so we defer its invocation to runtime where we'll have precise type information.
+    if isa(mi.def, Method) && hasgenerator(mi)
+        may_invoke_generator(mi) || return false
+    end
+    return src_inlining_policy(interp, src, info, stmt_flag)
+end
+
 function src_inlining_policy(interp::AbstractInterpreter,
     @nospecialize(src), @nospecialize(info::CallInfo), stmt_flag::UInt32)
     isa(src, OptimizationState) && (src = src.src)