Merge branch 'master' into oc_inworld

Author: adienes

.github/workflows/Typos.yml

@@ -11,7 +11,7 @@ jobs:
     timeout-minutes: 5
     steps:
       - name: Checkout the JuliaLang/julia repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
       - name: Check spelling with typos

.github/workflows/Whitespace.yml

@@ -15,7 +15,7 @@ jobs:
     timeout-minutes: 2
     steps:
       - name: Checkout the JuliaLang/julia repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
       - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2.6.1

.github/workflows/cffconvert.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out a copy of the repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
 

.vscode/c_cpp_properties.json

@@ -0,0 +1,20 @@
+{
+    "configurations": [
+        {
+            "name": "Julia",
+            "includePath": [
+                "${workspaceFolder}/src/**",
+                "${workspaceFolder}/usr/include/**"
+            ],
+            "cStandard": "c11",
+            "cppStandard": "c++17",
+            "compileCommands": [
+                "${workspaceFolder}/src/compile_commands.json",
+                "${workspaceFolder}/src/flisp/compile_commands.json",
+                "${workspaceFolder}/src/support/compile_commands.json"
+            ]
+        }
+    ],
+    "version": 4,
+    "enableConfigurationSquiggles": true
+}

AGENTS.md

@@ -19,14 +19,20 @@ will not be reflected, unless you use `Revise`.
 
 ## For all changes
 
-1. Run `make check-whitespace` before creating the PR to make sure you're not committing any whitespace errors.
+1. Run `make fix-whitespace` before creating the PR to make sure you're not committing any whitespace errors.
 
 ## Building Julia
 
 If you made changes to the runtime (any files in `src/`), you will need to rebuild
 julia. Run `make -j` to rebuild julia. This process may take up to 10 minutes
 depending on your changes.
 
+After making changes, run static analysis checks:
+  - First run `make -C src install-analysis-deps` to initialize dependencies (only needed once the first time).
+  - Run `make -C src analyze-<filename> --output-sync -j8` (replace `<filename>` with the basename of any C or C++ file you modified, excluding headers).
+  - Tests can also be rerun individually with `clang-sa-<filename>`, `clang-sagc-<filename>` or `clang-tidy-<filename>`.
+  - If `clang-sagc-<filename>` fails, it may require adding `JL_GC_PUSH` statements, or `JL_GC_PROMISE_ROOTED` statements., or require fixing locks. Remember arguments are assumed rooted, so check the callers to make sure that is handled. If the value is being temporarily moved around in a struct or arraylist, `JL_GC_PROMISE_ROOTED(struct->field)` may be needed as a statement (it return void) immediately after reloading the struct before any use of struct. Put that promise as early in the code as is legal, near the definition not the use.
+
 ## Using Revise
 
 If you have made changes to files included in the system image (base/ or stdlib/),
@@ -76,6 +82,13 @@ When modifying external dependencies (patches in `deps/patches/` or version upda
    - Prefer using the full upstream commit in `git am` format (e.g., `git format-patch`) which includes proper commit metadata
 3. When updating dependency versions, ensure all associated patches still apply
 
+### External JLLs
+
+To update a JLL to the latest version:
+- Update the version number in the appropriate jll folder
+- If the dependencies in the upstream jll changed, update the Project.toml
+- Run `make -f contrib/refresh_checksums.mk <jll>` to update the checksums. This may take a few minutes.
+
 ### Writing code
 After writing code, look up the docstring for each function you used. If there
 are recommendations or additional considerations that apply to these functions,
@@ -85,7 +98,7 @@ make sure to take them into account.
 - Do not `ccall` runtime C functions directly if there are existing wrappers for the function.
 - Do not explicitly add a module prefix if the code you're adding is in the same module. E.g. do not use `Base.` for code in Base unless required.
 
-## Commit message formatting
+## Commit messages and pull requests
 
 When writing commit messages, follow the format "component: Brief summary" for
 the title. In the body of the commit message, provide a brief prose summary
@@ -97,7 +110,7 @@ If your change fixes one or more issues, use the syntax "Fixes #" at the end of
 When referencing external GitHub PRs or issues, use proper GitHub interlinking format (e.g., `owner/repo#123` for PRs/issues).
 When fixing CI failures, include the link to the specific CI failure in the commit message.
 
-When creating pull requests, if the pull request consists of one commit only,
-use the body of the commit for the body of the pull request. If there are multiple
-commits in the pull request, follow the same guidelines for the pull request
-as for the commit body.
+When creating pull requests:
+1. If the pull request consists of one commit only, use the body of the commit for the body of the pull request.
+2. If there are multiple commits in the pull request, follow the same guidelines for the pull request as for the commit body.
+3. Make sure that the base commit of the pull request is recent (within the past two days) - if not rebase your changes first.

Compiler/src/Compiler.jl

@@ -48,12 +48,12 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali
     PARTITION_KIND_GLOBAL, PARTITION_KIND_UNDEF_CONST, PARTITION_KIND_BACKDATED_CONST, PARTITION_KIND_DECLARED,
     PARTITION_FLAG_DEPWARN,
     Base, BitVector, Bottom, Callable, DataTypeFieldDesc,
-    EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES,
-    OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME,
+    EffectsOverride, Filter, Generator, NUM_EFFECTS_OVERRIDES,
+    OneTo, Ordering, RefValue, _NAMEDTUPLE_NAME,
     _array_for, _bits_findnext, _methods_by_ftype, _uniontypes, all, allocatedinline, any,
     argument_datatype, binding_kind, cconvert, copy_exprargs, datatype_arrayelem,
     datatype_fieldcount, datatype_fieldtypes, datatype_layoutsize, datatype_nfields,
-    datatype_pointerfree, decode_effects_override, diff_names, fieldindex,
+    datatype_pointerfree, decode_effects_override, diff_names, fieldindex, visit,
     generating_output, get_nospecializeinfer_sig, get_world_counter, has_free_typevars,
     hasgenerator, hasintersect, indexed_iterate, isType, is_file_tracked, is_function_def,
     is_meta_expr, is_meta_expr_head, is_nospecialized, is_nospecializeinfer, is_defined_const_binding,
@@ -64,9 +64,11 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali
     partition_restriction, quoted, rename_unionall, rewrap_unionall, specialize_method,
     structdiff, tls_world_age, unconstrain_vararg_length, unionlen, uniontype_layout,
     uniontypes, unsafe_convert, unwrap_unionall, unwrapva, vect, widen_diagonal,
-    _uncompressed_ir, maybe_add_binding_backedge!, datatype_min_ninitialized,
+    _uncompressed_ir, datatype_min_ninitialized,
     partialstruct_init_undefs, fieldcount_noerror, _eval_import, _eval_using,
-    get_ci_mi
+    get_ci_mi, get_methodtable, morespecific, specializations, has_image_globalref,
+    PARTITION_MASK_KIND, PARTITION_KIND_GUARD, PARTITION_FLAG_EXPORTED, PARTITION_FLAG_DEPRECATED,
+    BINDING_FLAG_ANY_IMPLICIT_EDGES, is_some_implicit, IteratorSize, SizeUnknown, get_require_world, JLOptions
 
 using Base.Order
 
@@ -185,9 +187,14 @@ include("typeinfer.jl")
 include("optimize.jl")
 
 include("bootstrap.jl")
+include("precompile.jl")
 include("reflection_interface.jl")
 include("opaque_closure.jl")
 
+baremodule ReinferUtils end
+include(ReinferUtils, "reinfer.jl")
+include(ReinferUtils, "bindinginvalidations.jl")
+
 macro __SOURCE_FILE__()
     __source__.file === nothing && return nothing
     return QuoteNode(__source__.file::Symbol)

Compiler/src/abstractinterpretation.jl

@@ -369,7 +369,7 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes::
         end
         valid_worlds = intersect(valid_worlds, thismatches.valid_worlds)
         thisfullmatch = any(match::MethodMatch->match.fully_covers, thismatches)
-        mt = Core.GlobalMethods
+        mt = Core.methodtable
         thisinfo = MethodMatchInfo(thismatches, mt, sig_n, thisfullmatch)
         push!(infos, thisinfo)
         for idx = 1:length(thismatches)
@@ -390,7 +390,7 @@ function find_simple_method_matches(interp::AbstractInterpreter, @nospecialize(a
         return FailedMethodMatch("Too many methods matched")
     end
     fullmatch = any(match::MethodMatch->match.fully_covers, matches)
-    mt = Core.GlobalMethods
+    mt = Core.methodtable
     info = MethodMatchInfo(matches, mt, atype, fullmatch)
     applicable = MethodMatchTarget[MethodMatchTarget(matches[idx], info.edges, idx) for idx = 1:length(matches)]
     return MethodMatches(applicable, info, matches.valid_worlds)
@@ -1245,7 +1245,7 @@ function const_prop_methodinstance_heuristic(interp::AbstractInterpreter,
         if isa(code, CodeInstance)
             inferred = @atomic :monotonic code.inferred
             # TODO propagate a specific `CallInfo` that conveys information about this call
-            if src_inlining_policy(interp, inferred, NoCallInfo(), IR_FLAG_NULL)
+            if src_inlining_policy(interp, mi, inferred, NoCallInfo(), IR_FLAG_NULL)
                 return true
             end
         end
@@ -3255,7 +3255,7 @@ function abstract_eval_copyast(interp::AbstractInterpreter, e::Expr, sstate::Sta
     return RTEffects(rt, Any, effects)
 end
 
-function abstract_eval_isdefined_expr(interp::AbstractInterpreter, e::Expr, sstate::StatementState,
+function abstract_eval_isdefined_expr(::AbstractInterpreter, e::Expr, sstate::StatementState,
                                       sv::AbsIntState)
     sym = e.args[1]
     if isa(sym, SlotNumber) && sstate.vtypes !== nothing
@@ -3485,7 +3485,59 @@ function refine_partial_type(@nospecialize t)
     return t
 end
 
+abstract_eval_nonlinearized_foreigncall_name(interp::AbstractInterpreter, e, sstate::StatementState, sv::IRInterpretationState) = nothing
+
+function abstract_eval_nonlinearized_foreigncall_name(interp::AbstractInterpreter, e, sstate::StatementState, sv::AbsIntState)
+    if isexpr(e, :call)
+        n = length(e.args)
+        argtypes = Vector{Any}(undef, n)
+        callresult = Future{CallMeta}()
+        i::Int = 1
+        nextstate::UInt8 = 0x0
+        local ai, res
+        function evalargs(interp, sv)
+            if nextstate === 0x1
+                @goto state1
+            elseif nextstate === 0x2
+                @goto state2
+            end
+            while i <= n
+                ai = abstract_eval_nonlinearized_foreigncall_name(interp, e.args[i], sstate, sv)
+                if !isready(ai)
+                    nextstate = 0x1
+                    return false
+                    @label state1
+                end
+                argtypes[i] = ai[].rt
+                i += 1
+            end
+            res = abstract_call(interp, ArgInfo(e.args, argtypes), sstate, sv)
+            if !isready(res)
+                nextstate = 0x2
+                return false
+                @label state2
+            end
+            callresult[] = res[]
+            return true
+        end
+        evalargs(interp, sv) || push!(sv.tasks, evalargs)
+        return callresult
+    else
+        return Future(abstract_eval_basic_statement(interp, e, sstate, sv))
+    end
+end
+
 function abstract_eval_foreigncall(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState)
+    callee = e.args[1]
+    if isexpr(callee, :call) && length(callee.args) > 1 && callee.args[1] == GlobalRef(Core, :tuple)
+        # NOTE these expressions are not properly linearized
+        abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[2], sstate, sv)
+        if length(callee.args) > 2
+            abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[3], sstate, sv)
+        end
+    else
+        abstract_eval_value(interp, callee, sstate, sv)
+    end
     mi = frame_instance(sv)
     t = sp_type_rewrap(e.args[2], mi, true)
     for i = 3:length(e.args)

base/invalidation.jl renamed to Compiler/src/bindinginvalidations.jl

@@ -1,26 +1,17 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-struct GlobalRefIterator
-    mod::Module
-end
-IteratorSize(::Type{GlobalRefIterator}) = SizeUnknown()
-globalrefs(mod::Module) = GlobalRefIterator(mod)
-
-function iterate(gri::GlobalRefIterator, i = 1)
-    m = gri.mod
-    table = ccall(:jl_module_get_bindings, Ref{SimpleVector}, (Any,), m)
-    i > length(table) && return nothing
-    b = table[i]
-    b === nothing && return iterate(gri, i+1)
-    return ((b::Core.Binding).globalref, i+1)
-end
+using ..Compiler: _uncompressed_ir, specializations, get_ci_mi, convert, unsafe_load, cglobal, generating_output, has_image_globalref,
+    PARTITION_MASK_KIND, PARTITION_KIND_GUARD, PARTITION_FLAG_EXPORTED, PARTITION_FLAG_DEPRECATED,
+    BINDING_FLAG_ANY_IMPLICIT_EDGES, binding_kind, partition_restriction, is_some_imported,
+    is_some_binding_imported, is_some_implicit, SizeUnknown, maybe_add_binding_backedge!, walk_binding_partition, abstract_eval_partition_load, userefs
+using .Core: SimpleVector, CodeInfo
 
 function foreachgr(visit, src::CodeInfo)
     stmts = src.code
     for i = 1:length(stmts)
         stmt = stmts[i]
         isa(stmt, GlobalRef) && visit(stmt)
-        for ur in Compiler.userefs(stmt)
+        for ur in userefs(stmt)
             arg = ur[]
             isa(arg, GlobalRef) && visit(arg)
         end
@@ -35,7 +26,7 @@ function anygr(visit, src::CodeInfo)
             visit(stmt) && return true
             continue
         end
-        for ur in Compiler.userefs(stmt)
+        for ur in userefs(stmt)
             arg = ur[]
             isa(arg, GlobalRef) && visit(arg) && return true
         end
@@ -69,7 +60,7 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid
         src = _uncompressed_ir(method)
         invalidate_all = should_invalidate_code_for_globalref(gr, src)
     end
-    if invalidate_all && !Base.generating_output()
+    if invalidate_all && !generating_output()
         @atomic method.did_scan_source |= 0x4
     end
     invalidated_any = false
@@ -99,15 +90,15 @@ export_affecting_partition_flags(bpart::Core.BindingPartition) =
 function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt)
     gr = b.globalref
 
-    (_, (ib, ibpart)) = Compiler.walk_binding_partition(b, invalidated_bpart, new_max_world)
-    (_, (nb, nbpart)) = Compiler.walk_binding_partition(b, new_bpart, new_max_world+1)
+    (_, (ib, ibpart)) = walk_binding_partition(b, invalidated_bpart, new_max_world)
+    (_, (nb, nbpart)) = walk_binding_partition(b, new_bpart, new_max_world+1)
 
     # `abstract_eval_partition_load` is the maximum amount of information that inference
     # reads from a binding partition. If this information does not change - we do not need to
     # invalidate any code that inference created, because we know that the result will not change.
     need_to_invalidate_code =
-        Compiler.abstract_eval_partition_load(nothing, ib, ibpart) !==
-        Compiler.abstract_eval_partition_load(nothing, nb, nbpart)
+        abstract_eval_partition_load(nothing, ib, ibpart) !==
+        abstract_eval_partition_load(nothing, nb, nbpart)
 
     need_to_invalidate_export = export_affecting_partition_flags(invalidated_bpart) !==
                                 export_affecting_partition_flags(new_bpart)
@@ -155,9 +146,7 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core
                 latest_bpart = user_binding.partitions
                 latest_bpart.max_world == typemax(UInt) || continue
                 is_some_implicit(binding_kind(latest_bpart)) || continue
-                new_bpart = need_to_invalidate_export ?
-                    ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world) :
-                    latest_bpart
+                new_bpart = ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world)
                 if need_to_invalidate_code || new_bpart !== latest_bpart
                     push!(queued_bindings, (convert(Core.Binding, user_binding), latest_bpart, new_bpart))
                 end
@@ -173,12 +162,6 @@ end
 invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) =
     invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world)
 
-function maybe_add_binding_backedge!(b::Core.Binding, edge::Union{Method, CodeInstance})
-    meth = isa(edge, Method) ? edge : get_ci_mi(edge).def
-    ccall(:jl_maybe_add_binding_backedge, Cint, (Any, Any, Any), b, edge, meth)
-    return nothing
-end
-
 function binding_was_invalidated(b::Core.Binding)
     # At least one partition is required for invalidation
     !isdefined(b, :partitions) && return false
@@ -206,7 +189,7 @@ function scan_new_method!(method::Method, image_backedges_only::Bool)
 end
 
 function scan_new_methods!(extext_methods::Vector{Any}, internal_methods::Vector{Any}, image_backedges_only::Bool)
-    if image_backedges_only && Base.generating_output(true)
+    if image_backedges_only && generating_output(true)
         # Replacing image bindings is forbidden during incremental precompilation - skip backedge insertion
         return
     end

Compiler/src/bootstrap.jl

@@ -7,6 +7,8 @@
 
 function activate_codegen!()
     ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
+    # Register the new unified compile and emit function
+    ccall(:jl_set_compile_and_emit_func, Cvoid, (Any,), compile_and_emit_native)
     Core.eval(Compiler, quote
         let typeinf_world_age = Base.tls_world_age()
             @eval Core.OptimizedGenerics.CompilerPlugins.typeinf(::Nothing, mi::MethodInstance, source_mode::UInt8) =

Compiler/src/inferencestate.jl

@@ -587,7 +587,8 @@ end
 _should_instrument(loc::Symbol) = is_file_tracked(loc)
 _should_instrument(loc::Method) = _should_instrument(loc.file)
 _should_instrument(loc::MethodInstance) = _should_instrument(loc.def)
-_should_instrument(loc::Module) = false
+_should_instrument(::Module) = false
+_should_instrument(::Nothing) = false
 function _should_instrument(info::DebugInfo)
     linetable = info.linetable
     linetable === nothing || (_should_instrument(linetable) && return true)