Backports for Julia 1.11.0-rc2 (#54925)

Backported PRs:
- [x] #54201 <!-- Fix generic triangular solves with empty matrices -->
- [x] #54358 <!-- Create `jl_clear_coverage_data` to dynamically reset
coverage -->
- [x] #54908 <!-- LazyString in interpolated error messages in
threadingconstructs -->
- [x] #54952 <!-- LAPACK: Avoid repr call in `chkvalidparam` -->
- [x] #54898 <!-- fix concurrent module loading return value -->
- [x] #55082 <!-- Add fast method for copyto!(::Memory, ::Memory) -->
- [x] #55084 <!-- Use triple quotes in TOML.print when string contains
newline -->
- [x] #55141 <!-- Update the aarch64 devdocs to reflect the current
state of its support -->
- [x] #54955 <!-- don't throw EOFError from sleep -->
- [x] #54871 <!-- Make warn missed transformations pass optional -->
- [x] #55178 <!-- Compat for `Base.@nospecializeinfer` -->
- [x] #55197 <!-- compat notice for a[begin] indexing -->
- [x] #54917 <!-- Fix potential underrun with annotation merging -->
- [x] #55209 <!-- correction to compat notice for a[begin] -->
- [x] #55203 <!-- document mutable struct const fields -->
- [x] #54791 <!-- Bump libblastrampoline to v5.10.1 -->
- [x] #54950 <!-- SuiteSparse: Bump version -->
- [x] #54956 <!-- Fix accidental early evaluation of imported `using`
binding -->
- [x] #54996 <!-- inference: add missing `MustAlias` widening in
`_getfield_tfunc` -->
- [x] #55070 <!-- LinearAlgebra: LazyString in error messages for
Diagonal/Bidiagonal -->
- [x] #54574 <!-- Make ScopedValues public -->
- [x] #54739 <!-- finish implementation of upgradable stdlibs -->
- [x] #54965 <!-- RFC: Make `include_dependency(path;
track_content=true)` the default -->
- [x] #53286 <!-- Raise an error when using `include_dependency` with
non-existent file or directory -->
- [x] #55066 <!-- fix loading of repeated/concurrent modules -->
- [x] #52694 <!-- Reinstate similar for AbstractQ for backward
compatibility -->
- [x] #55218 <!-- Artifacts: use a different way of getting the UUID of
a module -->
- [x] #54891 <!-- #54739-related fixes for loading stdlibs -->
- [x] #55072 <!-- trace-compile: don't generate `precompile` statements
for OpaqueClosure methods -->
- [x] #55188 <!-- Make Core.TypeofUnion use the type method table -->

Need manual backport:
- [ ] #55052 <!-- Fix `(l/r)mul!` with `Diagonal`/`Bidiagonal` -->


Contains multiple commits, manual intervention needed:

Non-merged PRs with backport label:
- [ ] #55169 <!-- `propertynames` for SVD respects private argument -->
- [ ] #55148 <!-- Random: Mark unexported public symbols as public -->
- [ ] #55017 <!-- TOML: Make `Dates` a type parameter -->
- [ ] #55013 <!-- [docs] change docstring to match code -->
- [ ] #54919 <!-- Fix annotated join with non-concrete eltype iters -->
- [ ] #54457 <!-- Make `String(::Memory)` copy -->
- [ ] #53957 <!-- tweak how filtering is done for what packages should
be precompiled -->
- [ ] #51479 <!-- prevent code loading from lookin in the versioned
environment when building Julia -->
- [ ] #50813 <!-- More doctests for Sockets and capitalization fix -->
- [ ] #50157 <!-- improve docs for `@inbounds` and
`Base.@propagate_inbounds` -->
- [ ] #41244 <!-- Fix shell `cd` error when working dir has been deleted
-->

Author: KristofferC

NEWS.md

@@ -88,7 +88,6 @@ New library functions
 * `Sys.username()` can be used to return the current user's username ([#51897]).
 * `Sys.isreadable(), Sys.iswritable()` can be used to check if the current user has access permissions
   that permit reading and writing, respectively. ([#53320]).
-* `wrap(Array, m::Union{MemoryRef{T}, Memory{T}}, dims)` is the safe counterpart to `unsafe_wrap` ([#52049]).
 * `GC.logging_enabled()` can be used to test whether GC logging has been enabled via `GC.enable_logging` ([#51647]).
 * `IdSet` is now exported from Base and considered public ([#53262]).
 * `@time` now reports a count of any lock conflicts where a `ReentrantLock` had to wait, plus a new macro
@@ -142,12 +141,6 @@ Standard library changes
 * The new `@styled_str` string macro provides a convenient way of creating a
   `AnnotatedString` with various faces or other attributes applied ([#49586]).
 
-#### JuliaSyntaxHighlighting
-
-* A new standard library for applying syntax highlighting to Julia code, this
-  uses `JuliaSyntax` and `StyledStrings` to implement a `highlight` function
-  that creates an `AnnotatedString` with syntax highlighting applied.
-
 #### Libdl
 
 * A new `LazyLibrary` type is exported from `Libdl` for use in building chained lazy library

base/Base.jl

@@ -604,6 +604,10 @@ function __init__()
     init_active_project()
     append!(empty!(_sysimage_modules), keys(loaded_modules))
     empty!(explicit_loaded_modules)
+    @assert isempty(loaded_precompiles)
+    for (mod, key) in module_keys
+        loaded_precompiles[key => module_build_id(mod)] = mod
+    end
     if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES")
         MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"])
     end

base/array.jl

@@ -216,7 +216,7 @@ isbitsunion(u::Type) = u isa Union && allocatedinline(u)
 function _unsetindex!(A::Array, i::Int)
     @inline
     @boundscheck checkbounds(A, i)
-    @inbounds _unsetindex!(GenericMemoryRef(A.ref, i))
+    @inbounds _unsetindex!(memoryref(A.ref, i))
     return A
 end
 
@@ -239,14 +239,14 @@ function isassigned(a::Array, i::Int...)
     @_noub_if_noinbounds_meta
     @boundscheck checkbounds(Bool, a, i...) || return false
     ii = _sub2ind(size(a), i...)
-    return @inbounds isassigned(memoryref(a.ref, ii, false))
+    return @inbounds isassigned(memoryrefnew(a.ref, ii, false))
 end
 
 function isassigned(a::Vector, i::Int) # slight compiler simplification for the most common case
     @inline
     @_noub_if_noinbounds_meta
     @boundscheck checkbounds(Bool, a, i) || return false
-    return @inbounds isassigned(memoryref(a.ref, i, false))
+    return @inbounds isassigned(memoryrefnew(a.ref, i, false))
 end
 
 
@@ -281,7 +281,7 @@ the same manner as C.
 """
 function unsafe_copyto!(dest::Array, doffs, src::Array, soffs, n)
     n == 0 && return dest
-    unsafe_copyto!(GenericMemoryRef(dest.ref, doffs), GenericMemoryRef(src.ref, soffs), n)
+    unsafe_copyto!(memoryref(dest.ref, doffs), memoryref(src.ref, soffs), n)
     return dest
 end
 
@@ -303,8 +303,8 @@ function _copyto_impl!(dest::Union{Array,Memory}, doffs::Integer, src::Union{Arr
     n > 0 || _throw_argerror("Number of elements to copy must be non-negative.")
     @boundscheck checkbounds(dest, doffs:doffs+n-1)
     @boundscheck checkbounds(src, soffs:soffs+n-1)
-    @inbounds let dest = GenericMemoryRef(dest isa Array ? getfield(dest, :ref) : dest, doffs)
-                  src = GenericMemoryRef(src isa Array ? getfield(src, :ref) : src, soffs)
+    @inbounds let dest = memoryref(dest isa Array ? getfield(dest, :ref) : dest, doffs),
+                  src = memoryref(src isa Array ? getfield(src, :ref) : src, soffs)
         unsafe_copyto!(dest, src, n)
     end
     return dest
@@ -348,7 +348,7 @@ copy
     @_nothrow_meta
     ref = a.ref
     newmem = ccall(:jl_genericmemory_copy_slice, Ref{Memory{T}}, (Any, Ptr{Cvoid}, Int), ref.mem, ref.ptr_or_offset, length(a))
-    return $(Expr(:new, :(typeof(a)), :(Core.memoryref(newmem)), :(a.size)))
+    return $(Expr(:new, :(typeof(a)), :(memoryref(newmem)), :(a.size)))
 end
 
 ## Constructors ##
@@ -973,21 +973,21 @@ function setindex! end
 function setindex!(A::Array{T}, x, i::Int) where {T}
     @_noub_if_noinbounds_meta
     @boundscheck (i - 1)%UInt < length(A)%UInt || throw_boundserror(A, (i,))
-    memoryrefset!(memoryref(A.ref, i, false), x isa T ? x : convert(T,x)::T, :not_atomic, false)
+    memoryrefset!(memoryrefnew(A.ref, i, false), x isa T ? x : convert(T,x)::T, :not_atomic, false)
     return A
 end
 function setindex!(A::Array{T}, x, i1::Int, i2::Int, I::Int...) where {T}
     @inline
     @_noub_if_noinbounds_meta
     @boundscheck checkbounds(A, i1, i2, I...) # generally _to_linear_index requires bounds checking
-    memoryrefset!(memoryref(A.ref, _to_linear_index(A, i1, i2, I...), false), x isa T ? x : convert(T,x)::T, :not_atomic, false)
+    memoryrefset!(memoryrefnew(A.ref, _to_linear_index(A, i1, i2, I...), false), x isa T ? x : convert(T,x)::T, :not_atomic, false)
     return A
 end
 
 __safe_setindex!(A::Vector{Any}, @nospecialize(x), i::Int) = (@inline; @_nothrow_noub_meta;
-    memoryrefset!(memoryref(A.ref, i, false), x, :not_atomic, false); return A)
+    memoryrefset!(memoryrefnew(A.ref, i, false), x, :not_atomic, false); return A)
 __safe_setindex!(A::Vector{T}, x::T, i::Int) where {T} = (@inline; @_nothrow_noub_meta;
-    memoryrefset!(memoryref(A.ref, i, false), x, :not_atomic, false); return A)
+    memoryrefset!(memoryrefnew(A.ref, i, false), x, :not_atomic, false); return A)
 __safe_setindex!(A::Vector{T}, x,    i::Int) where {T} = (@inline;
     __safe_setindex!(A, convert(T, x)::T, i))
 
@@ -1059,7 +1059,7 @@ function _growbeg!(a::Vector, delta::Integer)
     setfield!(a, :size, (newlen,))
     # if offset is far enough advanced to fit data in existing memory without copying
     if delta <= offset - 1
-        setfield!(a, :ref, @inbounds GenericMemoryRef(ref, 1 - delta))
+        setfield!(a, :ref, @inbounds memoryref(ref, 1 - delta))
     else
         @noinline (function()
         memlen = length(mem)
@@ -1078,7 +1078,7 @@ function _growbeg!(a::Vector, delta::Integer)
             newmem = array_new_memory(mem, newmemlen)
         end
         unsafe_copyto!(newmem, newoffset + delta, mem, offset, len)
-        setfield!(a, :ref, @inbounds GenericMemoryRef(newmem, newoffset))
+        setfield!(a, :ref, @inbounds memoryref(newmem, newoffset))
         end)()
     end
     return
@@ -1113,7 +1113,7 @@ function _growend!(a::Vector, delta::Integer)
             newmem = array_new_memory(mem, newmemlen2)
             newoffset = offset
         end
-        newref = @inbounds GenericMemoryRef(newmem, newoffset)
+        newref = @inbounds memoryref(newmem, newoffset)
         unsafe_copyto!(newref, ref, len)
         setfield!(a, :ref, newref)
         end)()
@@ -1142,7 +1142,7 @@ function _growat!(a::Vector, i::Integer, delta::Integer)
     prefer_start = i <= div(len, 2)
     # if offset is far enough advanced to fit data in beginning of the memory
     if prefer_start && delta <= offset - 1
-        newref = @inbounds GenericMemoryRef(mem, offset - delta)
+        newref = @inbounds memoryref(mem, offset - delta)
         unsafe_copyto!(newref, ref, i)
         setfield!(a, :ref, newref)
         for j in i:i+delta-1
@@ -1159,7 +1159,7 @@ function _growat!(a::Vector, i::Integer, delta::Integer)
         newmemlen = max(overallocation(memlen), len+2*delta+1)
         newoffset = (newmemlen - newlen) ÷ 2 + 1
         newmem = array_new_memory(mem, newmemlen)
-        newref = @inbounds GenericMemoryRef(newmem, newoffset)
+        newref = @inbounds memoryref(newmem, newoffset)
         unsafe_copyto!(newref, ref, i-1)
         unsafe_copyto!(newmem, newoffset + delta + i - 1, mem, offset + i - 1, len - i + 1)
         setfield!(a, :ref, newref)
@@ -1176,7 +1176,7 @@ function _deletebeg!(a::Vector, delta::Integer)
     end
     newlen = len - delta
     if newlen != 0 # if newlen==0 we could accidentally index past the memory
-        newref = @inbounds GenericMemoryRef(a.ref, delta + 1)
+        newref = @inbounds memoryref(a.ref, delta + 1)
         setfield!(a, :ref, newref)
     end
     setfield!(a, :size, (newlen,))
@@ -1491,16 +1491,16 @@ function sizehint!(a::Vector, sz::Integer; first::Bool=false, shrink::Bool=true)
         end
         newmem = array_new_memory(mem, sz)
         if first
-            newref = GenericMemoryRef(newmem, inc + 1)
+            newref = memoryref(newmem, inc + 1)
         else
-            newref = GenericMemoryRef(newmem)
+            newref = memoryref(newmem)
         end
         unsafe_copyto!(newref, ref, len)
         setfield!(a, :ref, newref)
     elseif first
         _growbeg!(a, inc)
         newref = getfield(a, :ref)
-        newref = GenericMemoryRef(newref, inc + 1)
+        newref = memoryref(newref, inc + 1)
         setfield!(a, :size, (len,)) # undo the size change from _growbeg!
         setfield!(a, :ref, newref) # undo the offset change from _growbeg!
     else # last
@@ -3075,15 +3075,17 @@ function _wrap(ref::MemoryRef{T}, dims::NTuple{N, Int}) where {T, N}
     mem_len = length(mem) + 1 - memoryrefoffset(ref)
     len = Core.checked_dims(dims...)
     @boundscheck mem_len >= len || invalid_wrap_err(mem_len, dims, len)
-    if N != 1 && !(ref === GenericMemoryRef(mem) && len === mem_len)
+    if N != 1 && !(ref === memoryref(mem) && len === mem_len)
         mem = ccall(:jl_genericmemory_slice, Memory{T}, (Any, Ptr{Cvoid}, Int), mem, ref.ptr_or_offset, len)
-        ref = MemoryRef(mem)
+        ref = memoryref(mem)
     end
     return ref
 end
 
-@noinline invalid_wrap_err(len, dims, proddims) = throw(DimensionMismatch(
-    "Attempted to wrap a MemoryRef of length $len with an Array of size dims=$dims, which is invalid because prod(dims) = $proddims > $len, so that the array would have more elements than the underlying memory can store."))
+@noinline invalid_wrap_err(len, dims, proddims) = throw(DimensionMismatch(LazyString(
+    "Attempted to wrap a MemoryRef of length ", len, " with an Array of size dims=", dims,
+    " which is invalid because prod(dims) = ", proddims, " > ", len,
+    " so that the array would have more elements than the underlying memory can store.")))
 
 @eval @propagate_inbounds function wrap(::Type{Array}, m::MemoryRef{T}, dims::NTuple{N, Integer}) where {T, N}
     dims = convert(Dims, dims)
@@ -3093,7 +3095,7 @@ end
 
 @eval @propagate_inbounds function wrap(::Type{Array}, m::Memory{T}, dims::NTuple{N, Integer}) where {T, N}
     dims = convert(Dims, dims)
-    ref = _wrap(MemoryRef(m), dims)
+    ref = _wrap(memoryref(m), dims)
     $(Expr(:new, :(Array{T, N}), :ref, :dims))
 end
 @eval @propagate_inbounds function wrap(::Type{Array}, m::MemoryRef{T}, l::Integer) where {T}
@@ -3103,11 +3105,11 @@ end
 end
 @eval @propagate_inbounds function wrap(::Type{Array}, m::Memory{T}, l::Integer) where {T}
     dims = (Int(l),)
-    ref = _wrap(MemoryRef(m), (l,))
+    ref = _wrap(memoryref(m), (l,))
     $(Expr(:new, :(Array{T, 1}), :ref, :dims))
 end
 @eval @propagate_inbounds function wrap(::Type{Array}, m::Memory{T}) where {T}
-    ref = MemoryRef(m)
+    ref = memoryref(m)
     dims = (length(m),)
     $(Expr(:new, :(Array{T, 1}), :ref, :dims))
 end

base/asyncevent.jl

@@ -127,8 +127,11 @@ function _trywait(t::Union{Timer, AsyncCondition})
         t isa Timer || Core.Intrinsics.atomic_fence(:acquire_release)
     else
         if !isopen(t)
-            close(t) # wait for the close to complete
-            return false
+            set = t.set
+            if !set
+                close(t) # wait for the close to complete
+                return false
+            end
         end
         iolock_begin()
         set = t.set
@@ -151,7 +154,7 @@ function _trywait(t::Union{Timer, AsyncCondition})
         end
         iolock_end()
     end
-    @atomic :monotonic t.set = false
+    @atomic :monotonic t.set = false # if there are multiple waiters, an unspecified number may short-circuit past here
     return set
 end
 
@@ -161,14 +164,14 @@ function wait(t::Union{Timer, AsyncCondition})
 end
 
 
-isopen(t::Union{Timer, AsyncCondition}) = t.isopen && t.handle != C_NULL
+isopen(t::Union{Timer, AsyncCondition}) = @atomic :acquire t.isopen
 
 function close(t::Union{Timer, AsyncCondition})
-    t.handle == C_NULL && return # short-circuit path
+    t.handle == C_NULL && !t.isopen && return # short-circuit path, :monotonic
     iolock_begin()
     if t.handle != C_NULL
         if t.isopen
-            @atomic :monotonic t.isopen = false
+            @atomic :release t.isopen = false
             ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), t)
         end
         # implement _trywait here without the auto-reset function, just waiting for the final close signal
@@ -186,6 +189,8 @@ function close(t::Union{Timer, AsyncCondition})
             unlock(t.cond)
             unpreserve_handle(t)
         end
+    elseif t.isopen
+        @atomic :release t.isopen = false
     end
     iolock_end()
     nothing
@@ -198,8 +203,8 @@ function uvfinalize(t::Union{Timer, AsyncCondition})
         if t.handle != C_NULL
             disassociate_julia_struct(t.handle) # not going to call the usual close hooks anymore
             if t.isopen
-                @atomic :monotonic t.isopen = false
-                ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), t.handle)
+                @atomic :release t.isopen = false
+                ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), t.handle) # this will call Libc.free
             end
             @atomic :monotonic t.handle = C_NULL
             notify(t.cond, false)
@@ -214,8 +219,10 @@ end
 function _uv_hook_close(t::Union{Timer, AsyncCondition})
     lock(t.cond)
     try
-        @atomic :monotonic t.isopen = false
-        Libc.free(@atomicswap :monotonic t.handle = C_NULL)
+        handle = t.handle
+        @atomic :release t.isopen = false
+        @atomic :monotonic t.handle = C_NULL
+        Libc.free(handle)
         notify(t.cond, false)
     finally
         unlock(t.cond)
@@ -243,7 +250,7 @@ function uv_timercb(handle::Ptr{Cvoid})
         if ccall(:uv_timer_get_repeat, UInt64, (Ptr{Cvoid},), t) == 0
             # timer is stopped now
             if t.isopen
-                @atomic :monotonic t.isopen = false
+                @atomic :release t.isopen = false
                 ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), t)
             end
         end

base/boot.jl

@@ -518,15 +518,12 @@ const undef = UndefInitializer()
 # empty vector constructor
 (self::Type{GenericMemory{kind,T,addrspace}})() where {T,kind,addrspace} = self(undef, 0)
 
+memoryref(mem::GenericMemory) = memoryrefnew(mem)
+memoryref(mem::GenericMemory, i::Integer) = memoryrefnew(memoryrefnew(mem), Int(i), @_boundscheck)
+memoryref(ref::GenericMemoryRef, i::Integer) = memoryrefnew(ref, Int(i), @_boundscheck)
 GenericMemoryRef(mem::GenericMemory) = memoryref(mem)
-GenericMemoryRef(ref::GenericMemoryRef, i::Integer) = memoryref(ref, Int(i), @_boundscheck)
-GenericMemoryRef(mem::GenericMemory, i::Integer) = memoryref(memoryref(mem), Int(i), @_boundscheck)
-GenericMemoryRef{kind,<:Any,AS}(mem::GenericMemory{kind,<:Any,AS}) where {kind,AS} = memoryref(mem)
-GenericMemoryRef{kind,<:Any,AS}(ref::GenericMemoryRef{kind,<:Any,AS}, i::Integer) where {kind,AS} = memoryref(ref, Int(i), @_boundscheck)
-GenericMemoryRef{kind,<:Any,AS}(mem::GenericMemory{kind,<:Any,AS}, i::Integer) where {kind,AS}  = memoryref(memoryref(mem), Int(i), @_boundscheck)
-GenericMemoryRef{kind,T,AS}(mem::GenericMemory{kind,T,AS}) where {kind,T,AS}  = memoryref(mem)
-GenericMemoryRef{kind,T,AS}(ref::GenericMemoryRef{kind,T,AS}, i::Integer) where {kind,T,AS} = memoryref(ref, Int(i), @_boundscheck)
-GenericMemoryRef{kind,T,AS}(mem::GenericMemory{kind,T,AS}, i::Integer) where {kind,T,AS} = memoryref(memoryref(mem), Int(i), @_boundscheck)
+GenericMemoryRef(mem::GenericMemory, i::Integer) = memoryref(mem, i)
+GenericMemoryRef(mem::GenericMemoryRef, i::Integer) = memoryref(mem, i)
 
 const Memory{T} = GenericMemory{:not_atomic, T, CPU}
 const MemoryRef{T} = GenericMemoryRef{:not_atomic, T, CPU}
@@ -634,12 +631,12 @@ module IR
 export CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode,
     NewvarNode, SSAValue, SlotNumber, Argument,
     PiNode, PhiNode, PhiCNode, UpsilonNode, LineInfoNode,
-    Const, PartialStruct, InterConditional, EnterNode
+    Const, PartialStruct, InterConditional, EnterNode, memoryref
 
 using Core: CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode,
     NewvarNode, SSAValue, SlotNumber, Argument,
     PiNode, PhiNode, PhiCNode, UpsilonNode, LineInfoNode,
-    Const, PartialStruct, InterConditional, EnterNode
+    Const, PartialStruct, InterConditional, EnterNode, memoryref
 
 end # module IR
 

base/compiler/optimize.jl

@@ -677,7 +677,7 @@ function iscall_with_boundscheck(@nospecialize(stmt), sv::PostOptAnalysisState)
     f === nothing && return false
     if f === getfield
         nargs = 4
-    elseif f === memoryref || f === memoryrefget || f === memoryref_isassigned
+    elseif f === memoryrefnew || f === memoryrefget || f === memoryref_isassigned
         nargs = 4
     elseif f === memoryrefset!
         nargs = 5