Convert more stdlibs to LazyLibraries

This converts more JLL stdlibs to use lazily-loaded libraries, and as a
useful side-effect, causes them to be loaded by absolute path, isolating
them all from `LD_LIBRARY_PATH`-like effects.

fixups

lazify dSFMT_jll

lazify GMP_jll

lazify libLLVM_jll

lazify MPFR_jll

lazify OpenLibm_jll

lazify PCRE2_jll

lazify SuiteSparse_jll

add small precompiles

Add `is_available()` for `LibUV_jll`

Add initial `stdlib_dependencies` test

This allows for auditing the expressed library dependencies in our
LazyLibrary JLL definitions, to ensure that we don't get out of sync
with the actual binaries.

fix typo

add stdlib_dependencies.jl to choosetests

test fixes etc.

test fixes etc.

more test fixes

morefix

missing eager_modes

dep fixes

Author: staticfloat

stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl

@@ -64,13 +64,30 @@ if @isdefined(_libatomic_path)
     const libatomic = LazyLibrary(_libatomic_path)
 end
 const libgcc_s = LazyLibrary(_libgcc_s_path)
-libgfortran_deps = [libgcc_s]
-if @isdefined _libquadmath_path
-    const libquadmath = LazyLibrary(_libquadmath_path)
-    push!(libgfortran_deps, libquadmath)
+
+@static if Sys.isfreebsd()
+    _libgfortran_deps = LazyLibrary[]
+else
+    _libgfortran_deps = [libgcc_s]
+    if @isdefined _libquadmath_path
+        const libquadmath = LazyLibrary(_libquadmath_path)
+        push!(_libgfortran_deps, libquadmath)
+    end
+end
+
+const libgfortran = LazyLibrary(_libgfortran_path, dependencies=_libgfortran_deps)
+
+if Sys.isfreebsd()
+    _libstdcxx_dependencies = LazyLibrary[]
+elseif Sys.isapple()
+    _libstdcxx_dependencies = LazyLibrary[libgcc_s]
+elseif Sys.islinux()
+    _libstdcxx_dependencies = LazyLibrary[libgcc_s]
+else
+    _libstdcxx_dependencies = LazyLibrary[libgcc_s, libgfortran]
 end
-const libgfortran = LazyLibrary(_libgfortran_path, dependencies=libgfortran_deps)
-const libstdcxx = LazyLibrary(_libstdcxx_path, dependencies=[libgcc_s])
+const libstdcxx = LazyLibrary(_libstdcxx_path, dependencies=_libstdcxx_dependencies)
+
 const libgomp = LazyLibrary(_libgomp_path)
 
 # Some installations (such as those from-source) may not have `libssp`
@@ -116,4 +133,9 @@ function __init__()
     push!(LIBPATH_list, LIBPATH[])
 end
 
+if Base.generating_output()
+    precompile(eager_mode, ())
+    precompile(is_available, ())
+end
+
 end  # module CompilerSupportLibraries_jll

stdlib/GMP_jll/Project.toml

@@ -4,6 +4,7 @@ version = "6.3.0+2"
 
 [deps]
 Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
 [compat]

stdlib/GMP_jll/src/GMP_jll.jl

@@ -2,51 +2,62 @@
 
 ## dummy stub for https://github.com/JuliaBinaryWrappers/GMP_jll.jl
 baremodule GMP_jll
-using Base, Libdl
-
-const PATH_list = String[]
-const LIBPATH_list = String[]
+using Base, Libdl, CompilerSupportLibraries_jll
 
 export libgmp, libgmpxx
 
 # These get calculated in __init__()
 const PATH = Ref("")
+const PATH_list = String[]
 const LIBPATH = Ref("")
+const LIBPATH_list = String[]
 artifact_dir::String = ""
-libgmp_handle::Ptr{Cvoid} = C_NULL
 libgmp_path::String = ""
-libgmpxx_handle::Ptr{Cvoid} = C_NULL
 libgmpxx_path::String = ""
 
 if Sys.iswindows()
-    const libgmp = "libgmp-10.dll"
-    const libgmpxx = "libgmpxx-4.dll"
+    const _libgmp_path = BundledLazyLibraryPath("libgmp-10.dll")
+    const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx-4.dll")
 elseif Sys.isapple()
-    const libgmp = "@rpath/libgmp.10.dylib"
-    const libgmpxx = "@rpath/libgmpxx.4.dylib"
+    const _libgmp_path = BundledLazyLibraryPath("libgmp.10.dylib")
+    const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx.4.dylib")
 else
-    const libgmp = "libgmp.so.10"
-    const libgmpxx = "libgmpxx.so.4"
+    const _libgmp_path = BundledLazyLibraryPath("libgmp.so.10")
+    const _libgmpxx_path = BundledLazyLibraryPath("libgmpxx.so.4")
 end
 
+const libgmp = LazyLibrary(_libgmp_path)
+
+if Sys.isapple()
+    _libgmpxx_dependencies = LazyLibrary[libgmp]
+elseif Sys.isfreebsd()
+    _libgmpxx_dependencies = LazyLibrary[]
+else
+    _libgmpxx_dependencies = LazyLibrary[libgmp, libstdcxx, libgcc_s]
+end
+const libgmpxx = LazyLibrary(
+    _libgmpxx_path,
+    dependencies=_libgmpxx_dependencies,
+)
+
+function eager_mode()
+    CompilerSupportLibraries_jll.eager_mode()
+    dlopen(libgmp)
+    dlopen(libgmpxx)
+end
+is_available() = true
+
 function __init__()
-    global libgmp_handle = dlopen(libgmp)
-    global libgmp_path = dlpath(libgmp_handle)
-    global libgmpxx_handle = dlopen(libgmpxx)
-    global libgmpxx_path = dlpath(libgmpxx_handle)
+    global libgmp_path = string(_libgmp_path)
+    global libgmpxx_path = string(_libgmpxx_path)
     global artifact_dir = dirname(Sys.BINDIR)
     LIBPATH[] = dirname(libgmp_path)
     push!(LIBPATH_list, LIBPATH[])
 end
 
-# JLLWrappers API compatibility shims.  Note that not all of these will really make sense.
-# For instance, `find_artifact_dir()` won't actually be the artifact directory, because
-# there isn't one.  It instead returns the overall Julia prefix.
-is_available() = true
-find_artifact_dir() = artifact_dir
-dev_jll() = error("stdlib JLLs cannot be dev'ed")
-best_wrapper = nothing
-get_libgmp_path() = libgmp_path
-get_libgmpxx_path() = libgmpxx_path
+if Base.generating_output()
+    precompile(eager_mode, ())
+    precompile(is_available, ())
+end
 
 end  # module GMP_jll

stdlib/GMP_jll/test/runtests.jl

@@ -3,6 +3,6 @@
 using Test, Libdl, GMP_jll
 
 @testset "GMP_jll" begin
-    vn = VersionNumber(unsafe_string(unsafe_load(cglobal((:__gmp_version, libgmp), Ptr{Cchar}))))
+    vn = VersionNumber(unsafe_string(unsafe_load(cglobal(dlsym(libgmp, :__gmp_version), Ptr{Cchar}))))
     @test vn == v"6.3.0"
 end

stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl

@@ -5,38 +5,33 @@
 baremodule LLVMLibUnwind_jll
 using Base, Libdl
 
-const PATH_list = String[]
-const LIBPATH_list = String[]
-
 export llvmlibunwind
 
 # These get calculated in __init__()
 const PATH = Ref("")
+const PATH_list = String[]
 const LIBPATH = Ref("")
+const LIBPATH_list = String[]
 artifact_dir::String = ""
-llvmlibunwind_handle::Ptr{Cvoid} = C_NULL
 llvmlibunwind_path::String = ""
 
-const llvmlibunwind = "libunwind"
+const _llvmlibunwind_path = BundledLazyLibraryPath("libunwind")
+const llvmlibunwind = LazyLibrary(_llvmlibunwind_path)
+function eager_mode()
+    dlopen(llvmlibunwind)
+end
+is_available() = @static Sys.isapple() ? true : false
 
 function __init__()
-    # We only dlopen something on MacOS
-    @static if Sys.isapple()
-        global llvmlibunwind_handle = dlopen(llvmlibunwind)
-        global llvmlibunwind_path = dlpath(llvmlibunwind_handle)
-        global artifact_dir = dirname(Sys.BINDIR)
-        LIBPATH[] = dirname(llvmlibunwind_path)
-        push!(LIBPATH_list, LIBPATH[])
-    end
+    global llvmlibunwind_path = string(_llvmlibunwind_path)
+    global artifact_dir = dirname(Sys.BINDIR)
+    LIBPATH[] = dirname(llvmlibunwind_path)
+    push!(LIBPATH_list, LIBPATH[])
 end
 
-# JLLWrappers API compatibility shims.  Note that not all of these will really make sense.
-# For instance, `find_artifact_dir()` won't actually be the artifact directory, because
-# there isn't one.  It instead returns the overall Julia prefix.
-is_available() = @static Sys.isapple() ? true : false
-find_artifact_dir() = artifact_dir
-dev_jll() = error("stdlib JLLs cannot be dev'ed")
-best_wrapper = nothing
-get_llvmlibunwind_path() = llvmlibunwind_path
+if Base.generating_output()
+    precompile(eager_mode, ())
+    precompile(is_available, ())
+end
 
 end  # module LLVMLibUnwind_jll

stdlib/LLVMLibUnwind_jll/test/runtests.jl

@@ -1,16 +1,14 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-using Test, Libdl
-using LLVMLibUnwind_jll: llvmlibunwind_handle
-
+using Test, Libdl, LLVMLibUnwind_jll
 @testset "LLVMLibUnwind_jll" begin
     if Sys.isapple()
-        @test dlsym(llvmlibunwind_handle, :unw_getcontext; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_init_local; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_init_local_dwarf; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_step; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_get_reg; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_set_reg; throw_error=false) !== nothing
-        @test dlsym(llvmlibunwind_handle, :unw_resume; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_getcontext; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_init_local; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_init_local_dwarf; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_step; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_get_reg; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_set_reg; throw_error=false) !== nothing
+        @test dlsym(llvmlibunwind, :unw_resume; throw_error=false) !== nothing
     end
 end

stdlib/LibCURL_jll/src/LibCURL_jll.jl

@@ -9,41 +9,56 @@ if !(Sys.iswindows() || Sys.isapple())
     using OpenSSL_jll
 end
 
-const PATH_list = String[]
-const LIBPATH_list = String[]
-
 export libcurl
 
 # These get calculated in __init__()
 const PATH = Ref("")
+const PATH_list = String[]
 const LIBPATH = Ref("")
+const LIBPATH_list = String[]
 artifact_dir::String = ""
-libcurl_handle::Ptr{Cvoid} = C_NULL
 libcurl_path::String = ""
 
+if Sys.isfreebsd()
+    _libcurl_dependencies = LazyLibrary[]
+else
+    _libcurl_dependencies = LazyLibrary[libz, libnghttp2, libssh2]
+    if !(Sys.iswindows() || Sys.isapple())
+        append!(_libcurl_dependencies, [libssl, libcrypto])
+    end
+end
+
 if Sys.iswindows()
-    const libcurl = "libcurl-4.dll"
+    const _libcurl_path = BundledLazyLibraryPath("libcurl-4.dll")
 elseif Sys.isapple()
-    const libcurl = "@rpath/libcurl.4.dylib"
+    const _libcurl_path = BundledLazyLibraryPath("libcurl.4.dylib")
 else
-    const libcurl = "libcurl.so.4"
+    const _libcurl_path = BundledLazyLibraryPath("libcurl.so.4")
 end
 
+const libcurl = LazyLibrary(
+    _libcurl_path,
+    dependencies=_libcurl_dependencies,
+)
+
+function eager_mode()
+    Zlib_jll.eager_mode()
+    nghttp2_jll.eager_mode()
+    LibSSH2_jll.eager_mode()
+    dlopen(libcurl)
+end
+is_available() = true
+
 function __init__()
-    global libcurl_handle = dlopen(libcurl)
-    global libcurl_path = dlpath(libcurl_handle)
+    global libcurl_path = string(_libcurl_path)
     global artifact_dir = dirname(Sys.BINDIR)
     LIBPATH[] = dirname(libcurl_path)
     push!(LIBPATH_list, LIBPATH[])
 end
 
-# JLLWrappers API compatibility shims.  Note that not all of these will really make sense.
-# For instance, `find_artifact_dir()` won't actually be the artifact directory, because
-# there isn't one.  It instead returns the overall Julia prefix.
-is_available() = true
-find_artifact_dir() = artifact_dir
-dev_jll() = error("stdlib JLLs cannot be dev'ed")
-best_wrapper = nothing
-get_libcurl_path() = libcurl_path
+if Base.generating_output()
+    precompile(eager_mode, ())
+    precompile(is_available, ())
+end
 
 end  # module LibCURL_jll

stdlib/LibGit2_jll/src/LibGit2_jll.jl

@@ -9,41 +9,52 @@ if !(Sys.iswindows() || Sys.isapple())
     using OpenSSL_jll
 end
 
-const PATH_list = String[]
-const LIBPATH_list = String[]
-
 export libgit2
 
 # These get calculated in __init__()
 const PATH = Ref("")
+const PATH_list = String[]
 const LIBPATH = Ref("")
+const LIBPATH_list = String[]
 artifact_dir::String = ""
-libgit2_handle::Ptr{Cvoid} = C_NULL
 libgit2_path::String = ""
 
 if Sys.iswindows()
-    const libgit2 = "libgit2.dll"
+    const _libgit2_path = BundledLazyLibraryPath("libgit2.dll")
 elseif Sys.isapple()
-    const libgit2 = "@rpath/libgit2.1.9.dylib"
+    const _libgit2_path = BundledLazyLibraryPath("libgit2.1.9.dylib")
+else
+    const _libgit2_path = BundledLazyLibraryPath("libgit2.so.1.9")
+end
+
+if Sys.isfreebsd()
+    _libgit2_dependencies = LazyLibrary[]
+elseif Sys.islinux()
+    _libgit2_dependencies = LazyLibrary[libssh2, libssl, libcrypto]
 else
-    const libgit2 = "libgit2.so.1.9"
+    _libgit2_dependencies = LazyLibrary[libssh2]
+end
+const libgit2 = LazyLibrary(_libgit2_path, dependencies=_libgit2_dependencies)
+
+function eager_mode()
+    LibSSH2_jll.eager_mode()
+    @static if !(Sys.iswindows() || Sys.isapple())
+        OpenSSL_jll.eager_mode()
+    end
+    dlopen(libgit2)
 end
+is_available() = true
 
 function __init__()
-    global libgit2_handle = dlopen(libgit2)
-    global libgit2_path = dlpath(libgit2_handle)
+    global libgit2_path = string(_libgit2_path)
     global artifact_dir = dirname(Sys.BINDIR)
     LIBPATH[] = dirname(libgit2_path)
     push!(LIBPATH_list, LIBPATH[])
 end
 
-# JLLWrappers API compatibility shims.  Note that not all of these will really make sense.
-# For instance, `find_artifact_dir()` won't actually be the artifact directory, because
-# there isn't one.  It instead returns the overall Julia prefix.
-is_available() = true
-find_artifact_dir() = artifact_dir
-dev_jll() = error("stdlib JLLs cannot be dev'ed")
-best_wrapper = nothing
-get_libgit2_path() = libgit2_path
+if Base.generating_output()
+    precompile(eager_mode, ())
+    precompile(is_available, ())
+end
 
 end  # module LibGit2_jll

stdlib/LibSSH2_jll/Project.toml

@@ -6,6 +6,7 @@ version = "1.11.3+1"
 OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a"
 
 [compat]
 julia = "1.8"