diff --git a/Make.inc b/Make.inc index e0bc8ba224a6d..1e1e11457ab57 100644 --- a/Make.inc +++ b/Make.inc @@ -65,6 +65,7 @@ USE_SYSTEM_LIBGIT2:=0 USE_SYSTEM_PATCHELF:=0 USE_SYSTEM_LIBWHICH:=0 USE_SYSTEM_ZLIB:=0 +USE_SYSTEM_ZSTD:=0 USE_SYSTEM_P7ZIP:=0 USE_SYSTEM_LLD:=0 @@ -213,7 +214,7 @@ JULIA_COMMIT := $(JULIA_VERSION) endif # Whether to use GPL libraries or not. -USE_GPL_LIBS ?= 1 +USE_GPL_LIBS ?= 0 # Whether to install Julia as a framework on Darwin (Apple) platforms. DARWIN_FRAMEWORK ?= 0 @@ -1201,6 +1202,14 @@ else UTF8PROC_INC := $(build_includedir) endif +ifeq ($(USE_SYSTEM_ZSTD), 1) + LIBZSTD := -lzstd + ZSTD_INC := $(LOCALBASE)/include +else + LIBZSTD := -L$(build_shlibdir) -lzstd + ZSTD_INC := $(build_includedir) +endif + # We need python for things like BB triplet recognition. We don't really care # about version, generally, so just find something that works: PYTHON := $(shell which python 2>/dev/null || which python3 2>/dev/null || which python2 2>/dev/null || echo not found) @@ -1268,7 +1277,7 @@ CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.33|GLIBCXX_3\.5\.|GLIBCXX_4\. # Note: we explicitly _do not_ define `CSL` here, since it requires some more # advanced techniques to decide whether it should be installed from a BB source # or not. See `deps/csl.mk` for more detail. -BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP LLD LIBTRACYCLIENT +BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB ZSTD P7ZIP LLD LIBTRACYCLIENT define SET_BB_DEFAULT # First, check to see if BB is disabled on a global setting ifeq ($$(USE_BINARYBUILDER),0) @@ -1397,7 +1406,7 @@ JLDFLAGS += -Wl,--stack,8388608 --disable-auto-import --disable-runtime-pseudo-r ifeq ($(ARCH),i686) JLDFLAGS += -Wl,--large-address-aware endif -JCPPFLAGS += -D_WIN32_WINNT=0x0502 +JCPPFLAGS += -D_WIN32_WINNT=_WIN32_WINNT_WIN8 UNTRUSTED_SYSTEM_LIBM := 1 # Use hard links for files on windows, rather than soft links # https://stackoverflow.com/questions/3648819/how-to-make-a-symbolic-link-with-cygwin-in-windows-7 diff --git a/Makefile b/Makefile index 05683fda0a004..f96671f8d1cd2 100644 --- a/Makefile +++ b/Makefile @@ -206,8 +206,13 @@ JL_PRIVATE_LIBS-0 += libjulia-internal libjulia-codegen else ifeq ($(JULIA_BUILD_MODE),debug) JL_PRIVATE_LIBS-0 += libjulia-internal-debug libjulia-codegen-debug endif +# BSD-3-Clause +JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSUITESPARSE) += libamd libcamd libccolamd libcolamd libsuitesparseconfig +# LGPL-2.1+ +JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSUITESPARSE) += libbtf libklu libldl ifeq ($(USE_GPL_LIBS), 1) -JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSUITESPARSE) += libamd libbtf libcamd libccolamd libcholmod libcolamd libklu libldl librbio libspqr libsuitesparseconfig libumfpack +# GPL-2.0+ +JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSUITESPARSE) += libcholmod librbio libspqr libumfpack endif JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBBLASTRAMPOLINE) += libblastrampoline JL_PRIVATE_LIBS-$(USE_SYSTEM_PCRE) += libpcre2-8 @@ -225,6 +230,7 @@ JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += zlib else JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libz endif +JL_PRIVATE_LIBS-$(USE_SYSTEM_ZSTD) += libzstd ifeq ($(USE_LLVM_SHLIB),1) JL_PRIVATE_LIBS-$(USE_SYSTEM_LLVM) += libLLVM $(LLVM_SHARED_LIB_NAME) endif diff --git a/THIRDPARTY.md b/THIRDPARTY.md index 412b84b688758..ba0a4b9379eb9 100644 --- a/THIRDPARTY.md +++ b/THIRDPARTY.md @@ -41,7 +41,19 @@ Julia's `stdlib` uses the following external libraries, which have their own lic - [OPENBLAS](https://raw.github.com/xianyi/OpenBLAS/master/LICENSE) [BSD-3] - [LAPACK](https://netlib.org/lapack/LICENSE.txt) [BSD-3] - [PCRE](https://www.pcre.org/licence.txt) [BSD-3] -- [SUITESPARSE](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/master/LICENSE.txt) [mix of LGPL2+ and GPL2+; see individual module licenses] +- [SUITESPARSE](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/master/LICENSE.txt) [mix of BSD-3-Clause, LGPL2.1+ and GPL2+; see individual module licenses] + - [`libamd`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/AMD/Doc/License.txt) [BSD-3-Clause] + - [`libcamd`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/CAMD/Doc/License.txt) [BSD-3-Clause] + - [`libccolamd`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/CCOLAMD/Doc/License.txt) [BSD-3-Clause] + - [`libcolamd`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/COLAMD/Doc/License.txt) [BSD-3-Clause] + - [`libsuitesparseconfig`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/SuiteSparse_config/README.txt) [BSD-3-Clause] + - [`libbtf`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/BTF/Doc/License.txt) [LGPL-2.1+] + - [`libklu`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/KLU/Doc/License.txt) [LGPL-2.1+] + - [`libldl`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/LDL/Doc/License.txt) [LGPL-2.1+] + - [`libcholmod`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/CHOLMOD/Doc/License.txt) [LGPL-2.1+ and GPL-2.0+] + - [`librbio`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/RBio/Doc/License.txt) [GPL-2.0+] + - [`libspqr`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/SPQR/Doc/License.txt) [GPL-2.0+] + - [`libumfpack`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/UMFPACK/Doc/License.txt) [GPL-2.0+] - [LIBBLASTRAMPOLINE](https://github.com/staticfloat/libblastrampoline/blob/main/LICENSE) [MIT] - [NGHTTP2](https://github.com/nghttp2/nghttp2/blob/master/COPYING) [MIT] @@ -55,6 +67,7 @@ Julia bundles the following external programs and libraries: - [7-Zip](https://www.7-zip.org/license.txt) - [ZLIB](https://zlib.net/zlib_license.html) +- [ZSTD](https://github.com/facebook/zstd/blob/v1.5.7/LICENSE) On some platforms, distributions of Julia contain SSL certificate authority certificates, released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). diff --git a/base/options.jl b/base/options.jl index 296bd02111f38..c09f0ec6f246b 100644 --- a/base/options.jl +++ b/base/options.jl @@ -58,6 +58,7 @@ struct JLOptions strip_ir::Int8 permalloc_pkgimg::Int8 heap_size_hint::UInt64 + compress_sysimage::Int8 end # This runs early in the sysimage != is not defined yet diff --git a/base/util.jl b/base/util.jl index 3a621211162ec..f233e6f1d234d 100644 --- a/base/util.jl +++ b/base/util.jl @@ -245,6 +245,9 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Unio if opts.use_sysimage_native_code == 0 push!(addflags, "--sysimage-native-code=no") end + if opts.compress_sysimage == 1 + push!(addflags, "--compress-sysimage=yes") + end return `$julia -C $cpu_target -J$image_file $addflags` end diff --git a/contrib/refresh_checksums.mk b/contrib/refresh_checksums.mk index f67088141ccd4..506f524062538 100644 --- a/contrib/refresh_checksums.mk +++ b/contrib/refresh_checksums.mk @@ -24,7 +24,7 @@ CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS)) NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS)) # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded: -BB_PROJECTS=mbedtls libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient +BB_PROJECTS=mbedtls libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib zstd libsuitesparse openlibm blastrampoline libtracyclient BB_GCC_EXPANDED_PROJECTS=openblas csl BB_CXX_EXPANDED_PROJECTS=gmp llvm clang llvm-tools lld # These are non-BB source-only deps diff --git a/deps/Makefile b/deps/Makefile index 6a1b03eec919e..403c09aa50072 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -70,10 +70,12 @@ endif endif endif +PATCHELF_MANIFEST := ifneq (,$(findstring $(OS),Linux FreeBSD)) ifeq ($(USE_SYSTEM_PATCHELF), 0) DEP_LIBS += patchelf PATCHELF:=$(build_depsbindir)/patchelf +PATCHELF_MANIFEST:=$(build_prefix)/manifest/patchelf else PATCHELF:=patchelf endif @@ -141,11 +143,11 @@ ifeq ($(USE_SYSTEM_MPFR), 0) DEP_LIBS += mpfr endif -ifeq ($(USE_GPL_LIBS), 1) +# Only some of the modules in SuiteSparse are GPL. +# xref: `remove-libsuitesparse-gpl-lib` in libsuitesparse.mk ifeq ($(USE_SYSTEM_LIBSUITESPARSE), 0) DEP_LIBS += libsuitesparse endif -endif ifeq ($(USE_SYSTEM_UTF8PROC), 0) DEP_LIBS += utf8proc @@ -155,6 +157,10 @@ ifeq ($(USE_SYSTEM_ZLIB), 0) DEP_LIBS += zlib endif +ifeq ($(USE_SYSTEM_ZSTD), 0) +DEP_LIBS += zstd +endif + ifeq ($(USE_SYSTEM_P7ZIP), 0) DEP_LIBS += p7zip endif @@ -195,7 +201,7 @@ DEP_LIBS_STAGED := $(DEP_LIBS) # list all targets DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \ openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \ - objconv mbedtls libssh2 nghttp2 curl libgit2 libwhich zlib p7zip csl \ + objconv mbedtls libssh2 nghttp2 curl libgit2 libwhich zlib zstd p7zip csl \ sanitizers libsuitesparse lld libtracyclient ittapi JuliaSyntax terminfo DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL) @@ -244,6 +250,7 @@ include $(SRCDIR)/openblas.mk include $(SRCDIR)/utf8proc.mk include $(SRCDIR)/libsuitesparse.mk include $(SRCDIR)/zlib.mk +include $(SRCDIR)/zstd.mk include $(SRCDIR)/unwind.mk include $(SRCDIR)/gmp.mk include $(SRCDIR)/mpfr.mk diff --git a/deps/checksums/zstd b/deps/checksums/zstd new file mode 100644 index 0000000000000..aea151b266966 --- /dev/null +++ b/deps/checksums/zstd @@ -0,0 +1,38 @@ +Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/md5/d6b2fb32d705078dbc369986ac8b056b +Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/sha512/5dfcf36087ce8540b1f6a04181adee962e2164a763e758ac5cc256c332756774b381ca58e26641a15ce555d59641690a6da72a67bf935d8611734f2006bde504 +Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/md5/0c627ec83e426383c25eb4bc297f3548 +Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/sha512/1fdcf77e877f0676fc26a05e0cc20a1d6e1df731d81e0bba9a5657131116bbea75da4d38953969d8d07dce0bf2d7654075dbb285ebe5f4588c446e88774336c8 +Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/md5/cc9ada74a19db50d7dd6edd05866c902 +Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/sha512/0b33c0df144bb1e95290685f01695b26da834a70a365c0362314cb001ba611962a0876bc5baac31f19c80bcb110e030fb9840a56761b4d29a7893ca65f95b111 +Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/md5/5daa5b2bf2b856c448feaa8329d0de1b +Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/sha512/b39d025463b4bf21295fd5bbff91ba501506b3480363cdcfe6dd2f11d2e0afaf130f6c74d962e503fccb7a55bfcad0504ebb19f18b6b5c8b8103e7b9919df536 +Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/md5/f4218e8b4f8d415df49aeba9d43f0ba0 +Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/sha512/878d4f90160c6b0c341c61ecafbf5f5cb89c73db3175f272adc666bc25c88b127145d78946bc0fcb992489b54fbb48089bfcacf768397fc5d54d7cae4aeae9f9 +Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/md5/3c2e132ca47e6d1d23c149fdde9d8bd5 +Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/sha512/3745d99c9ca0ce9f98ff9393e405e8b382d05573a972067d57e800e282a9544fff7bc3d49b91eccc98d7736acdc3faa4c637911d79fab10f5a691d33ae775574 +Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/md5/926d765281bef388ecc25d04cbb66102 +Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/sha512/2d2c14587e2e7b2b147cb6423720cc30ed6aa57ed07372a1aa54e7f2e6badb5aa640b116e83371561d6f8f3a1b3f7fff7f6df137f8c7be788ee889bb30273eae +Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/md5/c25420561ce254e57d74e30c88fc53dd +Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/sha512/2f924e2089589057e8713d04db9a1cb2f2d571ad9e7eeda3b7f898c9a75f8fecf0647f2185d3c01fc3b399d3662ff3b1acb13429c8a953f0394a3ed9ca30b877 +Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/md5/3314bf1b52f2295555fb4ae44b1d9331 +Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/sha512/91502910a0c9b786d91499477fee2445b8f6de6bcb71af7d79c738ea2430c67cb1957866383ee3921ed1a23c53a80be19aea6abcf0e76056ffee69583728c3ed +Zstd.v1.5.7+1.i686-linux-musl.tar.gz/md5/845eddc06527a4c4b196666f7ac64ba3 +Zstd.v1.5.7+1.i686-linux-musl.tar.gz/sha512/bb15b4327cef32be38c2fd68afedb3245c7db881ad66d3ece2198ff3034be9c12efa3d62bcba2b8e6056e7d8cb5f1b3e33726f7d1e1bead235c38f8fa985b557 +Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/md5/9bc0b3c951f5e66393fd5433bf60a2c8 +Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/sha512/550b0189097e569f98404aa836b76a5cbdc36428292214c4af8916dea2713440cf3ba94125b3e5fa0c65b2bcb916733094fdef906ad19f923d90dabfc961c75a +Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/md5/468d930de7a27af961996e7c6ed35298 +Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/sha512/d680715b1ac9ff07d5662c499fbab67757509599335f861158b9dba32fe9b22da6e52d0db6b402dd4542799621ad3dccf254dfd9d3c8748bbd22f7446681539a +Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/md5/b93fef8db2b0b4417f7836d73c5fbe86 +Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/sha512/9f3ee42c7952aba2d2c26252f058bb7ab96828fafc978c9273b500ef15ccd271c51399d4b93eebd4c832b087ab5ed8a4847104ce9c83c9483aaa13c22df681bb +Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/md5/29a260789fae6f6b6df0e5cebdafd615 +Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/sha512/015045a1b7a477504057cb4c87428d42386218e48af38f83739dbe6b93961ca2c8dd4d794377a2d54b8cc284f5a467e3358d4f534cf8bcbcad886ef8cea038e9 +Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/md5/06656befb6ef9a8cc7f56e7152c2acc5 +Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/sha512/16aea0d95432a87d21d9a6f55d84e45df85caf1fda77c75b7e9a8bba519605168585f21a812773ddf1075d9bad68412e63b8cad1a143420e25ae4405bb41842e +Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/md5/da13dd1cc0d20ba9a06e9e79a588cda4 +Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/sha512/cd4218fa92dcf8772390788d5654ca12132af7829fb0ada016f3c663e2045e29e7d7587f2f5a4f057020cacca17c188c8537f284b1456100d57e84bb47c40e77 +Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/md5/bce5f37e53e330bfe4df4a28cf5c223b +Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/sha512/8f6bd7664efea537ac7815db0604ca1a07bcfb71b5152c22dc7f0a11b57643f059c341fa71d315407e2333e4c97e43e214471c73eed8b977680785302c7c2b3e +Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/md5/7cf3a740fa174004b94125e8754f4a19 +Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/sha512/faac37ad4dacb0f083364c593cd3bd1c0b592947341a631bd2fbc4081361d97ef89482f4459c46ad37ae030aa900c62305a8525e64a2ad8e91204d76dda89db1 +zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/md5/a679d9aa86549b5851100ac5d4044c68 +zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/sha512/27c6fff165abea694d91311a6657a939433ba1d707147ed9072b5e4ecce259b929970306788e0c3e95db38ce85e894e5025936b1faa81cf67741b8464e24fc4e diff --git a/deps/libsuitesparse.mk b/deps/libsuitesparse.mk index 7e36bce8f4f9d..006d1ea51f2c5 100644 --- a/deps/libsuitesparse.mk +++ b/deps/libsuitesparse.mk @@ -95,7 +95,7 @@ configure-libsuitesparse: extract-libsuitesparse compile-libsuitesparse: $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/build-compiled fastcheck-libsuitesparse: #none check-libsuitesparse: $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/build-checked -install-libsuitesparse: $(build_prefix)/manifest/libsuitesparse +install-libsuitesparse: $(build_prefix)/manifest/libsuitesparse remove-libsuitesparse-gpl-lib else # USE_BINARYBUILDER_LIBSUITESPARSE @@ -103,6 +103,7 @@ $(eval $(call bb-install,libsuitesparse,LIBSUITESPARSE,false)) # libsuitesparse depends on blastrampoline compile-libsuitesparse: | $(build_prefix)/manifest/blastrampoline +install-libsuitesparse: | remove-libsuitesparse-gpl-lib endif define manual_libsuitesparse @@ -110,3 +111,13 @@ uninstall-libsuitesparse: -rm -f $(build_prefix)/manifest/libsuitesparse -rm -f $(addprefix $(build_shlibdir)/lib,$3) endef + +remove-libsuitesparse-gpl-lib: +ifeq ($(USE_GPL_LIBS),0) + @echo Removing GPL libs... + -rm -f $(build_bindir)/libcholmod* + -rm -f $(build_bindir)/libklu_cholmod* + -rm -f $(build_bindir)/librbio* + -rm -f $(build_bindir)/libspqr* + -rm -f $(build_bindir)/libumfpack* +endif diff --git a/deps/zstd.mk b/deps/zstd.mk new file mode 100644 index 0000000000000..5ead77641858a --- /dev/null +++ b/deps/zstd.mk @@ -0,0 +1,60 @@ +## Zstd ## +ifneq ($(USE_BINARYBUILDER_ZSTD), 1) +ZSTD_GIT_URL := https://github.com/facebook/zstd.git +ZSTD_TAR_URL = https://api.github.com/repos/facebook/zstd/tarball/$1 +$(eval $(call git-external,zstd,ZSTD,,,$(BUILDDIR))) + +ZSTD_BUILD_OPTS := MOREFLAGS="-DZSTD_MULTITHREAD $(fPIC)" bindir=$(build_private_libexecdir) + +$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted + echo 1 > $@ + +$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured + $(MAKE) -C $(dir $<) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) + echo 1 > $@ + +$(eval $(call staged-install, \ + zstd,$(ZSTD_SRC_DIR), \ + MAKE_INSTALL,$(ZSTD_BUILD_OPTS) MT=1,, \ + $(INSTALL_NAME_CMD)libzstd.$(SHLIB_EXT) $(build_private_libexecdir)/libzstd.$(SHLIB_EXT))) + +clean-zstd: + -rm -f $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled + -$(MAKE) -C $(BUILDDIR)/$(ZSTD_SRC_DIR) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) clean + +get-zstd: $(ZSTD_SRC_FILE) +extract-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted +configure-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured +compile-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled +fastcheck-zstd: check-zstd +check-zstd: compile-zstd + +else # USE_BINARYBUILDER_ZSTD + +$(eval $(call bb-install,zstd,ZSTD,false)) +# move from bindir to shlibdir, where we expect to install it +install-zstd: post-install-zstd +uninstall-zstd: pre-uninstall-zstd +post-install-zstd: $(build_prefix)/manifest/zstd $(PATCHELF_MANIFEST) + mkdir -p $(build_private_libexecdir)/ + [ ! -e $(build_bindir)/zstdmt$(EXE) ] || mv $(build_bindir)/zstdmt$(EXE) $(build_private_libexecdir)/zstdmt$(EXE) + [ ! -e $(build_bindir)/zstd$(EXE) ] || mv $(build_bindir)/zstd$(EXE) $(build_private_libexecdir)/zstd$(EXE) + [ -e $(build_private_libexecdir)/zstd$(EXE) ] + [ -e $(build_private_libexecdir)/zstdmt$(EXE) ] +ifeq ($(OS), Darwin) + for j in zstd zstdmt ; do \ + [ -L $(build_private_libexecdir)/$$j ] && continue; \ + install_name_tool -rpath @executable_path/$(reverse_build_private_libexecdir_rel) @loader_path/$(build_libdir_rel) $(build_private_libexecdir)/$$j 2>/dev/null || true; \ + install_name_tool -rpath @loader_path/$(build_libdir_rel) @executable_path/$(reverse_build_private_libexecdir_rel) $(build_private_libexecdir)/$$j || exit 1; \ + done +else ifneq (,$(findstring $(OS),Linux FreeBSD)) + for j in zstd zstdmt ; do \ + [ -L $(build_private_libexecdir)/$$j ] && continue; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_build_private_libexecdir_rel)' $(build_private_libexecdir)/$$j || exit 1; \ + done +endif + +pre-uninstall-zstd: + -rm -f $(build_private_libexecdir)/zstd$(EXE) $(build_private_libexecdir)/zstdmt$(EXE) + +endif # USE_BINARYBUILDER_ZSTD diff --git a/deps/zstd.version b/deps/zstd.version new file mode 100644 index 0000000000000..d4d960aa6f04b --- /dev/null +++ b/deps/zstd.version @@ -0,0 +1,8 @@ +# -*- makefile -*- +## jll artifact +ZSTD_JLL_NAME := Zstd + +## source build +ZSTD_VER := 1.5.7 +ZSTD_BRANCH=v1.5.7 +ZSTD_SHA1=f8745da6ff1ad1e7bab384bd1f9d742439278e99 diff --git a/doc/Manifest.toml b/doc/Manifest.toml index 76bdc332ff36f..f1d4283eff7b9 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -47,7 +47,7 @@ version = "0.9.3" deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] git-tree-sha1 = "d0ea2c044963ed6f37703cead7e29f70cba13d7e" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.8.0" +version = "1.8.1" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] diff --git a/doc/make.jl b/doc/make.jl index e8ccbad85c468..bac4adfd736e9 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -27,6 +27,9 @@ cd(joinpath(@__DIR__, "src")) do mkdir("stdlib") for dir in readdir(STDLIB_DIR) sourcefile = joinpath(STDLIB_DIR, dir, "docs", "src") + if dir == "SparseArrays" + continue + end if dir in EXT_STDLIB_DOCS sourcefile = joinpath(sourcefile, "basedocs.md") else @@ -140,7 +143,6 @@ Manual = [ "manual/methods.md", "manual/constructors.md", "manual/conversion-and-promotion.md", - "manual/interfaces.md", "manual/modules.md", "manual/documentation.md", "manual/metaprogramming.md", @@ -306,12 +308,12 @@ for stdlib in STDLIB_DOCS end # A few standard libraries need more than just the module itself in the DocTestSetup. # This overwrites the existing ones from above though, hence the warn=false. -DocMeta.setdocmeta!( - SparseArrays, - :DocTestSetup, - maybe_revise(:(using SparseArrays, LinearAlgebra)); - recursive=true, warn=false, -) +# DocMeta.setdocmeta!( +# SparseArrays, +# :DocTestSetup, +# maybe_revise(:(using SparseArrays, LinearAlgebra)); +# recursive=true, warn=false, +# ) DocMeta.setdocmeta!( UUIDs, :DocTestSetup, @@ -376,6 +378,7 @@ makedocs( sitename = "The Julia Language", authors = "The Julia Project", pages = PAGES, + warnonly = Documenter.except(:cross_references), remotes = documenter_stdlib_remotes, ) diff --git a/doc/src/manual/interfaces.md b/doc/src/manual/interfaces.md deleted file mode 100644 index d158fb86575a2..0000000000000 --- a/doc/src/manual/interfaces.md +++ /dev/null @@ -1,886 +0,0 @@ -# Interfaces - -A lot of the power and extensibility in Julia comes from a collection of informal interfaces. - By extending a few specific methods to work for a custom type, objects of that type not only -receive those functionalities, but they are also able to be used in other methods that are written -to generically build upon those behaviors. - -## [Iteration](@id man-interface-iteration) - -There are two methods that are always required: - -| Required method | Brief description | -|:----------------------- |:---------------------------------------------------------------------------------------- | -| [`iterate(iter)`](@ref) | Returns either a tuple of the first item and initial state or [`nothing`](@ref) if empty | -| `iterate(iter, state)` | Returns either a tuple of the next item and next state or `nothing` if no items remain | - -There are several more methods that should be defined in some circumstances. -Please note that you should always define at least one of `Base.IteratorSize(IterType)` and `length(iter)` because the default definition of `Base.IteratorSize(IterType)` is `Base.HasLength()`. - -| Method | When should this method be defined? | Default definition | Brief description | -|:--- |:--- |:--- |:--- | -| [`Base.IteratorSize(IterType)`](@ref) | If default is not appropriate | `Base.HasLength()` | One of `Base.HasLength()`, `Base.HasShape{N}()`, `Base.IsInfinite()`, or `Base.SizeUnknown()` as appropriate | -| [`length(iter)`](@ref) | If `Base.IteratorSize()` returns `Base.HasLength()` or `Base.HasShape{N}()` | (*undefined*) | The number of items, if known | -| [`size(iter, [dim])`](@ref) | If `Base.IteratorSize()` returns `Base.HasShape{N}()` | (*undefined*) | The number of items in each dimension, if known | -| [`Base.IteratorEltype(IterType)`](@ref) | If default is not appropriate | `Base.HasEltype()` | Either `Base.EltypeUnknown()` or `Base.HasEltype()` as appropriate | -| [`eltype(IterType)`](@ref) | If default is not appropriate | `Any` | The type of the first entry of the tuple returned by `iterate()` | -| [`Base.isdone(iter, [state])`](@ref) | **Must** be defined if iterator is stateful | `missing` | Fast-path hint for iterator completion. If not defined for a stateful iterator then functions that check for done-ness, like `isempty()` and `zip()`, may mutate the iterator and cause buggy behaviour! | - -Sequential iteration is implemented by the [`iterate`](@ref) function. Instead -of mutating objects as they are iterated over, Julia iterators may keep track -of the iteration state externally from the object. The return value from iterate -is always either a tuple of a value and a state, or `nothing` if no elements remain. -The state object will be passed back to the iterate function on the next iteration -and is generally considered an implementation detail private to the iterable object. - -Any object that defines this function is iterable and can be used in the [many functions that rely upon iteration](@ref lib-collections-iteration). -It can also be used directly in a [`for`](@ref) loop since the syntax: - -```julia -for item in iter # or "for item = iter" - # body -end -``` - -is translated into: - -```julia -next = iterate(iter) -while next !== nothing - (item, state) = next - # body - next = iterate(iter, state) -end -``` - -A simple example is an iterable sequence of square numbers with a defined length: - -```jldoctest squaretype -julia> struct Squares - count::Int - end - -julia> Base.iterate(S::Squares, state=1) = state > S.count ? nothing : (state*state, state+1) -``` - -With only [`iterate`](@ref) definition, the `Squares` type is already pretty powerful. -We can iterate over all the elements: - -```jldoctest squaretype -julia> for item in Squares(7) - println(item) - end -1 -4 -9 -16 -25 -36 -49 -``` - -We can use many of the builtin methods that work with iterables, -like [`in`](@ref) or [`sum`](@ref): - -```jldoctest squaretype -julia> 25 in Squares(10) -true - -julia> sum(Squares(100)) -338350 -``` - -There are a few more methods we can extend to give Julia more information about this iterable -collection. We know that the elements in a `Squares` sequence will always be `Int`. By extending -the [`eltype`](@ref) method, we can give that information to Julia and help it make more specialized -code in the more complicated methods. We also know the number of elements in our sequence, so -we can extend [`length`](@ref), too: - -```jldoctest squaretype -julia> Base.eltype(::Type{Squares}) = Int # Note that this is defined for the type - -julia> Base.length(S::Squares) = S.count -``` - -Now, when we ask Julia to [`collect`](@ref) all the elements into an array it can preallocate a `Vector{Int}` -of the right size instead of naively [`push!`](@ref)ing each element into a `Vector{Any}`: - -```jldoctest squaretype -julia> collect(Squares(4)) -4-element Vector{Int64}: - 1 - 4 - 9 - 16 -``` - -While we can rely upon generic implementations, we can also extend specific methods where we know -there is a simpler algorithm. For example, there's a formula to compute the sum of squares, so -we can override the generic iterative version with a more performant solution: - -```jldoctest squaretype -julia> Base.sum(S::Squares) = (n = S.count; return n*(n+1)*(2n+1)÷6) - -julia> sum(Squares(1803)) -1955361914 -``` - -This is a very common pattern throughout Julia Base: a small set of required methods -define an informal interface that enable many fancier behaviors. In some cases, types will want -to additionally specialize those extra behaviors when they know a more efficient algorithm can -be used in their specific case. - -It is also often useful to allow iteration over a collection in *reverse order* -by iterating over [`Iterators.reverse(iterator)`](@ref). To actually support -reverse-order iteration, however, an iterator -type `T` needs to implement `iterate` for `Iterators.Reverse{T}`. -(Given `r::Iterators.Reverse{T}`, the underling iterator of type `T` is `r.itr`.) -In our `Squares` example, we would implement `Iterators.Reverse{Squares}` methods: - -```jldoctest squaretype -julia> Base.iterate(rS::Iterators.Reverse{Squares}, state=rS.itr.count) = state < 1 ? nothing : (state*state, state-1) - -julia> collect(Iterators.reverse(Squares(4))) -4-element Vector{Int64}: - 16 - 9 - 4 - 1 -``` - -## Indexing - -| Methods to implement | Brief description | -|:-------------------- |:-------------------------------- | -| `getindex(X, i)` | `X[i]`, indexed access, non-scalar `i` should allocate a copy | -| `setindex!(X, v, i)` | `X[i] = v`, indexed assignment | -| `firstindex(X)` | The first index, used in `X[begin]` | -| `lastindex(X)` | The last index, used in `X[end]` | - -For the `Squares` iterable above, we can easily compute the `i`th element of the sequence by squaring -it. We can expose this as an indexing expression `S[i]`. To opt into this behavior, `Squares` -simply needs to define [`getindex`](@ref): - -```jldoctest squaretype -julia> function Base.getindex(S::Squares, i::Int) - 1 <= i <= S.count || throw(BoundsError(S, i)) - return i*i - end - -julia> Squares(100)[23] -529 -``` - -Additionally, to support the syntax `S[begin]` and `S[end]`, we must define [`firstindex`](@ref) and -[`lastindex`](@ref) to specify the first and last valid indices, respectively: - -```jldoctest squaretype -julia> Base.firstindex(S::Squares) = 1 - -julia> Base.lastindex(S::Squares) = length(S) - -julia> Squares(23)[end] -529 -``` - -For multi-dimensional `begin`/`end` indexing as in `a[3, begin, 7]`, for example, -you should define `firstindex(a, dim)` and `lastindex(a, dim)` -(which default to calling `first` and `last` on `axes(a, dim)`, respectively). - -Note, though, that the above *only* defines [`getindex`](@ref) with one integer index. Indexing with -anything other than an `Int` will throw a [`MethodError`](@ref) saying that there was no matching method. -In order to support indexing with ranges or vectors of `Int`s, separate methods must be written: - -```jldoctest squaretype -julia> Base.getindex(S::Squares, i::Number) = S[convert(Int, i)] - -julia> Base.getindex(S::Squares, I) = [S[i] for i in I] - -julia> Squares(10)[[3,4.,5]] -3-element Vector{Int64}: - 9 - 16 - 25 -``` - -While this is starting to support more of the [indexing operations supported by some of the builtin types](@ref man-array-indexing), -there's still quite a number of behaviors missing. This `Squares` sequence is starting to look -more and more like a vector as we've added behaviors to it. Instead of defining all these behaviors -ourselves, we can officially define it as a subtype of an [`AbstractArray`](@ref). - -## [Abstract Arrays](@id man-interface-array) - -| Methods to implement | | Brief description | -|:----------------------------------------------- |:-------------------------------------- |:------------------------------------------------------------------------------------- | -| `size(A)` | | Returns a tuple containing the dimensions of `A` | -| `getindex(A, i::Int)` | | (if `IndexLinear`) Linear scalar indexing | -| `getindex(A, I::Vararg{Int, N})` | | (if `IndexCartesian`, where `N = ndims(A)`) N-dimensional scalar indexing | -| **Optional methods** | **Default definition** | **Brief description** | -| `IndexStyle(::Type)` | `IndexCartesian()` | Returns either `IndexLinear()` or `IndexCartesian()`. See the description below. | -| `setindex!(A, v, i::Int)` | | (if `IndexLinear`) Scalar indexed assignment | -| `setindex!(A, v, I::Vararg{Int, N})` | | (if `IndexCartesian`, where `N = ndims(A)`) N-dimensional scalar indexed assignment | -| `getindex(A, I...)` | defined in terms of scalar `getindex` | [Multidimensional and nonscalar indexing](@ref man-array-indexing) | -| `setindex!(A, X, I...)` | defined in terms of scalar `setindex!` | [Multidimensional and nonscalar indexed assignment](@ref man-array-indexing) | -| `iterate` | defined in terms of scalar `getindex` | Iteration | -| `length(A)` | `prod(size(A))` | Number of elements | -| `similar(A)` | `similar(A, eltype(A), size(A))` | Return a mutable array with the same shape and element type | -| `similar(A, ::Type{S})` | `similar(A, S, size(A))` | Return a mutable array with the same shape and the specified element type | -| `similar(A, dims::Dims)` | `similar(A, eltype(A), dims)` | Return a mutable array with the same element type and size *dims* | -| `similar(A, ::Type{S}, dims::Dims)` | `Array{S}(undef, dims)` | Return a mutable array with the specified element type and size | -| **Non-traditional indices** | **Default definition** | **Brief description** | -| `axes(A)` | `map(OneTo, size(A))` | Return a tuple of `AbstractUnitRange{<:Integer}` of valid indices. The axes should be their own axes, that is `axes.(axes(A),1) == axes(A)` should be satisfied. | -| `similar(A, ::Type{S}, inds)` | `similar(A, S, Base.to_shape(inds))` | Return a mutable array with the specified indices `inds` (see below) | -| `similar(T::Union{Type,Function}, inds)` | `T(Base.to_shape(inds))` | Return an array similar to `T` with the specified indices `inds` (see below) | - -If a type is defined as a subtype of `AbstractArray`, it inherits a very large set of rich behaviors -including iteration and multidimensional indexing built on top of single-element access. See -the [arrays manual page](@ref man-multi-dim-arrays) and the [Julia Base section](@ref lib-arrays) for more supported methods. - -A key part in defining an `AbstractArray` subtype is [`IndexStyle`](@ref). Since indexing is -such an important part of an array and often occurs in hot loops, it's important to make both -indexing and indexed assignment as efficient as possible. Array data structures are typically -defined in one of two ways: either it most efficiently accesses its elements using just one index -(linear indexing) or it intrinsically accesses the elements with indices specified for every dimension. - These two modalities are identified by Julia as `IndexLinear()` and `IndexCartesian()`. - Converting a linear index to multiple indexing subscripts is typically very expensive, so this -provides a traits-based mechanism to enable efficient generic code for all array types. - -This distinction determines which scalar indexing methods the type must define. `IndexLinear()` -arrays are simple: just define `getindex(A::ArrayType, i::Int)`. When the array is subsequently -indexed with a multidimensional set of indices, the fallback `getindex(A::AbstractArray, I...)` -efficiently converts the indices into one linear index and then calls the above method. `IndexCartesian()` -arrays, on the other hand, require methods to be defined for each supported dimensionality with -`ndims(A)` `Int` indices. For example, [`SparseMatrixCSC`](@ref) from the `SparseArrays` standard -library module, only supports two dimensions, so it just defines -`getindex(A::SparseMatrixCSC, i::Int, j::Int)`. The same holds for [`setindex!`](@ref). - -Returning to the sequence of squares from above, we could instead define it as a subtype of an -`AbstractArray{Int, 1}`: - -```jldoctest squarevectype -julia> struct SquaresVector <: AbstractArray{Int, 1} - count::Int - end - -julia> Base.size(S::SquaresVector) = (S.count,) - -julia> Base.IndexStyle(::Type{<:SquaresVector}) = IndexLinear() - -julia> Base.getindex(S::SquaresVector, i::Int) = i*i -``` - -Note that it's very important to specify the two parameters of the `AbstractArray`; the first -defines the [`eltype`](@ref), and the second defines the [`ndims`](@ref). That supertype and those three -methods are all it takes for `SquaresVector` to be an iterable, indexable, and completely functional -array: - -```jldoctest squarevectype -julia> s = SquaresVector(4) -4-element SquaresVector: - 1 - 4 - 9 - 16 - -julia> s[s .> 8] -2-element Vector{Int64}: - 9 - 16 - -julia> s + s -4-element Vector{Int64}: - 2 - 8 - 18 - 32 - -julia> sin.(s) -4-element Vector{Float64}: - 0.8414709848078965 - -0.7568024953079282 - 0.4121184852417566 - -0.2879033166650653 -``` - -As a more complicated example, let's define our own toy N-dimensional sparse-like array type built -on top of [`Dict`](@ref): - -```jldoctest squarevectype -julia> struct SparseArray{T,N} <: AbstractArray{T,N} - data::Dict{NTuple{N,Int}, T} - dims::NTuple{N,Int} - end - -julia> SparseArray(::Type{T}, dims::Int...) where {T} = SparseArray(T, dims); - -julia> SparseArray(::Type{T}, dims::NTuple{N,Int}) where {T,N} = SparseArray{T,N}(Dict{NTuple{N,Int}, T}(), dims); - -julia> Base.size(A::SparseArray) = A.dims - -julia> Base.similar(A::SparseArray, ::Type{T}, dims::Dims) where {T} = SparseArray(T, dims) - -julia> Base.getindex(A::SparseArray{T,N}, I::Vararg{Int,N}) where {T,N} = get(A.data, I, zero(T)) - -julia> Base.setindex!(A::SparseArray{T,N}, v, I::Vararg{Int,N}) where {T,N} = (A.data[I] = v) -``` - -Notice that this is an `IndexCartesian` array, so we must manually define [`getindex`](@ref) and [`setindex!`](@ref) -at the dimensionality of the array. Unlike the `SquaresVector`, we are able to define [`setindex!`](@ref), -and so we can mutate the array: - -```jldoctest squarevectype -julia> A = SparseArray(Float64, 3, 3) -3×3 SparseArray{Float64, 2}: - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - -julia> fill!(A, 2) -3×3 SparseArray{Float64, 2}: - 2.0 2.0 2.0 - 2.0 2.0 2.0 - 2.0 2.0 2.0 - -julia> A[:] = 1:length(A); A -3×3 SparseArray{Float64, 2}: - 1.0 4.0 7.0 - 2.0 5.0 8.0 - 3.0 6.0 9.0 -``` - -The result of indexing an `AbstractArray` can itself be an array (for instance when indexing by -an `AbstractRange`). The `AbstractArray` fallback methods use [`similar`](@ref) to allocate an `Array` -of the appropriate size and element type, which is filled in using the basic indexing method described -above. However, when implementing an array wrapper you often want the result to be wrapped as -well: - -```jldoctest squarevectype -julia> A[1:2,:] -2×3 SparseArray{Float64, 2}: - 1.0 4.0 7.0 - 2.0 5.0 8.0 -``` - -In this example it is accomplished by defining `Base.similar(A::SparseArray, ::Type{T}, dims::Dims) where T` -to create the appropriate wrapped array. (Note that while `similar` supports 1- and 2-argument -forms, in most case you only need to specialize the 3-argument form.) For this to work it's important -that `SparseArray` is mutable (supports `setindex!`). Defining `similar`, `getindex` and -`setindex!` for `SparseArray` also makes it possible to [`copy`](@ref) the array: - -```jldoctest squarevectype -julia> copy(A) -3×3 SparseArray{Float64, 2}: - 1.0 4.0 7.0 - 2.0 5.0 8.0 - 3.0 6.0 9.0 -``` - -In addition to all the iterable and indexable methods from above, these types can also interact -with each other and use most of the methods defined in Julia Base for `AbstractArrays`: - -```jldoctest squarevectype -julia> A[SquaresVector(3)] -3-element SparseArray{Float64, 1}: - 1.0 - 4.0 - 9.0 - -julia> sum(A) -45.0 -``` - -If you are defining an array type that allows non-traditional indexing (indices that start at -something other than 1), you should specialize [`axes`](@ref). You should also specialize [`similar`](@ref) -so that the `dims` argument (ordinarily a `Dims` size-tuple) can accept `AbstractUnitRange` objects, -perhaps range-types `Ind` of your own design. For more information, see -[Arrays with custom indices](@ref man-custom-indices). - -## [Strided Arrays](@id man-interface-strided-arrays) - -| Methods to implement | | Brief description | -|:----------------------------------------------- |:-------------------------------------- |:------------------------------------------------------------------------------------- | -| `strides(A)` | | Return the distance in memory (in number of elements) between adjacent elements in each dimension as a tuple. If `A` is an `AbstractArray{T,0}`, this should return an empty tuple. | -| `Base.unsafe_convert(::Type{Ptr{T}}, A)` | | Return the native address of an array. | -| `Base.elsize(::Type{<:A})` | | Return the stride between consecutive elements in the array. | -| **Optional methods** | **Default definition** | **Brief description** | -| `stride(A, i::Int)` | `strides(A)[i]` | Return the distance in memory (in number of elements) between adjacent elements in dimension k. | - -A strided array is a subtype of `AbstractArray` whose entries are stored in memory with fixed strides. -Provided the element type of the array is compatible with BLAS, a strided array can utilize BLAS and LAPACK routines -for more efficient linear algebra routines. A typical example of a user-defined strided array is one -that wraps a standard `Array` with additional structure. - -Warning: do not implement these methods if the underlying storage is not actually strided, as it -may lead to incorrect results or segmentation faults. - -Here are some examples to demonstrate which type of arrays are strided and which are not: -```julia -1:5 # not strided (there is no storage associated with this array.) -Vector(1:5) # is strided with strides (1,) -A = [1 5; 2 6; 3 7; 4 8] # is strided with strides (1,4) -V = view(A, 1:2, :) # is strided with strides (1,4) -V = view(A, 1:2:3, 1:2) # is strided with strides (2,4) -V = view(A, [1,2,4], :) # is not strided, as the spacing between rows is not fixed. -``` - - - - - -## [Customizing broadcasting](@id man-interfaces-broadcasting) - -| Methods to implement | Brief description | -|:-------------------- |:----------------- | -| `Base.BroadcastStyle(::Type{SrcType}) = SrcStyle()` | Broadcasting behavior of `SrcType` | -| `Base.similar(bc::Broadcasted{DestStyle}, ::Type{ElType})` | Allocation of output container | -| **Optional methods** | | | -| `Base.BroadcastStyle(::Style1, ::Style2) = Style12()` | Precedence rules for mixing styles | -| `Base.axes(x)` | Declaration of the indices of `x`, as per [`axes(x)`](@ref). | -| `Base.broadcastable(x)` | Convert `x` to an object that has `axes` and supports indexing | -| **Bypassing default machinery** | | -| `Base.copy(bc::Broadcasted{DestStyle})` | Custom implementation of `broadcast` | -| `Base.copyto!(dest, bc::Broadcasted{DestStyle})` | Custom implementation of `broadcast!`, specializing on `DestStyle` | -| `Base.copyto!(dest::DestType, bc::Broadcasted{Nothing})` | Custom implementation of `broadcast!`, specializing on `DestType` | -| `Base.Broadcast.broadcasted(f, args...)` | Override the default lazy behavior within a fused expression | -| `Base.Broadcast.instantiate(bc::Broadcasted{DestStyle})` | Override the computation of the lazy broadcast's axes | - -[Broadcasting](@ref) is triggered by an explicit call to `broadcast` or `broadcast!`, or implicitly by -"dot" operations like `A .+ b` or `f.(x, y)`. Any object that has [`axes`](@ref) and supports -indexing can participate as an argument in broadcasting, and by default the result is stored -in an `Array`. This basic framework is extensible in three major ways: - -* Ensuring that all arguments support broadcast -* Selecting an appropriate output array for the given set of arguments -* Selecting an efficient implementation for the given set of arguments - -Not all types support `axes` and indexing, but many are convenient to allow in broadcast. -The [`Base.broadcastable`](@ref) function is called on each argument to broadcast, allowing -it to return something different that supports `axes` and indexing. By -default, this is the identity function for all `AbstractArray`s and `Number`s — they already -support `axes` and indexing. - -If a type is intended to act like a "0-dimensional scalar" (a single object) rather than as a -container for broadcasting, then the following method should be defined: -```julia -Base.broadcastable(o::MyType) = Ref(o) -``` -that returns the argument wrapped in a 0-dimensional [`Ref`](@ref) container. For example, such a wrapper -method is defined for types themselves, functions, special singletons like [`missing`](@ref) and [`nothing`](@ref), and dates. - -Custom array-like types can specialize -`Base.broadcastable` to define their shape, but they should follow the convention that -`collect(Base.broadcastable(x)) == collect(x)`. A notable exception is `AbstractString`; -strings are special-cased to behave as scalars for the purposes of broadcast even though -they are iterable collections of their characters (see [Strings](@ref) for more). - -The next two steps (selecting the output array and implementation) are dependent upon -determining a single answer for a given set of arguments. Broadcast must take all the varied -types of its arguments and collapse them down to just one output array and one -implementation. Broadcast calls this single answer a "style". Every broadcastable object -each has its own preferred style, and a promotion-like system is used to combine these -styles into a single answer — the "destination style". - -### Broadcast Styles - -`Base.BroadcastStyle` is the abstract type from which all broadcast styles are derived. When used as a -function it has two possible forms, unary (single-argument) and binary. The unary variant states -that you intend to implement specific broadcasting behavior and/or output type, and do not wish to -rely on the default fallback [`Broadcast.DefaultArrayStyle`](@ref). - -To override these defaults, you can define a custom `BroadcastStyle` for your object: - -```julia -struct MyStyle <: Broadcast.BroadcastStyle end -Base.BroadcastStyle(::Type{<:MyType}) = MyStyle() -``` - -In some cases it might be convenient not to have to define `MyStyle`, in which case you can -leverage one of the general broadcast wrappers: - - - `Base.BroadcastStyle(::Type{<:MyType}) = Broadcast.Style{MyType}()` can be - used for arbitrary types. - - `Base.BroadcastStyle(::Type{<:MyType}) = Broadcast.ArrayStyle{MyType}()` is preferred - if `MyType` is an `AbstractArray`. - - For `AbstractArrays` that only support a certain dimensionality, create a subtype of `Broadcast.AbstractArrayStyle{N}` (see below). - -When your broadcast operation involves several arguments, individual argument styles get -combined to determine a single `DestStyle` that controls the type of the output container. -For more details, see [below](@ref writing-binary-broadcasting-rules). - -### Selecting an appropriate output array - -The broadcast style is computed for every broadcasting operation to allow for -dispatch and specialization. The actual allocation of the result array is -handled by `similar`, using the Broadcasted object as its first argument. - -```julia -Base.similar(bc::Broadcasted{DestStyle}, ::Type{ElType}) -``` - -The fallback definition is - -```julia -similar(bc::Broadcasted{DefaultArrayStyle{N}}, ::Type{ElType}) where {N,ElType} = - similar(Array{ElType}, axes(bc)) -``` - -However, if needed you can specialize on any or all of these arguments. The final argument -`bc` is a lazy representation of a (potentially fused) broadcast operation, a `Broadcasted` -object. For these purposes, the most important fields of the wrapper are -`f` and `args`, describing the function and argument list, respectively. Note that the argument -list can — and often does — include other nested `Broadcasted` wrappers. - -For a complete example, let's say you have created a type, `ArrayAndChar`, that stores an -array and a single character: - -```jldoctest ArrayAndChar; output = false -struct ArrayAndChar{T,N} <: AbstractArray{T,N} - data::Array{T,N} - char::Char -end -Base.size(A::ArrayAndChar) = size(A.data) -Base.getindex(A::ArrayAndChar{T,N}, inds::Vararg{Int,N}) where {T,N} = A.data[inds...] -Base.setindex!(A::ArrayAndChar{T,N}, val, inds::Vararg{Int,N}) where {T,N} = A.data[inds...] = val -Base.showarg(io::IO, A::ArrayAndChar, toplevel) = print(io, typeof(A), " with char '", A.char, "'") -# output - -``` - -You might want broadcasting to preserve the `char` "metadata". First we define - -```jldoctest ArrayAndChar; output = false -Base.BroadcastStyle(::Type{<:ArrayAndChar}) = Broadcast.ArrayStyle{ArrayAndChar}() -# output - -``` - -This means we must also define a corresponding `similar` method: -```jldoctest ArrayAndChar; output = false -function Base.similar(bc::Broadcast.Broadcasted{Broadcast.ArrayStyle{ArrayAndChar}}, ::Type{ElType}) where ElType - # Scan the inputs for the ArrayAndChar: - A = find_aac(bc) - # Use the char field of A to create the output - ArrayAndChar(similar(Array{ElType}, axes(bc)), A.char) -end - -"`A = find_aac(As)` returns the first ArrayAndChar among the arguments." -find_aac(bc::Base.Broadcast.Broadcasted) = find_aac(bc.args) -find_aac(args::Tuple) = find_aac(find_aac(args[1]), Base.tail(args)) -find_aac(x) = x -find_aac(::Tuple{}) = nothing -find_aac(a::ArrayAndChar, rest) = a -find_aac(::Any, rest) = find_aac(rest) -# output -find_aac (generic function with 6 methods) -``` - -From these definitions, one obtains the following behavior: -```jldoctest ArrayAndChar -julia> a = ArrayAndChar([1 2; 3 4], 'x') -2×2 ArrayAndChar{Int64, 2} with char 'x': - 1 2 - 3 4 - -julia> a .+ 1 -2×2 ArrayAndChar{Int64, 2} with char 'x': - 2 3 - 4 5 - -julia> a .+ [5,10] -2×2 ArrayAndChar{Int64, 2} with char 'x': - 6 7 - 13 14 -``` - -### [Extending broadcast with custom implementations](@id extending-in-place-broadcast) - -In general, a broadcast operation is represented by a lazy `Broadcasted` container that holds onto -the function to be applied alongside its arguments. Those arguments may themselves be more nested -`Broadcasted` containers, forming a large expression tree to be evaluated. A nested tree of -`Broadcasted` containers is directly constructed by the implicit dot syntax; `5 .+ 2.*x` is -transiently represented by `Broadcasted(+, 5, Broadcasted(*, 2, x))`, for example. This is -invisible to users as it is immediately realized through a call to `copy`, but it is this container -that provides the basis for broadcast's extensibility for authors of custom types. The built-in -broadcast machinery will then determine the result type and size based upon the arguments, allocate -it, and then finally copy the realization of the `Broadcasted` object into it with a default -`copyto!(::AbstractArray, ::Broadcasted)` method. The built-in fallback `broadcast` and -`broadcast!` methods similarly construct a transient `Broadcasted` representation of the operation -so they can follow the same codepath. This allows custom array implementations to -provide their own `copyto!` specialization to customize and -optimize broadcasting. This is again determined by the computed broadcast style. This is such -an important part of the operation that it is stored as the first type parameter of the -`Broadcasted` type, allowing for dispatch and specialization. - -For some types, the machinery to "fuse" operations across nested levels of broadcasting -is not available or could be done more efficiently incrementally. In such cases, you may -need or want to evaluate `x .* (x .+ 1)` as if it had been -written `broadcast(*, x, broadcast(+, x, 1))`, where the inner operation is evaluated before -tackling the outer operation. This sort of eager operation is directly supported by a bit -of indirection; instead of directly constructing `Broadcasted` objects, Julia lowers the -fused expression `x .* (x .+ 1)` to `Broadcast.broadcasted(*, x, Broadcast.broadcasted(+, x, 1))`. Now, -by default, `broadcasted` just calls the `Broadcasted` constructor to create the lazy representation -of the fused expression tree, but you can choose to override it for a particular combination -of function and arguments. - -As an example, the builtin `AbstractRange` objects use this machinery to optimize pieces -of broadcasted expressions that can be eagerly evaluated purely in terms of the start, -step, and length (or stop) instead of computing every single element. Just like all the -other machinery, `broadcasted` also computes and exposes the combined broadcast style of its -arguments, so instead of specializing on `broadcasted(f, args...)`, you can specialize on -`broadcasted(::DestStyle, f, args...)` for any combination of style, function, and arguments. - -For example, the following definition supports the negation of ranges: - -```julia -broadcasted(::DefaultArrayStyle{1}, ::typeof(-), r::OrdinalRange) = range(-first(r), step=-step(r), length=length(r)) -``` - -### [Extending in-place broadcasting](@id extending-in-place-broadcast) - -In-place broadcasting can be supported by defining the appropriate `copyto!(dest, bc::Broadcasted)` -method. Because you might want to specialize either on `dest` or the specific subtype of `bc`, -to avoid ambiguities between packages we recommend the following convention. - -If you wish to specialize on a particular style `DestStyle`, define a method for -```julia -copyto!(dest, bc::Broadcasted{DestStyle}) -``` -Optionally, with this form you can also specialize on the type of `dest`. - -If instead you want to specialize on the destination type `DestType` without specializing -on `DestStyle`, then you should define a method with the following signature: - -```julia -copyto!(dest::DestType, bc::Broadcasted{Nothing}) -``` - -This leverages a fallback implementation of `copyto!` that converts the wrapper into a -`Broadcasted{Nothing}`. Consequently, specializing on `DestType` has lower precedence than -methods that specialize on `DestStyle`. - -Similarly, you can completely override out-of-place broadcasting with a `copy(::Broadcasted)` -method. - -#### Working with `Broadcasted` objects - -In order to implement such a `copy` or `copyto!`, method, of course, you must -work with the `Broadcasted` wrapper to compute each element. There are two main -ways of doing so: - -* `Broadcast.flatten` recomputes the potentially nested operation into a single - function and flat list of arguments. You are responsible for implementing the - broadcasting shape rules yourself, but this may be helpful in limited situations. -* Iterating over the `CartesianIndices` of the `axes(::Broadcasted)` and using - indexing with the resulting `CartesianIndex` object to compute the result. - -### [Writing binary broadcasting rules](@id writing-binary-broadcasting-rules) - -The precedence rules are defined by binary `BroadcastStyle` calls: - -```julia -Base.BroadcastStyle(::Style1, ::Style2) = Style12() -``` - -where `Style12` is the `BroadcastStyle` you want to choose for outputs involving -arguments of `Style1` and `Style2`. For example, - -```julia -Base.BroadcastStyle(::Broadcast.Style{Tuple}, ::Broadcast.AbstractArrayStyle{0}) = Broadcast.Style{Tuple}() -``` - -indicates that `Tuple` "wins" over zero-dimensional arrays (the output container will be a tuple). -It is worth noting that you do not need to (and should not) define both argument orders -of this call; defining one is sufficient no matter what order the user supplies the arguments in. - -For `AbstractArray` types, defining a `BroadcastStyle` supersedes the fallback choice, -[`Broadcast.DefaultArrayStyle`](@ref). `DefaultArrayStyle` and the abstract supertype, `AbstractArrayStyle`, store the dimensionality as a type parameter to support specialized -array types that have fixed dimensionality requirements. - -`DefaultArrayStyle` "loses" to any other -`AbstractArrayStyle` that has been defined because of the following methods: - -```julia -BroadcastStyle(a::AbstractArrayStyle{Any}, ::DefaultArrayStyle) = a -BroadcastStyle(a::AbstractArrayStyle{N}, ::DefaultArrayStyle{N}) where N = a -BroadcastStyle(a::AbstractArrayStyle{M}, ::DefaultArrayStyle{N}) where {M,N} = - typeof(a)(Val(max(M, N))) -``` - -You do not need to write binary `BroadcastStyle` -rules unless you want to establish precedence for -two or more non-`DefaultArrayStyle` types. - -If your array type does have fixed dimensionality requirements, then you should -subtype `AbstractArrayStyle`. For example, the sparse array code has the following definitions: - -```julia -struct SparseVecStyle <: Broadcast.AbstractArrayStyle{1} end -struct SparseMatStyle <: Broadcast.AbstractArrayStyle{2} end -Base.BroadcastStyle(::Type{<:SparseVector}) = SparseVecStyle() -Base.BroadcastStyle(::Type{<:SparseMatrixCSC}) = SparseMatStyle() -``` - -Whenever you subtype `AbstractArrayStyle`, you also need to define rules for combining -dimensionalities, by creating a constructor for your style that takes a `Val(N)` argument. -For example: - -```julia -SparseVecStyle(::Val{0}) = SparseVecStyle() -SparseVecStyle(::Val{1}) = SparseVecStyle() -SparseVecStyle(::Val{2}) = SparseMatStyle() -SparseVecStyle(::Val{N}) where N = Broadcast.DefaultArrayStyle{N}() -``` - -These rules indicate that the combination of a `SparseVecStyle` with 0- or 1-dimensional arrays -yields another `SparseVecStyle`, that its combination with a 2-dimensional array -yields a `SparseMatStyle`, and anything of higher dimensionality falls back to the dense arbitrary-dimensional framework. -These rules allow broadcasting to keep the sparse representation for operations that result -in one or two dimensional outputs, but produce an `Array` for any other dimensionality. - -## [Instance Properties](@id man-instance-properties) - -| Methods to implement | Default definition | Brief description | -|:--------------------------------- |:---------------------------- |:------------------------------------------------------------------------------------- | -| `propertynames(x::ObjType, private::Bool=false)` | `fieldnames(typeof(x))` | Return a tuple of the properties (`x.property`) of an object `x`. If `private=true`, also return property names intended to be kept as private | -| `getproperty(x::ObjType, s::Symbol)` | `getfield(x, s)` | Return property `s` of `x`. `x.s` calls `getproperty(x, :s)`. | -| `setproperty!(x::ObjType, s::Symbol, v)` | `setfield!(x, s, v)` | Set property `s` of `x` to `v`. `x.s = v` calls `setproperty!(x, :s, v)`. Should return `v`.| - -Sometimes, it is desirable to change how the end-user interacts with the fields of an object. -Instead of granting direct access to type fields, an extra layer of abstraction between -the user and the code can be provided by overloading `object.field`. Properties are what the -user *sees of* the object, fields what the object *actually is*. - -By default, properties and fields are the same. However, this behavior can be changed. -For example, take this representation of a point in a plane in [polar coordinates](https://en.wikipedia.org/wiki/Polar_coordinate_system): - -```jldoctest polartype -julia> mutable struct Point - r::Float64 - ϕ::Float64 - end - -julia> p = Point(7.0, pi/4) -Point(7.0, 0.7853981633974483) -``` - -As described in the table above dot access `p.r` is the same as `getproperty(p, :r)` which is by default the same as `getfield(p, :r)`: - -```jldoctest polartype -julia> propertynames(p) -(:r, :ϕ) - -julia> getproperty(p, :r), getproperty(p, :ϕ) -(7.0, 0.7853981633974483) - -julia> p.r, p.ϕ -(7.0, 0.7853981633974483) - -julia> getfield(p, :r), getproperty(p, :ϕ) -(7.0, 0.7853981633974483) -``` - -However, we may want users to be unaware that `Point` stores the coordinates as `r` and `ϕ` (fields), -and instead interact with `x` and `y` (properties). The methods in the first column can be -defined to add new functionality: - -```jldoctest polartype -julia> Base.propertynames(::Point, private::Bool=false) = private ? (:x, :y, :r, :ϕ) : (:x, :y) - -julia> function Base.getproperty(p::Point, s::Symbol) - if s === :x - return getfield(p, :r) * cos(getfield(p, :ϕ)) - elseif s === :y - return getfield(p, :r) * sin(getfield(p, :ϕ)) - else - # This allows accessing fields with p.r and p.ϕ - return getfield(p, s) - end - end - -julia> function Base.setproperty!(p::Point, s::Symbol, f) - if s === :x - y = p.y - setfield!(p, :r, sqrt(f^2 + y^2)) - setfield!(p, :ϕ, atan(y, f)) - return f - elseif s === :y - x = p.x - setfield!(p, :r, sqrt(x^2 + f^2)) - setfield!(p, :ϕ, atan(f, x)) - return f - else - # This allow modifying fields with p.r and p.ϕ - return setfield!(p, s, f) - end - end -``` - -It is important that `getfield` and `setfield` are used inside `getproperty` and `setproperty!` instead of the dot syntax, -since the dot syntax would make the functions recursive which can lead to type inference issues. We can now -try out the new functionality: - -```jldoctest polartype -julia> propertynames(p) -(:x, :y) - -julia> p.x -4.949747468305833 - -julia> p.y = 4.0 -4.0 - -julia> p.r -6.363961030678928 -``` - -Finally, it is worth noting that adding instance properties like this is quite -rarely done in Julia and should in general only be done if there is a good -reason for doing so. - -## [Rounding](@id man-rounding-interface) - -| Methods to implement | Default definition | Brief description | -|:--------------------------------------------- |:------------------------- |:--------------------------------------------------------------------------------------------------- | -| `round(x::ObjType, r::RoundingMode)` | none | Round `x` and return the result. If possible, round should return an object of the same type as `x` | -| `round(T::Type, x::ObjType, r::RoundingMode)` | `convert(T, round(x, r))` | Round `x`, returning the result as a `T` | - -To support rounding on a new type it is typically sufficient to define the single method -`round(x::ObjType, r::RoundingMode)`. The passed rounding mode determines in which direction -the value should be rounded. The most commonly used rounding modes are `RoundNearest`, -`RoundToZero`, `RoundDown`, and `RoundUp`, as these rounding modes are used in the -definitions of the one argument `round`, method, and `trunc`, `floor`, and `ceil`, -respectively. - -In some cases, it is possible to define a three-argument `round` method that is more -accurate or performant than the two-argument method followed by conversion. In this case it -is acceptable to define the three argument method in addition to the two argument method. -If it is impossible to represent the rounded result as an object of the type `T`, -then the three argument method should throw an `InexactError`. - -For example, if we have an `Interval` type which represents a range of possible values -similar to https://github.com/JuliaPhysics/Measurements.jl, we may define rounding on that -type with the following - -```jldoctest -julia> struct Interval{T} - min::T - max::T - end - -julia> Base.round(x::Interval, r::RoundingMode) = Interval(round(x.min, r), round(x.max, r)) - -julia> x = Interval(1.7, 2.2) -Interval{Float64}(1.7, 2.2) - -julia> round(x) -Interval{Float64}(2.0, 2.0) - -julia> floor(x) -Interval{Float64}(1.0, 2.0) - -julia> ceil(x) -Interval{Float64}(2.0, 3.0) - -julia> trunc(x) -Interval{Float64}(1.0, 2.0) -``` diff --git a/julia.spdx.json b/julia.spdx.json index 63683dd302a39..21d07c68560a0 100644 --- a/julia.spdx.json +++ b/julia.spdx.json @@ -420,6 +420,18 @@ "copyrightText": "Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler", "summary": "A massively spiffy yet delicately unobtrusive compression library." }, + { + "name": "zstd", + "SPDXID": "SPDXRef-zstd", + "downloadLocation": "git+https://github.com/facebook/zstd.git", + "filesAnalyzed": false, + "homepage": "https://www.zstd.net", + "sourceInfo": "The git hash of the version in use can be found in the file deps/zstd.version", + "licenseConcluded": "BSD-3-Clause", + "licenseDeclared": "GPL-2.0+ OR BSD-3-Clause", + "copyrightText": "Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.", + "summary": "Zstandard, or zstd as short version, is a fast lossless compression algorithm." + }, { "name": "patchelf", "SPDXID": "SPDXRef-patchelf", @@ -627,6 +639,11 @@ "relationshipType": "BUILD_DEPENDENCY_OF", "relatedSpdxElement": "SPDXRef-JuliaMain" }, + { + "spdxElementId": "SPDXRef-zstd", + "relationshipType": "BUILD_DEPENDENCY_OF", + "relatedSpdxElement": "SPDXRef-JuliaMain" + }, { "spdxElementId": "SPDXRef-patchelf", "relationshipType": "BUILD_TOOL_OF", diff --git a/src/Makefile b/src/Makefile index bf9001e5fba93..d1cfe867431fc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -17,7 +17,8 @@ FLAGS := \ -D_GNU_SOURCE -I$(BUILDDIR) -I$(SRCDIR) \ -I$(SRCDIR)/flisp -I$(SRCDIR)/support \ -I$(LIBUV_INC) -I$(build_includedir) \ - -I$(JULIAHOME)/deps/valgrind + -I$(JULIAHOME)/deps/valgrind \ + -I$(ZSTD_INC) FLAGS += -Wall -Wno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden -fno-common \ -Wno-comment -Wpointer-arith -Wundef ifeq ($(USEGCC),1) # GCC bug #25509 (void)__attribute__((warn_unused_result)) @@ -46,7 +47,8 @@ SRCS := \ simplevector runtime_intrinsics precompile jloptions mtarraylist \ threading scheduler stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \ jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \ - crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall + crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall \ + null_sysimage RT_LLVMLINK := CG_LLVMLINK := @@ -56,7 +58,8 @@ CODEGEN_SRCS := codegen jitlayers aotcompile debuginfo disasm llvm-simdloop llvm llvm-final-gc-lowering llvm-pass-helpers llvm-late-gc-lowering llvm-ptls \ llvm-lower-handlers llvm-gc-invariant-verifier llvm-propagate-addrspaces \ llvm-multiversioning llvm-alloc-opt llvm-alloc-helpers cgmemmgr llvm-remove-addrspaces \ - llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures pipeline llvm_api + llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures pipeline llvm_api \ + null_sysimage FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --includedir) CG_LLVM_LIBS := all ifeq ($(USE_POLLY),1) @@ -160,15 +163,15 @@ endif CLANG_LDFLAGS := $(LLVM_LDFLAGS) ifeq ($(OS), Darwin) CLANG_LDFLAGS += -Wl,-undefined,dynamic_lookup -OSLIBS += -Wl,-U,__dyld_atfork_parent -Wl,-U,__dyld_atfork_prepare -Wl,-U,__dyld_dlopen_atfork_parent -Wl,-U,__dyld_dlopen_atfork_prepare -Wl,-U,_jl_image_pointers -Wl,-U,_jl_system_image_data -Wl,-U,_jl_system_image_size +OSLIBS += -Wl,-U,__dyld_atfork_parent -Wl,-U,__dyld_atfork_prepare -Wl,-U,__dyld_dlopen_atfork_parent -Wl,-U,__dyld_dlopen_atfork_prepare LIBJULIA_PATH_REL := @rpath/libjulia else LIBJULIA_PATH_REL := libjulia endif COMMON_LIBPATHS := -L$(build_libdir) -L$(build_shlibdir) -RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) -CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) +RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) $(LIBZSTD) +CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) $(LIBZSTD) RT_DEBUG_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp-debug.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport-debug.a -ljulia-debug $(RT_LIBS) CG_DEBUG_LIBS := $(COMMON_LIBPATHS) $(CG_LIBS) -ljulia-debug -ljulia-internal-debug RT_RELEASE_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport.a -ljulia $(RT_LIBS) @@ -471,7 +474,7 @@ $(build_shlibdir)/lib%Plugin.$(SHLIB_EXT): $(SRCDIR)/clangsa/%.cpp $(LLVM_CONFIG # before attempting this static analysis, so that all necessary headers # and dependencies are properly installed: # make -C src install-analysis-deps -ANALYSIS_DEPS := llvm clang llvm-tools libuv utf8proc +ANALYSIS_DEPS := llvm clang llvm-tools libuv utf8proc zstd ifeq ($(OS),Darwin) ANALYSIS_DEPS += llvmunwind else ifneq ($(OS),WINNT) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 97734852f9479..8132628b5d370 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -57,9 +57,10 @@ #include #include - using namespace llvm; +#include + #include "jitlayers.h" #include "serialize.h" #include "julia_assert.h" @@ -1659,12 +1660,25 @@ void jl_dump_native_impl(void *native_code, sysimgM.setDataLayout(DL); sysimgM.setStackProtectorGuard(StackProtectorGuard); sysimgM.setOverrideStackAlignment(OverrideStackAlignment); - Constant *data = ConstantDataArray::get(Context, - ArrayRef((const unsigned char*)z->buf, z->size)); + + int compression = jl_options.compress_sysimage ? 15 : 0; + ArrayRef sysimg_data{z->buf, (size_t)z->size}; + SmallVector compressed_data; + if (compression) { + compressed_data.resize(ZSTD_compressBound(z->size)); + size_t comp_size = ZSTD_compress(compressed_data.data(), compressed_data.size(), + z->buf, z->size, compression); + compressed_data.resize(comp_size); + sysimg_data = compressed_data; + ios_close(z); + free(z); + } + + Constant *data = ConstantDataArray::get(Context, sysimg_data); auto sysdata = new GlobalVariable(sysimgM, data->getType(), false, GlobalVariable::ExternalLinkage, data, "jl_system_image_data"); - sysdata->setAlignment(Align(64)); + sysdata->setAlignment(Align(jl_page_size)); #if JL_LLVM_VERSION >= 180000 sysdata->setCodeModel(CodeModel::Large); #else @@ -1672,14 +1686,27 @@ void jl_dump_native_impl(void *native_code, sysdata->setSection(".ldata"); #endif addComdat(sysdata, TheTriple); - Constant *len = ConstantInt::get(sysimgM.getDataLayout().getIntPtrType(Context), z->size); + Constant *len = ConstantInt::get(sysimgM.getDataLayout().getIntPtrType(Context), sysimg_data.size()); addComdat(new GlobalVariable(sysimgM, len->getType(), true, GlobalVariable::ExternalLinkage, len, "jl_system_image_size"), TheTriple); - // Free z here, since we've copied out everything into data - // Results in serious memory savings - ios_close(z); - free(z); + + const char *unpack_func = compression ? "jl_image_unpack_zstd" : "jl_image_unpack_uncomp"; + auto unpack = new GlobalVariable(sysimgM, DL.getIntPtrType(Context), true, + GlobalVariable::ExternalLinkage, nullptr, + unpack_func); + addComdat(new GlobalVariable(sysimgM, PointerType::getUnqual(Context), true, + GlobalVariable::ExternalLinkage, unpack, + "jl_image_unpack"), + TheTriple); + + if (!compression) { + // Free z here, since we've copied out everything into data + // Results in serious memory savings + ios_close(z); + free(z); + } + compressed_data.clear(); // Note that we don't set z to null, this allows the check in WRITE_ARCHIVE // to function as expected // no need to free the module/context, destructor handles that @@ -1833,7 +1860,7 @@ void jl_dump_native_impl(void *native_code, builder.CreateRet(ConstantInt::get(T_int32, 1)); } if (imaging_mode) { - auto specs = jl_get_llvm_clone_targets(); + auto specs = jl_get_llvm_clone_targets(jl_options.cpu_target); const uint32_t base_flags = has_veccall ? JL_TARGET_VEC_CALL : 0; SmallVector data; auto push_i32 = [&] (uint32_t v) { diff --git a/src/init.c b/src/init.c index a5c9a6b19f94d..3517aa9d8fa5a 100644 --- a/src/init.c +++ b/src/init.c @@ -847,19 +847,30 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ { JL_TIMING(JULIA_INIT, JULIA_INIT); jl_resolve_sysimg_location(rel); + // loads sysimg if available, and conditionally sets jl_options.cpu_target - if (rel == JL_IMAGE_IN_MEMORY) - jl_set_sysimg_so(jl_exe_handle); + jl_image_buf_t sysimage = { JL_IMAGE_KIND_NONE }; + if (rel == JL_IMAGE_IN_MEMORY) { + sysimage = jl_set_sysimg_so(jl_exe_handle); + jl_options.image_file = jl_options.julia_bin; + } else if (jl_options.image_file) - jl_preload_sysimg_so(jl_options.image_file); + sysimage = jl_preload_sysimg(jl_options.image_file); + if (jl_options.cpu_target == NULL) jl_options.cpu_target = "native"; - jl_init_codegen(); + // Parse image, perform relocations, and init JIT targets, etc. + jl_image_t parsed_image = jl_init_processor_sysimg(sysimage, jl_options.cpu_target); + + jl_init_codegen(); jl_init_common_symbols(); - if (jl_options.image_file) { - jl_restore_system_image(jl_options.image_file); + + if (sysimage.kind != JL_IMAGE_KIND_NONE) { + // Load the .ji or .so sysimage + jl_restore_system_image(&parsed_image, sysimage); } else { + // No sysimage provided, init a minimal environment jl_init_types(); jl_global_roots_list = (jl_genericmemory_t*)jl_an_empty_memory_any; jl_global_roots_keyset = (jl_genericmemory_t*)jl_an_empty_memory_any; @@ -868,7 +879,7 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ jl_init_flisp(); jl_init_serializer(); - if (!jl_options.image_file) { + if (sysimage.kind == JL_IMAGE_KIND_NONE) { jl_top_module = jl_core_module; jl_init_intrinsic_functions(); jl_init_primitives(); @@ -892,7 +903,8 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ jl_gc_enable(1); - if (jl_options.image_file && (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) { + if ((sysimage.kind != JL_IMAGE_KIND_NONE) && + (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) { jl_array_t *init_order = jl_module_init_order; JL_GC_PUSH1(&init_order); jl_module_init_order = NULL; diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index b3ed1e069de16..e418ce06a74a7 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1139,7 +1139,7 @@ namespace { options.ExplicitEmulatedTLS = true; #endif uint32_t target_flags = 0; - auto target = jl_get_llvm_target(imaging_default(), target_flags); + auto target = jl_get_llvm_target(jl_options.cpu_target, jl_generating_output(), target_flags); auto &TheCPU = target.first; SmallVector targetFeatures(target.second.begin(), target.second.end()); std::string errorstr; diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 05057cfd80861..66db481187b47 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -370,7 +370,7 @@ XX(jl_pointerset) \ XX(jl_pop_handler) \ XX(jl_pop_handler_noexcept) \ - XX(jl_preload_sysimg_so) \ + XX(jl_preload_sysimg) \ XX(jl_prepend_cwd) \ XX(jl_printf) \ XX(jl_print_backtrace) \ @@ -400,7 +400,6 @@ XX(jl_restore_incremental) \ XX(jl_restore_package_image_from_file) \ XX(jl_restore_system_image) \ - XX(jl_restore_system_image_data) \ XX(jl_rethrow) \ XX(jl_rethrow_other) \ XX(jl_running_on_valgrind) \ diff --git a/src/jloptions.c b/src/jloptions.c index ad13151703531..2b1c0d40005e6 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -101,6 +101,7 @@ JL_DLLEXPORT void jl_init_options(void) 0, // strip-ir 0, // permalloc_pkgimg 0, // heap-size-hint + 0, // compress_sysimage }; jl_options_initialized = 1; } @@ -222,10 +223,13 @@ static const char opts_hidden[] = " Enable or disable JIT compiler, or request exhaustive or minimal compilation\n\n" // compiler output options - " --output-o Generate an object file (including system image data)\n" - " --output-ji Generate a system image data file (.ji)\n" - " --strip-metadata Remove docstrings and source location info from system image\n" - " --strip-ir Remove IR (intermediate representation) of compiled functions\n\n" + " --output-o Generate an object file (including system image data)\n" + " --output-ji Generate a system image data file (.ji)\n" + " --strip-metadata Remove docstrings and source location info from system image\n" + " --strip-ir Remove IR (intermediate representation) of compiled functions\n" + " --compress-sysimage={yes|no*} Compress the sys/pkgimage heap at the expense of\n" + " slightly increased load time.\n" + "\n" // compiler debugging (see the devdocs for tips on using these options) " --output-unopt-bc Generate unoptimized LLVM bitcode (.bc)\n" @@ -282,7 +286,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_strip_ir, opt_heap_size_hint, opt_gc_threads, - opt_permalloc_pkgimg + opt_permalloc_pkgimg, + opt_compress_sysimage, }; static const char* const shortopts = "+vhqH:e:E:L:J:C:it:p:O:g:"; static const struct option longopts[] = { @@ -344,6 +349,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "strip-ir", no_argument, 0, opt_strip_ir }, { "permalloc-pkgimg",required_argument, 0, opt_permalloc_pkgimg }, { "heap-size-hint", required_argument, 0, opt_heap_size_hint }, + { "compress-sysimage", required_argument, 0, opt_compress_sysimage }, { 0, 0, 0, 0 } }; @@ -895,6 +901,12 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) else jl_errorf("julia: invalid argument to --permalloc-pkgimg={yes|no} (%s)", optarg); break; + case opt_compress_sysimage: + if (!strcmp(optarg,"yes")) + jl_options.compress_sysimage = 1; + else if (!strcmp(optarg,"no")) + jl_options.compress_sysimage = 0; + break; default: jl_errorf("julia: unhandled option -- %c\n" "This is a bug, please report it.", c); diff --git a/src/jloptions.h b/src/jloptions.h index f72e72b462066..ca938aa3fb503 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -62,6 +62,7 @@ typedef struct { int8_t strip_ir; int8_t permalloc_pkgimg; uint64_t heap_size_hint; + int8_t compress_sysimage; } jl_options_t; #endif diff --git a/src/julia.h b/src/julia.h index 2bb8bc5d0b59f..b8783ff821d00 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2030,6 +2030,25 @@ typedef enum { JL_IMAGE_IN_MEMORY = 2 } JL_IMAGE_SEARCH; +typedef enum { + JL_IMAGE_KIND_NONE = 0, + JL_IMAGE_KIND_JI, + JL_IMAGE_KIND_SO, +} jl_image_kind_t; + +// A loaded, but unparsed .ji or .so image file +typedef struct { + jl_image_kind_t kind; + void *handle; + const void *pointers; // jl_image_pointers_t * + const char *data; + size_t size; + uint64_t base; +} jl_image_buf_t; + +struct _jl_image_t; +typedef struct _jl_image_t jl_image_t; + JL_DLLIMPORT const char *jl_get_libdir(void); JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel); JL_DLLEXPORT void jl_init(void); @@ -2046,11 +2065,10 @@ JL_DLLEXPORT const char *jl_pathname_for_handle(void *handle); JL_DLLEXPORT jl_gcframe_t **jl_adopt_thread(void); JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s); -JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname); -JL_DLLEXPORT void jl_set_sysimg_so(void *handle); +JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname); +JL_DLLEXPORT jl_image_buf_t jl_set_sysimg_so(void *handle); JL_DLLEXPORT void jl_create_system_image(void **, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos); -JL_DLLEXPORT void jl_restore_system_image(const char *fname); -JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); +JL_DLLEXPORT void jl_restore_system_image(jl_image_t *image, jl_image_buf_t buf); JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int complete, const char *pkgimage); JL_DLLEXPORT void jl_set_newly_inferred(jl_value_t *newly_inferred); diff --git a/src/julia_internal.h b/src/julia_internal.h index 72d9dfdad5c7e..dff769e7e0f66 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1710,24 +1710,6 @@ jl_sym_t *_jl_symbol(const char *str, size_t len) JL_NOTSAFEPOINT; #define JL_GC_ASSERT_LIVE(x) (void)(x) #endif -#ifdef _OS_WINDOWS_ -// On Windows, weak symbols do not default to 0 due to a GCC bug -// (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90826), use symbol -// aliases with a known value instead. -#define JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(sym) __attribute__((weak,alias(#sym))) -#define JL_WEAK_SYMBOL_DEFAULT(sym) &sym -#else -#define JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(sym) __attribute__((weak)) -#define JL_WEAK_SYMBOL_DEFAULT(sym) NULL -#endif - -//JL_DLLEXPORT float julia__gnu_h2f_ieee(half param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT half julia__gnu_f2h_ieee(float param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT half julia__truncdfhf2(double param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT float julia__truncsfbf2(float param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT float julia__truncdfbf2(double param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT double julia__extendhfdf2(half n) JL_NOTSAFEPOINT; - JL_DLLEXPORT uint32_t jl_crc32c(uint32_t crc, const char *buf, size_t len); // -- exports from codegen -- // diff --git a/src/llvm-multiversioning.cpp b/src/llvm-multiversioning.cpp index 1901a721e5e08..299213090f064 100644 --- a/src/llvm-multiversioning.cpp +++ b/src/llvm-multiversioning.cpp @@ -215,7 +215,7 @@ static void annotate_module_clones(Module &M) { if (auto maybe_specs = get_target_specs(M)) { specs = std::move(*maybe_specs); } else { - auto full_specs = jl_get_llvm_clone_targets(); + auto full_specs = jl_get_llvm_clone_targets(jl_options.cpu_target); specs.reserve(full_specs.size()); for (auto &spec: full_specs) { specs.push_back(TargetSpec::fromSpec(spec)); diff --git a/src/null_sysimage.c b/src/null_sysimage.c new file mode 100644 index 0000000000000..da50640d11b18 --- /dev/null +++ b/src/null_sysimage.c @@ -0,0 +1,13 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include +#include "processor.h" + +/** + * These symbols support statically linking the sysimage with libjulia-internal. + * + * Here we provide dummy definitions that are used when these are not linked + * together (the default build configuration). The 0 value of jl_image_unpack + * is used as a sentinel to indicate that the sysimage should be loaded externally. + **/ +jl_image_unpack_func_t *jl_image_unpack = NULL; diff --git a/src/processor.cpp b/src/processor.cpp index 730e470f4153d..28f5d80d77cd2 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -504,7 +504,8 @@ static inline llvm::SmallVector, 0> parse_cmdline(const char *option, F &&feature_cb) { if (!option) - option = "native"; + abort(); + llvm::SmallVector, 0> res; TargetData arg{}; auto reset_arg = [&] { @@ -612,36 +613,29 @@ parse_cmdline(const char *option, F &&feature_cb) // Cached version of command line parsing template -static inline llvm::SmallVector, 0> &get_cmdline_targets(F &&feature_cb) +static inline llvm::SmallVector, 0> &get_cmdline_targets(const char *cpu_target, F &&feature_cb) { static llvm::SmallVector, 0> targets = - parse_cmdline(jl_options.cpu_target, std::forward(feature_cb)); + parse_cmdline(cpu_target, std::forward(feature_cb)); return targets; } -extern "C" { -void *image_pointers_unavailable; -extern void * JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(image_pointers_unavailable) jl_image_pointers; -} - // Load sysimg, use the `callback` for dispatch and perform all relocations // for the selected target. template -static inline jl_image_t parse_sysimg(void *hdl, F &&callback) +static inline jl_image_t parse_sysimg(jl_image_buf_t image, F &&callback, void *ctx) { JL_TIMING(LOAD_IMAGE, LOAD_Processor); jl_image_t res{}; - const jl_image_pointers_t *pointers; - if (hdl == jl_exe_handle && &jl_image_pointers != JL_WEAK_SYMBOL_DEFAULT(image_pointers_unavailable)) - pointers = (const jl_image_pointers_t *)&jl_image_pointers; - else - jl_dlsym(hdl, "jl_image_pointers", (void**)&pointers, 1); + if (image.kind != JL_IMAGE_KIND_SO) + return res; + const jl_image_pointers_t *pointers = (const jl_image_pointers_t *)image.pointers; const void *ids = pointers->target_data; jl_value_t* rejection_reason = nullptr; JL_GC_PUSH1(&rejection_reason); - uint32_t target_idx = callback(ids, &rejection_reason); + uint32_t target_idx = callback(ctx, ids, &rejection_reason); if (target_idx == UINT32_MAX) { jl_error(jl_string_ptr(rejection_reason)); } @@ -799,17 +793,7 @@ static inline jl_image_t parse_sysimg(void *hdl, F &&callback) res.fptrs.nclones = clones.size(); } -#ifdef _OS_WINDOWS_ - res.base = (intptr_t)hdl; -#else - Dl_info dlinfo; - if (dladdr((void*)pointers, &dlinfo) != 0) { - res.base = (intptr_t)dlinfo.dli_fbase; - } - else { - res.base = 0; - } -#endif + res.base = image.base; { void *pgcstack_func_slot = pointers->ptls->pgcstack_func_slot; @@ -1025,7 +1009,7 @@ JL_DLLEXPORT jl_value_t *jl_get_cpu_features(void) } extern "C" JL_DLLEXPORT jl_value_t* jl_reflect_clone_targets() { - auto specs = jl_get_llvm_clone_targets(); + auto specs = jl_get_llvm_clone_targets(jl_options.cpu_target); const uint32_t base_flags = 0; llvm::SmallVector data; auto push_i32 = [&] (uint32_t v) { diff --git a/src/processor.h b/src/processor.h index 82a1121aaf7c4..965e020429598 100644 --- a/src/processor.h +++ b/src/processor.h @@ -64,6 +64,7 @@ JL_DLLEXPORT int jl_test_cpu_feature(jl_cpu_feature_t feature); static const uint32_t jl_sysimg_tag_mask = 0x80000000u; static const uint32_t jl_sysimg_val_mask = ~((uint32_t)0x80000000u); +// A parsed image file typedef struct _jl_image_fptrs_t { // number of functions uint32_t nptrs; @@ -82,14 +83,14 @@ typedef struct _jl_image_fptrs_t { const uint32_t *clone_idxs; } jl_image_fptrs_t; -typedef struct { +struct _jl_image_t { uint64_t base; const char *gvars_base; const int32_t *gvars_offsets; uint32_t ngvars; jl_image_fptrs_t fptrs; void **jl_small_typeof; -} jl_image_t; +}; // The header for each image // Details important counts about the image @@ -206,8 +207,8 @@ typedef struct { * * Return the data about the function pointers selected. */ -jl_image_t jl_init_processor_sysimg(void *hdl); -jl_image_t jl_init_processor_pkgimg(void *hdl); +jl_image_t jl_init_processor_sysimg(jl_image_buf_t image, const char *cpu_target); +jl_image_t jl_init_processor_pkgimg(jl_image_buf_t image); // Return the name of the host CPU as a julia string. JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void); @@ -224,6 +225,17 @@ JL_DLLEXPORT int32_t jl_set_zero_subnormals(int8_t isZero); JL_DLLEXPORT int32_t jl_get_zero_subnormals(void); JL_DLLEXPORT int32_t jl_set_default_nans(int8_t isDefault); JL_DLLEXPORT int32_t jl_get_default_nans(void); + +/** + * System image contents. + * + * These symbols are typically dummy values, unless statically linking + * libjulia-* and the sysimage together (see null_sysimage.c), in which + * case they allow accessing the local copy of the sysimage. + **/ +typedef void jl_image_unpack_func_t(void *handle, jl_image_buf_t *image); +extern jl_image_unpack_func_t *jl_image_unpack; + #ifdef __cplusplus } @@ -239,7 +251,7 @@ extern JL_DLLEXPORT bool jl_processor_print_help; * If the detected/specified CPU name is not available on the LLVM version specified, * a fallback CPU name will be used. Unsupported features will be ignored. */ -extern "C" JL_DLLEXPORT std::pair> jl_get_llvm_target(bool imaging, uint32_t &flags) JL_NOTSAFEPOINT; +extern "C" JL_DLLEXPORT std::pair> jl_get_llvm_target(const char *cpu_target, bool imaging, uint32_t &flags) JL_NOTSAFEPOINT; /** * Returns the CPU name and feature string to be used by LLVM disassembler. @@ -263,7 +275,7 @@ struct jl_target_spec_t { /** * Return the list of targets to clone */ -extern "C" JL_DLLEXPORT llvm::SmallVector jl_get_llvm_clone_targets(void) JL_NOTSAFEPOINT; +extern "C" JL_DLLEXPORT llvm::SmallVector jl_get_llvm_clone_targets(const char *cpu_target) JL_NOTSAFEPOINT; // NOLINTEND(clang-diagnostic-return-type-c-linkage) struct FeatureName { const char *name; diff --git a/src/processor_arm.cpp b/src/processor_arm.cpp index 0d9009afabec7..7df2b74773af4 100644 --- a/src/processor_arm.cpp +++ b/src/processor_arm.cpp @@ -1510,7 +1510,7 @@ static inline void disable_depends(FeatureList &features) ::disable_depends(features, Feature::deps, sizeof(Feature::deps) / sizeof(FeatureDep)); } -static const llvm::SmallVector, 0> &get_cmdline_targets(void) +static const llvm::SmallVector, 0> &get_cmdline_targets(const char *cpu_target) { auto feature_cb = [] (const char *str, size_t len, FeatureList &list) { #ifdef _CPU_AARCH64_ @@ -1527,7 +1527,7 @@ static const llvm::SmallVector, 0> &get_cmdline_targets(v set_bit(list, fbit, true); return true; }; - auto &targets = ::get_cmdline_targets(feature_cb); + auto &targets = ::get_cmdline_targets(cpu_target, feature_cb); for (auto &t: targets) { if (auto nname = normalize_cpu_name(t.name)) { t.name = nname; @@ -1590,10 +1590,11 @@ static int max_vector_size(const FeatureList &features) #endif } -static uint32_t sysimg_init_cb(const void *id, jl_value_t **rejection_reason) +static uint32_t sysimg_init_cb(void *ctx, const void *id, jl_value_t **rejection_reason) { // First see what target is requested for the JIT. - auto &cmdline = get_cmdline_targets(); + const char *cpu_target = (const char *)ctx; + auto &cmdline = get_cmdline_targets(cpu_target); TargetData target = arg_target_data(cmdline[0], true); // Then find the best match in the sysimg auto sysimg = deserialize_target_data((const uint8_t*)id); @@ -1617,7 +1618,7 @@ static uint32_t sysimg_init_cb(const void *id, jl_value_t **rejection_reason) return match.best_idx; } -static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason JL_REQUIRE_ROOTED_SLOT) +static uint32_t pkgimg_init_cb(void *ctx, const void *id, jl_value_t **rejection_reason JL_REQUIRE_ROOTED_SLOT) { TargetData target = jit_targets.front(); auto pkgimg = deserialize_target_data((const uint8_t*)id); @@ -1630,9 +1631,9 @@ static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason JL_ return match.best_idx; } -static void ensure_jit_target(bool imaging) +static void ensure_jit_target(const char *cpu_target, bool imaging) { - auto &cmdline = get_cmdline_targets(); + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, imaging); if (!jit_targets.empty()) return; @@ -1843,36 +1844,36 @@ JL_DLLEXPORT jl_value_t *jl_cpu_has_fma(int bits) #endif } -jl_image_t jl_init_processor_sysimg(void *hdl) +jl_image_t jl_init_processor_sysimg(jl_image_buf_t image, const char *cpu_target) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); - return parse_sysimg(hdl, sysimg_init_cb); + return parse_sysimg(image, sysimg_init_cb, (void *)cpu_target); } -jl_image_t jl_init_processor_pkgimg(void *hdl) +jl_image_t jl_init_processor_pkgimg(jl_image_buf_t image) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); if (jit_targets.size() > 1) jl_error("Expected only one JIT target"); - return parse_sysimg(hdl, pkgimg_init_cb); + return parse_sysimg(image, pkgimg_init_cb, NULL); } JL_DLLEXPORT jl_value_t* jl_check_pkgimage_clones(char *data) { jl_value_t *rejection_reason = NULL; JL_GC_PUSH1(&rejection_reason); - uint32_t match_idx = pkgimg_init_cb(data, &rejection_reason); + uint32_t match_idx = pkgimg_init_cb(NULL, data, &rejection_reason); JL_GC_POP(); if (match_idx == UINT32_MAX) return rejection_reason; return jl_nothing; } -std::pair> jl_get_llvm_target(bool imaging, uint32_t &flags) +std::pair> jl_get_llvm_target(const char *cpu_target, bool imaging, uint32_t &flags) { - ensure_jit_target(imaging); + ensure_jit_target(cpu_target, imaging); flags = jit_targets[0].en.flags; return get_llvm_target_vec(jit_targets[0]); } @@ -1891,10 +1892,10 @@ const std::pair &jl_get_llvm_disasm_target(void) } #ifndef __clang_gcanalyzer__ -llvm::SmallVector jl_get_llvm_clone_targets(void) +llvm::SmallVector jl_get_llvm_clone_targets(const char *cpu_target) { - auto &cmdline = get_cmdline_targets(); + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, true); llvm::SmallVector, 0> image_targets; for (auto &arg: cmdline) { diff --git a/src/processor_fallback.cpp b/src/processor_fallback.cpp index f8d9eb9fd9e73..c8c8feb072345 100644 --- a/src/processor_fallback.cpp +++ b/src/processor_fallback.cpp @@ -13,12 +13,12 @@ static inline const std::string &host_cpu_name() return name; } -static const llvm::SmallVector, 0> &get_cmdline_targets(void) +static const llvm::SmallVector, 0> &get_cmdline_targets(const char *cpu_target) { auto feature_cb = [] (const char*, size_t, FeatureList<1>&) { return false; }; - return ::get_cmdline_targets<1>(feature_cb); + return ::get_cmdline_targets<1>(cpu_target, feature_cb); } static llvm::SmallVector, 0> jit_targets; @@ -36,10 +36,11 @@ static TargetData<1> arg_target_data(const TargetData<1> &arg, bool require_host return res; } -static uint32_t sysimg_init_cb(const void *id, jl_value_t **rejection_reason) +static uint32_t sysimg_init_cb(void *ctx, const void *id, jl_value_t **rejection_reason) { // First see what target is requested for the JIT. - auto &cmdline = get_cmdline_targets(); + const char *cpu_target = (const char *)ctx; + auto &cmdline = get_cmdline_targets(cpu_target); TargetData<1> target = arg_target_data(cmdline[0], true); // Find the last name match or use the default one. uint32_t best_idx = 0; @@ -54,7 +55,7 @@ static uint32_t sysimg_init_cb(const void *id, jl_value_t **rejection_reason) return best_idx; } -static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason) +static uint32_t pkgimg_init_cb(void *ctx, const void *id, jl_value_t **rejection_reason) { TargetData<1> target = jit_targets.front(); // Find the last name match or use the default one. @@ -70,9 +71,9 @@ static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason) return best_idx; } -static void ensure_jit_target(bool imaging) +static void ensure_jit_target(const char *cpu_target, bool imaging) { - auto &cmdline = get_cmdline_targets(); + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, imaging); if (!jit_targets.empty()) return; @@ -115,25 +116,25 @@ get_llvm_target_str(const TargetData<1> &data) using namespace Fallback; -jl_image_t jl_init_processor_sysimg(void *hdl) +jl_image_t jl_init_processor_sysimg(jl_image_buf_t image, const char *cpu_target) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); - return parse_sysimg(hdl, sysimg_init_cb); + return parse_sysimg(image, sysimg_init_cb, (void *)cpu_target); } -jl_image_t jl_init_processor_pkgimg(void *hdl) +jl_image_t jl_init_processor_pkgimg(jl_image_buf_t image) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); if (jit_targets.size() > 1) jl_error("Expected only one JIT target"); - return parse_sysimg(hdl, pkgimg_init_cb); + return parse_sysimg(image, pkgimg_init_cb, NULL); } -std::pair> jl_get_llvm_target(bool imaging, uint32_t &flags) +std::pair> jl_get_llvm_target(const char *cpu_target, bool imaging, uint32_t &flags) { - ensure_jit_target(imaging); + ensure_jit_target(cpu_target, imaging); flags = jit_targets[0].en.flags; return get_llvm_target_vec(jit_targets[0]); } @@ -145,10 +146,10 @@ const std::pair &jl_get_llvm_disasm_target(void) return res; } #ifndef __clang_gcanalyzer__ -llvm::SmallVector jl_get_llvm_clone_targets(void) +llvm::SmallVector jl_get_llvm_clone_targets(const char *cpu_target) { - auto &cmdline = get_cmdline_targets(); + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, true); llvm::SmallVector, 0> image_targets; for (auto &arg: cmdline) { @@ -192,7 +193,7 @@ JL_DLLEXPORT jl_value_t* jl_check_pkgimage_clones(char *data) { jl_value_t *rejection_reason = NULL; JL_GC_PUSH1(&rejection_reason); - uint32_t match_idx = pkgimg_init_cb(data, &rejection_reason); + uint32_t match_idx = pkgimg_init_cb(NULL, data, &rejection_reason); JL_GC_POP(); if (match_idx == UINT32_MAX) return rejection_reason; diff --git a/src/processor_x86.cpp b/src/processor_x86.cpp index db954680289ea..dbd3de8bffa0c 100644 --- a/src/processor_x86.cpp +++ b/src/processor_x86.cpp @@ -782,7 +782,7 @@ static inline void disable_depends(FeatureList &features) ::disable_depends(features, Feature::deps, sizeof(Feature::deps) / sizeof(FeatureDep)); } -static const llvm::SmallVector, 0> &get_cmdline_targets(void) +static const llvm::SmallVector, 0> &get_cmdline_targets(const char *cpu_target) { auto feature_cb = [] (const char *str, size_t len, FeatureList &list) { auto fbit = find_feature_bit(feature_names, nfeature_names, str, len); @@ -791,7 +791,7 @@ static const llvm::SmallVector, 0> &get_cmdline_targets(v set_bit(list, fbit, true); return true; }; - auto &targets = ::get_cmdline_targets(feature_cb); + auto &targets = ::get_cmdline_targets(cpu_target, feature_cb); for (auto &t: targets) { if (auto nname = normalize_cpu_name(t.name)) { t.name = nname; @@ -851,10 +851,11 @@ static int max_vector_size(const FeatureList &features) return 16; } -static uint32_t sysimg_init_cb(const void *id, jl_value_t** rejection_reason) +static uint32_t sysimg_init_cb(void *ctx, const void *id, jl_value_t** rejection_reason) { // First see what target is requested for the JIT. - auto &cmdline = get_cmdline_targets(); + const char *cpu_target = (const char *)ctx; + auto &cmdline = get_cmdline_targets(cpu_target); TargetData target = arg_target_data(cmdline[0], true); // Then find the best match in the sysimg auto sysimg = deserialize_target_data((const uint8_t*)id); @@ -897,7 +898,7 @@ static uint32_t sysimg_init_cb(const void *id, jl_value_t** rejection_reason) return match.best_idx; } -static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason) +static uint32_t pkgimg_init_cb(void *ctx, const void *id, jl_value_t **rejection_reason) { TargetData target = jit_targets.front(); auto pkgimg = deserialize_target_data((const uint8_t*)id); @@ -912,9 +913,9 @@ static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason) //This function serves as a fallback during bootstrapping, at that point we don't have a sysimage with native code // so we won't call sysimg_init_cb, else this function shouldn't do anything. -static void ensure_jit_target(bool imaging) +static void ensure_jit_target(const char *cpu_target, bool imaging) { - auto &cmdline = get_cmdline_targets(); + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, imaging); if (!jit_targets.empty()) return; @@ -1058,7 +1059,7 @@ JL_DLLEXPORT jl_value_t* jl_check_pkgimage_clones(char *data) { jl_value_t *rejection_reason = NULL; JL_GC_PUSH1(&rejection_reason); - uint32_t match_idx = pkgimg_init_cb(data, &rejection_reason); + uint32_t match_idx = pkgimg_init_cb(NULL, data, &rejection_reason); JL_GC_POP(); if (match_idx == UINT32_MAX) return rejection_reason; @@ -1075,25 +1076,25 @@ JL_DLLEXPORT jl_value_t *jl_cpu_has_fma(int bits) return jl_false; } -jl_image_t jl_init_processor_sysimg(void *hdl) +jl_image_t jl_init_processor_sysimg(jl_image_buf_t image, const char *cpu_target) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); - return parse_sysimg(hdl, sysimg_init_cb); + return parse_sysimg(image, sysimg_init_cb, (void *)cpu_target); } -jl_image_t jl_init_processor_pkgimg(void *hdl) +jl_image_t jl_init_processor_pkgimg(jl_image_buf_t image) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); if (jit_targets.size() > 1) jl_error("Expected only one JIT target"); - return parse_sysimg(hdl, pkgimg_init_cb); + return parse_sysimg(image, pkgimg_init_cb, NULL); } -std::pair> jl_get_llvm_target(bool imaging, uint32_t &flags) +std::pair> jl_get_llvm_target(const char *cpu_target, bool imaging, uint32_t &flags) { - ensure_jit_target(imaging); + ensure_jit_target(cpu_target, imaging); flags = jit_targets[0].en.flags; return get_llvm_target_vec(jit_targets[0]); } @@ -1106,9 +1107,10 @@ const std::pair &jl_get_llvm_disasm_target(void) } //This function parses the -C command line to figure out which targets to multiversion to. #ifndef __clang_gcanalyzer__ -llvm::SmallVector jl_get_llvm_clone_targets(void) +llvm::SmallVector jl_get_llvm_clone_targets(const char *cpu_target) { - auto &cmdline = get_cmdline_targets(); + + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, true); llvm::SmallVector, 0> image_targets; for (auto &arg: cmdline) { diff --git a/src/staticdata.c b/src/staticdata.c index cac3b78545cca..339e79f91f76e 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -74,6 +74,8 @@ External links: #include // printf #include // PRIxPTR +#include + #include "julia.h" #include "julia_internal.h" #include "julia_gcext.h" @@ -81,8 +83,11 @@ External links: #include "processor.h" #include "serialize.h" -#ifndef _OS_WINDOWS_ +#ifdef _OS_WINDOWS_ +#include +#else #include +#include #endif #include "valgrind.h" @@ -607,8 +612,7 @@ typedef struct { } pkgcachesizes; // --- Static Compile --- -static void *jl_sysimg_handle = NULL; -static jl_image_t sysimage; +static jl_image_buf_t jl_sysimage_buf = { JL_IMAGE_KIND_NONE }; static inline uintptr_t *sysimg_gvars(const char *base, const int32_t *offsets, size_t idx) { @@ -620,28 +624,6 @@ JL_DLLEXPORT int jl_running_on_valgrind(void) return RUNNING_ON_VALGRIND; } -void *system_image_data_unavailable; -extern void * JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(system_image_data_unavailable) jl_system_image_data; -extern void * JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(system_image_data_unavailable) jl_system_image_size; -static void jl_load_sysimg_so(void) -{ - const char *sysimg_data; - assert(sysimage.fptrs.ptrs); // jl_init_processor_sysimg should already be run - if (jl_sysimg_handle == jl_exe_handle && - &jl_system_image_data != JL_WEAK_SYMBOL_DEFAULT(system_image_data_unavailable)) - sysimg_data = (const char*)&jl_system_image_data; - else - jl_dlsym(jl_sysimg_handle, "jl_system_image_data", (void **)&sysimg_data, 1); - size_t *plen; - if (jl_sysimg_handle == jl_exe_handle && - &jl_system_image_size != JL_WEAK_SYMBOL_DEFAULT(system_image_data_unavailable)) - plen = (size_t *)&jl_system_image_size; - else - jl_dlsym(jl_sysimg_handle, "jl_system_image_size", (void **)&plen, 1); - jl_restore_system_image_data(sysimg_data, *plen); -} - - // --- serializer --- #define NBOX_C 1024 @@ -3057,33 +3039,172 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli JL_DLLEXPORT size_t ios_write_direct(ios_t *dest, ios_t *src); -// Takes in a path of the form "usr/lib/julia/sys.so" (jl_restore_system_image should be passed the same string) -JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname) +// Takes in a path of the form "usr/lib/julia/sys.so" +JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname) { - if (jl_sysimg_handle) - return; // embedded target already called jl_set_sysimg_so + if (jl_sysimage_buf.kind != JL_IMAGE_KIND_NONE) + return jl_sysimage_buf; char *dot = (char*) strrchr(fname, '.'); int is_ji = (dot && !strcmp(dot, ".ji")); - // Get handle to sys.so - if (!is_ji) // .ji extension => load .ji file only - jl_set_sysimg_so(jl_load_dynamic_library(fname, JL_RTLD_LOCAL | JL_RTLD_NOW, 1)); + if (is_ji) { + // .ji extension => load .ji file only + ios_t f; + + if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) + jl_errorf("System image file \"%s\" not found.", fname); + ios_bufmode(&f, bm_none); + + JL_SIGATOMIC_BEGIN(); + + ios_seek_end(&f); + size_t len = ios_pos(&f); + char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); + ios_seek(&f, 0); + + if (ios_readall(&f, sysimg, len) != len) + jl_errorf("Error reading system image file."); + + ios_close(&f); + + JL_SIGATOMIC_END(); + + jl_sysimage_buf = (jl_image_buf_t) { + .kind = JL_IMAGE_KIND_JI, + .handle = NULL, + .pointers = NULL, + .data = sysimg, + .size = len, + .base = 0, + }; + return jl_sysimage_buf; + } else { + // Get handle to sys.so + return jl_set_sysimg_so(jl_load_dynamic_library(fname, JL_RTLD_LOCAL | JL_RTLD_NOW, 1)); + } } -// Allow passing in a module handle directly, rather than a path -JL_DLLEXPORT void jl_set_sysimg_so(void *handle) + +static void jl_prefetch_system_image(const char *data, size_t size) +{ + size_t page_size = jl_getpagesize(); /* jl_page_size is not set yet when loading sysimg */ + void *start = (void *)((uintptr_t)data & ~(page_size - 1)); + size_t size_aligned = LLT_ALIGN(size, page_size); +#ifdef _OS_WINDOWS_ + WIN32_MEMORY_RANGE_ENTRY entry = {start, size_aligned}; + PrefetchVirtualMemory(GetCurrentProcess(), 1, &entry, 0); +#else + madvise(start, size_aligned, MADV_WILLNEED); +#endif +} + +JL_DLLEXPORT void jl_image_unpack_uncomp(void *handle, jl_image_buf_t *image) { + size_t *plen; + jl_dlsym(handle, "jl_system_image_size", (void **)&plen, 1); + jl_dlsym(handle, "jl_system_image_data", (void **)&image->data, 1); + jl_dlsym(handle, "jl_image_pointers", (void**)&image->pointers, 1); + image->size = *plen; + jl_prefetch_system_image(image->data, image->size); +} + +JL_DLLEXPORT void jl_image_unpack_zstd(void *handle, jl_image_buf_t *image) +{ + size_t *plen; + const char *data; + jl_dlsym(handle, "jl_system_image_size", (void **)&plen, 1); + jl_dlsym(handle, "jl_system_image_data", (void **)&data, 1); + jl_dlsym(handle, "jl_image_pointers", (void **)&image->pointers, 1); + jl_prefetch_system_image(data, *plen); + image->size = ZSTD_getFrameContentSize(data, *plen); + size_t page_size = jl_getpagesize(); /* jl_page_size is not set yet when loading sysimg */ + size_t aligned_size = LLT_ALIGN(image->size, page_size); +#if defined(_OS_WINDOWS_) + size_t large_page_size = GetLargePageMinimum(); + if (image->size > 4 * large_page_size) { + size_t aligned_size = LLT_ALIGN(image->size, large_page_size); + image->data = (char *)VirtualAlloc( + NULL, aligned_size, MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES, PAGE_READWRITE); + } + else { + image->data = (char *)VirtualAlloc(NULL, aligned_size, MEM_COMMIT | MEM_RESERVE, + PAGE_READWRITE); + } +#else + image->data = + (char *)mmap(NULL, aligned_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); +#endif + if (!image->data || image->data == (void *)-1) { + jl_printf(JL_STDERR, "ERROR: failed to allocate space for system image\n"); + jl_exit(1); + } + + ZSTD_decompress((void *)image->data, image->size, data, *plen); + size_t len = (*plen) & ~(page_size - 1); +#ifdef _OS_WINDOWS_ + if (len) + VirtualFree((void *)data, len, MEM_RELEASE); +#else + munmap((void *)data, len); +#endif +} + +// From a shared library handle, verify consistency and return a jl_image_buf_t +static jl_image_buf_t get_image_buf(void *handle, int is_pkgimage) +{ + // verify that the linker resolved the symbols in this image against ourselves (libjulia-internal) void** (*get_jl_RTLD_DEFAULT_handle_addr)(void) = NULL; if (handle != jl_RTLD_DEFAULT_handle) { int symbol_found = jl_dlsym(handle, "get_jl_RTLD_DEFAULT_handle_addr", (void **)&get_jl_RTLD_DEFAULT_handle_addr, 0); if (!symbol_found || (void*)&jl_RTLD_DEFAULT_handle != (get_jl_RTLD_DEFAULT_handle_addr())) - jl_error("System image file failed consistency check: maybe opened the wrong version?"); + jl_error("Image file failed consistency check: maybe opened the wrong version?"); } - if (jl_options.cpu_target == NULL) - jl_options.cpu_target = "native"; - jl_sysimg_handle = handle; - sysimage = jl_init_processor_sysimg(handle); + + jl_image_unpack_func_t **unpack; + jl_image_buf_t image = { + .kind = JL_IMAGE_KIND_SO, + .handle = handle, + .pointers = NULL, + .data = NULL, + .size = 0, + .base = 0, + }; + + // verification passed, lookup the buffer pointers + if (jl_image_unpack == NULL || is_pkgimage) { + // in the usual case, the sysimage was not statically linked to libjulia-internal + // look up the external sysimage symbols via the dynamic linker + jl_dlsym(handle, "jl_image_unpack", (void **)&unpack, 1); + } + else { + // the sysimage was statically linked directly against libjulia-internal + // use the internal symbols + unpack = &jl_image_unpack; + } + (*unpack)(handle, &image); + +#ifdef _OS_WINDOWS_ + image.base = (intptr_t)handle; +#else + Dl_info dlinfo; + if (dladdr((void*)image.pointers, &dlinfo) != 0) + image.base = (intptr_t)dlinfo.dli_fbase; + else + image.base = 0; +#endif + + return image; +} + +// Allow passing in a module handle directly, rather than a path +JL_DLLEXPORT jl_image_buf_t jl_set_sysimg_so(void *handle) +{ + if (jl_sysimage_buf.kind != JL_IMAGE_KIND_NONE) + return jl_sysimage_buf; + + jl_sysimage_buf = get_image_buf(handle, /* is_pkgimage */ 0); + return jl_sysimage_buf; } #ifndef JL_NDEBUG @@ -3103,7 +3224,8 @@ extern void export_jl_small_typeof(void); // into the native code of the image. See https://github.com/JuliaLang/julia/pull/52123#issuecomment-1959965395. int IMAGE_NATIVE_CODE_TAINTED = 0; -static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_array_t *depmods, uint64_t checksum, +static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, void *image_handle, + jl_array_t *depmods, uint64_t checksum, /* outputs */ jl_array_t **restored, jl_array_t **init_order, jl_array_t **extext_methods, jl_array_t **internal_methods, jl_array_t **new_ext_cis, jl_array_t **method_roots_list, @@ -3568,8 +3690,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl s.s = &sysimg; jl_update_all_fptrs(&s, image); // fptr relocs and registration if (!ccallable_list) { - // TODO: jl_sysimg_handle or img_handle? - jl_reinit_ccallable(&s.ccallable_list, image_base, jl_sysimg_handle); + jl_reinit_ccallable(&s.ccallable_list, image_base, image_handle); arraylist_free(&s.ccallable_list); } s.s = NULL; @@ -3667,7 +3788,7 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i ios_close(f); ios_static_buffer(f, sysimg, len); pkgcachesizes cachesizes; - jl_restore_system_image_from_stream_(f, image, depmods, checksum, (jl_array_t**)&restored, &init_order, &extext_methods, &internal_methods, &new_ext_cis, &method_roots_list, + jl_restore_system_image_from_stream_(f, image, pkgimage_handle, depmods, checksum, (jl_array_t**)&restored, &init_order, &extext_methods, &internal_methods, &new_ext_cis, &method_roots_list, &ext_targets, &edges, &base, &ccallable_list, &cachesizes); JL_SIGATOMIC_END(); @@ -3716,16 +3837,16 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i return restored; } -static void jl_restore_system_image_from_stream(ios_t *f, jl_image_t *image, uint32_t checksum) +static void jl_restore_system_image_from_stream(ios_t *f, jl_image_t *image, void *image_handle, uint32_t checksum) { JL_TIMING(LOAD_IMAGE, LOAD_Sysimg); - jl_restore_system_image_from_stream_(f, image, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + jl_restore_system_image_from_stream_(f, image, image_handle, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } -JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(void* pkgimage_handle, const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int completeinfo, const char *pkgname, int needs_permalloc) +JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(void* pkgimage_handle, jl_image_buf_t buf, jl_image_t *image, jl_array_t *depmods, int completeinfo, const char *pkgname, int needs_permalloc) { ios_t f; - ios_static_buffer(&f, (char*)buf, sz); + ios_static_buffer(&f, (char*)buf.data, buf.size); jl_value_t *ret = jl_restore_package_image_from_stream(pkgimage_handle, &f, image, depmods, completeinfo, pkgname, needs_permalloc); ios_close(&f); return ret; @@ -3744,47 +3865,22 @@ JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *d return ret; } -// TODO: need to enforce that the alignment of the buffer is suitable for vectors -JL_DLLEXPORT void jl_restore_system_image(const char *fname) +JL_DLLEXPORT void jl_restore_system_image(jl_image_t *image, jl_image_buf_t buf) { -#ifndef JL_NDEBUG - char *dot = fname ? (char*)strrchr(fname, '.') : NULL; - int is_ji = (dot && !strcmp(dot, ".ji")); - assert((is_ji || jl_sysimg_handle) && "System image file not preloaded"); -#endif + ios_t f; - if (jl_sysimg_handle) { - // load the pre-compiled sysimage from jl_sysimg_handle - jl_load_sysimg_so(); - } - else { - ios_t f; - if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) - jl_errorf("System image file \"%s\" not found.", fname); - ios_bufmode(&f, bm_none); - JL_SIGATOMIC_BEGIN(); - ios_seek_end(&f); - size_t len = ios_pos(&f); - char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); - ios_seek(&f, 0); - if (ios_readall(&f, sysimg, len) != len) - jl_errorf("Error reading system image file."); - ios_close(&f); - uint32_t checksum = jl_crc32c(0, sysimg, len); - ios_static_buffer(&f, sysimg, len); - jl_restore_system_image_from_stream(&f, &sysimage, checksum); - ios_close(&f); - JL_SIGATOMIC_END(); - } -} + if (buf.kind == JL_IMAGE_KIND_NONE) + return; + + if (buf.kind == JL_IMAGE_KIND_SO) + assert(image->fptrs.ptrs); // jl_init_processor_sysimg should already be run -JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len) -{ - ios_t f; JL_SIGATOMIC_BEGIN(); - ios_static_buffer(&f, (char*)buf, len); - uint32_t checksum = jl_crc32c(0, buf, len); - jl_restore_system_image_from_stream(&f, &sysimage, checksum); + ios_static_buffer(&f, (char *)buf.data, buf.size); + + uint32_t checksum = jl_crc32c(0, buf.data, buf.size); + jl_restore_system_image_from_stream(&f, image, buf.handle, checksum); + ios_close(&f); JL_SIGATOMIC_END(); } @@ -3803,12 +3899,11 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j #endif jl_errorf("Error opening package file %s: %s\n", fname, reason); } - const char *pkgimg_data; - jl_dlsym(pkgimg_handle, "jl_system_image_data", (void **)&pkgimg_data, 1); - size_t *plen; - jl_dlsym(pkgimg_handle, "jl_system_image_size", (void **)&plen, 1); - jl_image_t pkgimage = jl_init_processor_pkgimg(pkgimg_handle); + jl_image_buf_t buf = get_image_buf(pkgimg_handle, /* is_pkgimage */ 1); + + // Despite the name, this function actually parses the pkgimage + jl_image_t pkgimage = jl_init_processor_pkgimg(buf); if (ignore_native) { // Must disable using native code in possible downstream users of this code: @@ -3817,7 +3912,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j IMAGE_NATIVE_CODE_TAINTED = 1; } - jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_handle, pkgimg_data, &pkgimage, *plen, depmods, completeinfo, pkgname, 0); + jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_handle, buf, &pkgimage, depmods, completeinfo, pkgname, 0); return mod; } diff --git a/src/threading.c b/src/threading.c index 62af7e60bf0f1..a17d3b371acf5 100644 --- a/src/threading.c +++ b/src/threading.c @@ -250,10 +250,6 @@ void jl_set_pgcstack(jl_gcframe_t **pgcstack) JL_NOTSAFEPOINT { *jl_pgcstack_key() = pgcstack; } -# if JL_USE_IFUNC -JL_DLLEXPORT __attribute__((weak)) -void jl_register_pgcstack_getter(void); -# endif static jl_gcframe_t **jl_get_pgcstack_init(void); static jl_get_pgcstack_func *jl_get_pgcstack_cb = jl_get_pgcstack_init; static jl_gcframe_t **jl_get_pgcstack_init(void) @@ -266,15 +262,8 @@ static jl_gcframe_t **jl_get_pgcstack_init(void) // This is clearly not thread-safe but should be fine since we // make sure the tls states callback is finalized before adding // multiple threads -# if JL_USE_IFUNC - if (jl_register_pgcstack_getter) - jl_register_pgcstack_getter(); - else -# endif - { - jl_get_pgcstack_cb = jl_get_pgcstack_fallback; - jl_pgcstack_key = &jl_pgcstack_addr_fallback; - } + jl_get_pgcstack_cb = jl_get_pgcstack_fallback; + jl_pgcstack_key = &jl_pgcstack_addr_fallback; return jl_get_pgcstack_cb(); } diff --git a/stdlib/Makefile b/stdlib/Makefile index d478d2ad4c188..bdb6f8a733200 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -15,11 +15,13 @@ include $(JULIAHOME)/deps/*.version VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) DIRS := $(build_datarootdir)/julia/stdlib/$(VERSDIR) $(build_prefix)/manifest/$(VERSDIR) +LIBDIR := $(build_datarootdir)/lib/julia + $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) JLLS = DSFMT GMP CURL LIBGIT2 LLVM LIBSSH2 LIBUV MBEDTLS MPFR NGHTTP2 \ BLASTRAMPOLINE OPENBLAS OPENLIBM P7ZIP PCRE LIBSUITESPARSE ZLIB \ - LLVMUNWIND CSL UNWIND LLD + ZSTD LLVMUNWIND CSL UNWIND LLD # Initialize this with JLLs that aren't in "deps/$(LibName).version" JLL_NAMES := MozillaCACerts_jll @@ -61,8 +63,19 @@ $(foreach module, $(STDLIBS), $(eval $(call symlink_target,$$(JULIAHOME)/stdlib/ STDLIBS_LINK_TARGETS := $(addprefix $(build_datarootdir)/julia/stdlib/$(VERSDIR)/,$(STDLIBS)) +remove-gpl-libs: +ifeq ($(USE_GPL_LIBS),0) + @echo Removing GPL libs... + -rm -f $(LIBDIR)/libcholmod* + -rm -f $(LIBDIR)/libklu_cholmod* + -rm -f $(LIBDIR)/librbio* + -rm -f $(LIBDIR)/libspqr* + -rm -f $(LIBDIR)/libumfpack* +endif + getall get: $(addprefix get-, $(STDLIBS_EXT) $(JLL_NAMES)) -install: version-check $(addprefix install-, $(STDLIBS_EXT) $(JLL_NAMES)) $(STDLIBS_LINK_TARGETS) + +install: version-check $(addprefix install-, $(STDLIBS_EXT) $(JLL_NAMES)) $(STDLIBS_LINK_TARGETS) remove-gpl-libs version-check: $(addprefix version-check-, $(STDLIBS_EXT)) uninstall: $(addprefix uninstall-, $(STDLIBS_EXT)) extstdlibclean: diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index 5ad675ac21510..489baa7b9732d 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -268,6 +268,11 @@ deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" version = "1.2.13+1" +[[deps.Zstd_jll]] +deps = ["Libdl"] +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + [[deps.dSFMT_jll]] deps = ["Artifacts", "Libdl"] uuid = "05ff407c-b0c1-5878-9df8-858cc2e60c36" diff --git a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl index e5edfa76997e1..9e03033c4e3fa 100644 --- a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl +++ b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl @@ -81,29 +81,34 @@ else end function __init__() + # BSD-3-Clause + global libamd_handle = dlopen(libamd) + global libamd_path = dlpath(libamd_handle) + global libcamd_handle = dlopen(libcamd) + global libcamd_path = dlpath(libcamd_handle) + global libccolamd_handle = dlopen(libccolamd) + global libccolamd_path = dlpath(libccolamd_handle) + global libcolamd_handle = dlopen(libcolamd) + global libcolamd_path = dlpath(libcolamd_handle) + global libsuitesparseconfig_handle = dlopen(libsuitesparseconfig) + global libsuitesparseconfig_path = dlpath(libsuitesparseconfig_handle) + + # LGPL-2.1+ + global libbtf_handle = dlopen(libbtf) + global libbtf_path = dlpath(libbtf_handle) + global libklu_handle = dlopen(libklu) + global libklu_path = dlpath(libklu_handle) + global libldl_handle = dlopen(libldl) + global libldl_path = dlpath(libldl_handle) + + # GPL-2.0+ if Base.USE_GPL_LIBS - global libamd_handle = dlopen(libamd) - global libamd_path = dlpath(libamd_handle) - global libbtf_handle = dlopen(libbtf) - global libbtf_path = dlpath(libbtf_handle) - global libcamd_handle = dlopen(libcamd) - global libcamd_path = dlpath(libcamd_handle) - global libccolamd_handle = dlopen(libccolamd) - global libccolamd_path = dlpath(libccolamd_handle) global libcholmod_handle = dlopen(libcholmod) global libcholmod_path = dlpath(libcholmod_handle) - global libcolamd_handle = dlopen(libcolamd) - global libcolamd_path = dlpath(libcolamd_handle) - global libklu_handle = dlopen(libklu) - global libklu_path = dlpath(libklu_handle) - global libldl_handle = dlopen(libldl) - global libldl_path = dlpath(libldl_handle) global librbio_handle = dlopen(librbio) global librbio_path = dlpath(librbio_handle) global libspqr_handle = dlopen(libspqr) global libspqr_path = dlpath(libspqr_handle) - global libsuitesparseconfig_handle = dlopen(libsuitesparseconfig) - global libsuitesparseconfig_path = dlpath(libsuitesparseconfig_handle) global libumfpack_handle = dlopen(libumfpack) global libumfpack_path = dlpath(libumfpack_handle) end diff --git a/stdlib/Zstd_jll/Project.toml b/stdlib/Zstd_jll/Project.toml new file mode 100644 index 0000000000000..467516843390a --- /dev/null +++ b/stdlib/Zstd_jll/Project.toml @@ -0,0 +1,15 @@ +name = "Zstd_jll" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[deps] +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[compat] +julia = "1.6" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/stdlib/Zstd_jll/src/Zstd_jll.jl b/stdlib/Zstd_jll/src/Zstd_jll.jl new file mode 100644 index 0000000000000..c16413f963d0b --- /dev/null +++ b/stdlib/Zstd_jll/src/Zstd_jll.jl @@ -0,0 +1,73 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## dummy stub for https://github.com/JuliaBinaryWrappers/Zstd_jll.j: +# +baremodule Zstd_jll +using Base, Libdl + +export libzstd, zstd, zstdmt + +# These get calculated in __init__() +libzstd_handle::Ptr{Cvoid} = C_NULL + +if Sys.iswindows() + const libzstd = "libzstd-1.dll" +elseif Sys.isapple() + const libzstd = "@rpath/libzstd.1.dylib" +else + const libzstd = "libzstd.so.1" +end + +if Sys.iswindows() + const zstd_exe = "zstd.exe" + const zstdmt_exe = "zstdmt.exe" +else + const zstd_exe = "zstd" + const zstdmt_exe = "zstdmt" +end + +if Sys.iswindows() + const pathsep = ';' +elseif Sys.isapple() + const pathsep = ':' +else + const pathsep = ':' +end + +if Sys.iswindows() +function adjust_ENV(cmd::Cmd) + dllPATH = Sys.BINDIR + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? dllPATH : "$dllPATH$pathsep$oldPATH" + return addenv(cmd, "PATH"=>newPATH) +end +else +adjust_ENV(cmd::Cmd) = cmd +end + +function adjust_ENV() + addPATH = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR) + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? addPATH : "$addPATH$pathsep$oldPATH" + return ("PATH"=>newPATH,) +end + +function zstd(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstd()) + end +end +function zstdmt(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstdmt()) + end +end +zstd() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstd_exe))`) +zstdmt() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstdmt_exe))`) + +function __init__() + global libzstd_handle = dlopen(libzstd) + nothing +end + +end # module Zstd_jll diff --git a/stdlib/Zstd_jll/test/runtests.jl b/stdlib/Zstd_jll/test/runtests.jl new file mode 100644 index 0000000000000..5cfa2a1375c73 --- /dev/null +++ b/stdlib/Zstd_jll/test/runtests.jl @@ -0,0 +1,7 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test, Zstd_jll + +@testset "Zstd_jll" begin + @test ccall((:ZSTD_versionNumber, libzstd), Cuint, ()) == 1_05_07 +end diff --git a/stdlib/stdlib.mk b/stdlib/stdlib.mk index 021639ec0bad0..9670fe49bc0fe 100644 --- a/stdlib/stdlib.mk +++ b/stdlib/stdlib.mk @@ -9,7 +9,7 @@ INDEPENDENT_STDLIBS := \ SparseArrays Statistics StyledStrings SuiteSparse_jll Tar Test TOML Unicode UUIDs \ dSFMT_jll GMP_jll libLLVM_jll LLD_jll LLVMLibUnwind_jll LibUnwind_jll LibUV_jll \ LibCURL_jll LibSSH2_jll LibGit2_jll nghttp2_jll MozillaCACerts_jll MbedTLS_jll \ - MPFR_jll OpenLibm_jll PCRE2_jll p7zip_jll Zlib_jll + MPFR_jll OpenLibm_jll PCRE2_jll p7zip_jll Zlib_jll Zstd_jll STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS) VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) diff --git a/test/choosetests.jl b/test/choosetests.jl index 36da1ab39bc57..a5dff61a5ec9d 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -227,7 +227,7 @@ function choosetests(choices = []) # Filter out tests from the test groups in the stdlibs filter!(!in(tests), unhandled) filter!(!in(skip_tests), tests) - + filter!(!contains("SparseArrays/fixed"), tests) # temporarily disable SparseArrays/fixed if !isempty(unhandled) @warn "Not skipping tests: $(join(unhandled, ", "))" end diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 43cdf5e2696a6..1c70c5d3f2cab 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -979,7 +979,7 @@ let exename = `$(Base.julia_cmd().exec[1]) -t 1` p = run(pipeline(`$exename --sysimage=$libjulia`, stderr=err), wait=false) close(err.in) let s = read(err, String) - @test s == "ERROR: System image file failed consistency check: maybe opened the wrong version?\n" + @test s == "ERROR: Image file failed consistency check: maybe opened the wrong version?\n" end @test errors_not_signals(p) @test p.exitcode == 1