From de03507853b8fab2fb08a3f540acd5849b61684c Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Wed, 26 Feb 2025 19:15:21 -0500 Subject: [PATCH 01/18] Remove usages of weak symbols (#57522) Inspired by a bug I hit on MinGW recently with weak symbol resolution. I'm not sure why we started using these in 70000ac7c3d5d5f21e42555cdf99e699a246f8ec, since I believe we should be able to lookup these symbols just fine in our `jl_exe_handle` as long as it was linked / compiled with `-rdynamic` Migrating away from weak symbols seems a good idea, given their uneven implementation across platforms. (cherry picked from commit 4a6ada6eb36eaeb9bde9b94c01bcf6f0e3ce7ae6) --- src/Makefile | 8 +++++--- src/julia_internal.h | 18 ------------------ src/null_sysimage.c | 15 +++++++++++++++ src/processor.cpp | 11 +++-------- src/processor.h | 12 ++++++++++++ src/staticdata.c | 27 ++++++++++++++------------- src/threading.c | 15 ++------------- 7 files changed, 51 insertions(+), 55 deletions(-) create mode 100644 src/null_sysimage.c diff --git a/src/Makefile b/src/Makefile index bf9001e5fba93..c25649d2cab7c 100644 --- a/src/Makefile +++ b/src/Makefile @@ -46,7 +46,8 @@ SRCS := \ simplevector runtime_intrinsics precompile jloptions mtarraylist \ threading scheduler stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \ jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \ - crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall + crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall \ + null_sysimage RT_LLVMLINK := CG_LLVMLINK := @@ -56,7 +57,8 @@ CODEGEN_SRCS := codegen jitlayers aotcompile debuginfo disasm llvm-simdloop llvm llvm-final-gc-lowering llvm-pass-helpers llvm-late-gc-lowering llvm-ptls \ llvm-lower-handlers llvm-gc-invariant-verifier llvm-propagate-addrspaces \ llvm-multiversioning llvm-alloc-opt llvm-alloc-helpers cgmemmgr llvm-remove-addrspaces \ - llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures pipeline llvm_api + llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures pipeline llvm_api \ + null_sysimage FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --includedir) CG_LLVM_LIBS := all ifeq ($(USE_POLLY),1) @@ -160,7 +162,7 @@ endif CLANG_LDFLAGS := $(LLVM_LDFLAGS) ifeq ($(OS), Darwin) CLANG_LDFLAGS += -Wl,-undefined,dynamic_lookup -OSLIBS += -Wl,-U,__dyld_atfork_parent -Wl,-U,__dyld_atfork_prepare -Wl,-U,__dyld_dlopen_atfork_parent -Wl,-U,__dyld_dlopen_atfork_prepare -Wl,-U,_jl_image_pointers -Wl,-U,_jl_system_image_data -Wl,-U,_jl_system_image_size +OSLIBS += -Wl,-U,__dyld_atfork_parent -Wl,-U,__dyld_atfork_prepare -Wl,-U,__dyld_dlopen_atfork_parent -Wl,-U,__dyld_dlopen_atfork_prepare LIBJULIA_PATH_REL := @rpath/libjulia else LIBJULIA_PATH_REL := libjulia diff --git a/src/julia_internal.h b/src/julia_internal.h index 72d9dfdad5c7e..dff769e7e0f66 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1710,24 +1710,6 @@ jl_sym_t *_jl_symbol(const char *str, size_t len) JL_NOTSAFEPOINT; #define JL_GC_ASSERT_LIVE(x) (void)(x) #endif -#ifdef _OS_WINDOWS_ -// On Windows, weak symbols do not default to 0 due to a GCC bug -// (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90826), use symbol -// aliases with a known value instead. -#define JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(sym) __attribute__((weak,alias(#sym))) -#define JL_WEAK_SYMBOL_DEFAULT(sym) &sym -#else -#define JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(sym) __attribute__((weak)) -#define JL_WEAK_SYMBOL_DEFAULT(sym) NULL -#endif - -//JL_DLLEXPORT float julia__gnu_h2f_ieee(half param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT half julia__gnu_f2h_ieee(float param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT half julia__truncdfhf2(double param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT float julia__truncsfbf2(float param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT float julia__truncdfbf2(double param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT double julia__extendhfdf2(half n) JL_NOTSAFEPOINT; - JL_DLLEXPORT uint32_t jl_crc32c(uint32_t crc, const char *buf, size_t len); // -- exports from codegen -- // diff --git a/src/null_sysimage.c b/src/null_sysimage.c new file mode 100644 index 0000000000000..386842f0c4e77 --- /dev/null +++ b/src/null_sysimage.c @@ -0,0 +1,15 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include +#include "processor.h" + +/** + * These symbols support statically linking the sysimage with libjulia-internal. + * + * Here we provide dummy definitions that are used when these are not linked + * together (the default build configuration). The 0 value of jl_system_image_size + * is used as a sentinel to indicate that the sysimage should be loaded externally. + **/ +char jl_system_image_data = 0; +size_t jl_system_image_size = 0; +jl_image_pointers_t jl_image_pointers = { 0 }; diff --git a/src/processor.cpp b/src/processor.cpp index 730e470f4153d..ed50c609895cc 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -619,11 +619,6 @@ static inline llvm::SmallVector, 0> &get_cmdline_targets(F &&featu return targets; } -extern "C" { -void *image_pointers_unavailable; -extern void * JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(image_pointers_unavailable) jl_image_pointers; -} - // Load sysimg, use the `callback` for dispatch and perform all relocations // for the selected target. template @@ -633,10 +628,10 @@ static inline jl_image_t parse_sysimg(void *hdl, F &&callback) jl_image_t res{}; const jl_image_pointers_t *pointers; - if (hdl == jl_exe_handle && &jl_image_pointers != JL_WEAK_SYMBOL_DEFAULT(image_pointers_unavailable)) - pointers = (const jl_image_pointers_t *)&jl_image_pointers; - else + if (jl_system_image_size == 0) jl_dlsym(hdl, "jl_image_pointers", (void**)&pointers, 1); + else + pointers = &jl_image_pointers; // libjulia-internal and sysimage statically linked const void *ids = pointers->target_data; jl_value_t* rejection_reason = nullptr; diff --git a/src/processor.h b/src/processor.h index 82a1121aaf7c4..214f51845b12d 100644 --- a/src/processor.h +++ b/src/processor.h @@ -224,6 +224,18 @@ JL_DLLEXPORT int32_t jl_set_zero_subnormals(int8_t isZero); JL_DLLEXPORT int32_t jl_get_zero_subnormals(void); JL_DLLEXPORT int32_t jl_set_default_nans(int8_t isDefault); JL_DLLEXPORT int32_t jl_get_default_nans(void); + +/** + * System image contents. + * + * These symbols are typically dummy values, unless statically linking + * libjulia-* and the sysimage together (see null_sysimage.c), in which + * case they allow accessing the local copy of the sysimage. + **/ +extern char jl_system_image_data; +extern size_t jl_system_image_size; +extern jl_image_pointers_t jl_image_pointers; + #ifdef __cplusplus } diff --git a/src/staticdata.c b/src/staticdata.c index cac3b78545cca..793a2ccd19574 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -620,24 +620,25 @@ JL_DLLEXPORT int jl_running_on_valgrind(void) return RUNNING_ON_VALGRIND; } -void *system_image_data_unavailable; -extern void * JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(system_image_data_unavailable) jl_system_image_data; -extern void * JL_WEAK_SYMBOL_OR_ALIAS_DEFAULT(system_image_data_unavailable) jl_system_image_size; static void jl_load_sysimg_so(void) { - const char *sysimg_data; assert(sysimage.fptrs.ptrs); // jl_init_processor_sysimg should already be run - if (jl_sysimg_handle == jl_exe_handle && - &jl_system_image_data != JL_WEAK_SYMBOL_DEFAULT(system_image_data_unavailable)) - sysimg_data = (const char*)&jl_system_image_data; - else - jl_dlsym(jl_sysimg_handle, "jl_system_image_data", (void **)&sysimg_data, 1); + size_t *plen; - if (jl_sysimg_handle == jl_exe_handle && - &jl_system_image_size != JL_WEAK_SYMBOL_DEFAULT(system_image_data_unavailable)) - plen = (size_t *)&jl_system_image_size; - else + const char *sysimg_data; + + if (jl_system_image_size == 0) { + // in the usual case, the sysimage was not statically linked to libjulia-internal + // look up the external sysimage symbols via the dynamic linker jl_dlsym(jl_sysimg_handle, "jl_system_image_size", (void **)&plen, 1); + jl_dlsym(jl_sysimg_handle, "jl_system_image_data", (void **)&sysimg_data, 1); + } else { + // the sysimage was statically linked directly against libjulia-internal + // use the internal symbols + plen = &jl_system_image_size; + sysimg_data = &jl_system_image_data; + } + jl_restore_system_image_data(sysimg_data, *plen); } diff --git a/src/threading.c b/src/threading.c index 62af7e60bf0f1..a17d3b371acf5 100644 --- a/src/threading.c +++ b/src/threading.c @@ -250,10 +250,6 @@ void jl_set_pgcstack(jl_gcframe_t **pgcstack) JL_NOTSAFEPOINT { *jl_pgcstack_key() = pgcstack; } -# if JL_USE_IFUNC -JL_DLLEXPORT __attribute__((weak)) -void jl_register_pgcstack_getter(void); -# endif static jl_gcframe_t **jl_get_pgcstack_init(void); static jl_get_pgcstack_func *jl_get_pgcstack_cb = jl_get_pgcstack_init; static jl_gcframe_t **jl_get_pgcstack_init(void) @@ -266,15 +262,8 @@ static jl_gcframe_t **jl_get_pgcstack_init(void) // This is clearly not thread-safe but should be fine since we // make sure the tls states callback is finalized before adding // multiple threads -# if JL_USE_IFUNC - if (jl_register_pgcstack_getter) - jl_register_pgcstack_getter(); - else -# endif - { - jl_get_pgcstack_cb = jl_get_pgcstack_fallback; - jl_pgcstack_key = &jl_pgcstack_addr_fallback; - } + jl_get_pgcstack_cb = jl_get_pgcstack_fallback; + jl_pgcstack_key = &jl_pgcstack_addr_fallback; return jl_get_pgcstack_cb(); } From b4c714a7127d69d3e3ea8d648a94310c6e2157e7 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Thu, 27 Feb 2025 22:20:02 -0500 Subject: [PATCH 02/18] staticdata: Refactor sysimage loading (#57542) Introduce `jl_image_buf_t` to represent the in-memory details of an unparsed system image and clean-up several global variables and split loading logic in staticdata.c so that we get (almost) everything we need from the sysimage handle at once. This allows sysimage loading to be separated into three phases: 1. Lookup the raw sysimage buffer as a `jl_image_buf_t` 2. For .so sysimages, parse the sysimage and initialize JIT targets, etc. 3. Finally load the sysimage into a `jl_image_t` Care was taken to preserve the existing behavior of calling `jl_set_sysimg_so` to configure the sysimage before initializing Julia, although this is likely to be next on the chopping block in a follow-up. Dependent on #57523. --- src/aotcompile.cpp | 2 +- src/init.c | 28 +++-- src/jitlayers.cpp | 2 +- src/jl_exported_funcs.inc | 3 +- src/julia.h | 26 ++++- src/llvm-multiversioning.cpp | 2 +- src/processor.cpp | 33 ++---- src/processor.h | 13 ++- src/processor_arm.cpp | 33 +++--- src/processor_fallback.cpp | 33 +++--- src/processor_x86.cpp | 34 +++--- src/staticdata.c | 208 ++++++++++++++++++++--------------- test/cmdlineargs.jl | 2 +- 13 files changed, 235 insertions(+), 184 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 97734852f9479..05853828d36ce 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -1833,7 +1833,7 @@ void jl_dump_native_impl(void *native_code, builder.CreateRet(ConstantInt::get(T_int32, 1)); } if (imaging_mode) { - auto specs = jl_get_llvm_clone_targets(); + auto specs = jl_get_llvm_clone_targets(jl_options.cpu_target); const uint32_t base_flags = has_veccall ? JL_TARGET_VEC_CALL : 0; SmallVector data; auto push_i32 = [&] (uint32_t v) { diff --git a/src/init.c b/src/init.c index a5c9a6b19f94d..3517aa9d8fa5a 100644 --- a/src/init.c +++ b/src/init.c @@ -847,19 +847,30 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ { JL_TIMING(JULIA_INIT, JULIA_INIT); jl_resolve_sysimg_location(rel); + // loads sysimg if available, and conditionally sets jl_options.cpu_target - if (rel == JL_IMAGE_IN_MEMORY) - jl_set_sysimg_so(jl_exe_handle); + jl_image_buf_t sysimage = { JL_IMAGE_KIND_NONE }; + if (rel == JL_IMAGE_IN_MEMORY) { + sysimage = jl_set_sysimg_so(jl_exe_handle); + jl_options.image_file = jl_options.julia_bin; + } else if (jl_options.image_file) - jl_preload_sysimg_so(jl_options.image_file); + sysimage = jl_preload_sysimg(jl_options.image_file); + if (jl_options.cpu_target == NULL) jl_options.cpu_target = "native"; - jl_init_codegen(); + // Parse image, perform relocations, and init JIT targets, etc. + jl_image_t parsed_image = jl_init_processor_sysimg(sysimage, jl_options.cpu_target); + + jl_init_codegen(); jl_init_common_symbols(); - if (jl_options.image_file) { - jl_restore_system_image(jl_options.image_file); + + if (sysimage.kind != JL_IMAGE_KIND_NONE) { + // Load the .ji or .so sysimage + jl_restore_system_image(&parsed_image, sysimage); } else { + // No sysimage provided, init a minimal environment jl_init_types(); jl_global_roots_list = (jl_genericmemory_t*)jl_an_empty_memory_any; jl_global_roots_keyset = (jl_genericmemory_t*)jl_an_empty_memory_any; @@ -868,7 +879,7 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ jl_init_flisp(); jl_init_serializer(); - if (!jl_options.image_file) { + if (sysimage.kind == JL_IMAGE_KIND_NONE) { jl_top_module = jl_core_module; jl_init_intrinsic_functions(); jl_init_primitives(); @@ -892,7 +903,8 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ jl_gc_enable(1); - if (jl_options.image_file && (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) { + if ((sysimage.kind != JL_IMAGE_KIND_NONE) && + (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) { jl_array_t *init_order = jl_module_init_order; JL_GC_PUSH1(&init_order); jl_module_init_order = NULL; diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index b3ed1e069de16..e418ce06a74a7 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1139,7 +1139,7 @@ namespace { options.ExplicitEmulatedTLS = true; #endif uint32_t target_flags = 0; - auto target = jl_get_llvm_target(imaging_default(), target_flags); + auto target = jl_get_llvm_target(jl_options.cpu_target, jl_generating_output(), target_flags); auto &TheCPU = target.first; SmallVector targetFeatures(target.second.begin(), target.second.end()); std::string errorstr; diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 05057cfd80861..66db481187b47 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -370,7 +370,7 @@ XX(jl_pointerset) \ XX(jl_pop_handler) \ XX(jl_pop_handler_noexcept) \ - XX(jl_preload_sysimg_so) \ + XX(jl_preload_sysimg) \ XX(jl_prepend_cwd) \ XX(jl_printf) \ XX(jl_print_backtrace) \ @@ -400,7 +400,6 @@ XX(jl_restore_incremental) \ XX(jl_restore_package_image_from_file) \ XX(jl_restore_system_image) \ - XX(jl_restore_system_image_data) \ XX(jl_rethrow) \ XX(jl_rethrow_other) \ XX(jl_running_on_valgrind) \ diff --git a/src/julia.h b/src/julia.h index 2bb8bc5d0b59f..b8783ff821d00 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2030,6 +2030,25 @@ typedef enum { JL_IMAGE_IN_MEMORY = 2 } JL_IMAGE_SEARCH; +typedef enum { + JL_IMAGE_KIND_NONE = 0, + JL_IMAGE_KIND_JI, + JL_IMAGE_KIND_SO, +} jl_image_kind_t; + +// A loaded, but unparsed .ji or .so image file +typedef struct { + jl_image_kind_t kind; + void *handle; + const void *pointers; // jl_image_pointers_t * + const char *data; + size_t size; + uint64_t base; +} jl_image_buf_t; + +struct _jl_image_t; +typedef struct _jl_image_t jl_image_t; + JL_DLLIMPORT const char *jl_get_libdir(void); JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel); JL_DLLEXPORT void jl_init(void); @@ -2046,11 +2065,10 @@ JL_DLLEXPORT const char *jl_pathname_for_handle(void *handle); JL_DLLEXPORT jl_gcframe_t **jl_adopt_thread(void); JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s); -JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname); -JL_DLLEXPORT void jl_set_sysimg_so(void *handle); +JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname); +JL_DLLEXPORT jl_image_buf_t jl_set_sysimg_so(void *handle); JL_DLLEXPORT void jl_create_system_image(void **, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos); -JL_DLLEXPORT void jl_restore_system_image(const char *fname); -JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); +JL_DLLEXPORT void jl_restore_system_image(jl_image_t *image, jl_image_buf_t buf); JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int complete, const char *pkgimage); JL_DLLEXPORT void jl_set_newly_inferred(jl_value_t *newly_inferred); diff --git a/src/llvm-multiversioning.cpp b/src/llvm-multiversioning.cpp index 1901a721e5e08..299213090f064 100644 --- a/src/llvm-multiversioning.cpp +++ b/src/llvm-multiversioning.cpp @@ -215,7 +215,7 @@ static void annotate_module_clones(Module &M) { if (auto maybe_specs = get_target_specs(M)) { specs = std::move(*maybe_specs); } else { - auto full_specs = jl_get_llvm_clone_targets(); + auto full_specs = jl_get_llvm_clone_targets(jl_options.cpu_target); specs.reserve(full_specs.size()); for (auto &spec: full_specs) { specs.push_back(TargetSpec::fromSpec(spec)); diff --git a/src/processor.cpp b/src/processor.cpp index ed50c609895cc..28f5d80d77cd2 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -504,7 +504,8 @@ static inline llvm::SmallVector, 0> parse_cmdline(const char *option, F &&feature_cb) { if (!option) - option = "native"; + abort(); + llvm::SmallVector, 0> res; TargetData arg{}; auto reset_arg = [&] { @@ -612,31 +613,29 @@ parse_cmdline(const char *option, F &&feature_cb) // Cached version of command line parsing template -static inline llvm::SmallVector, 0> &get_cmdline_targets(F &&feature_cb) +static inline llvm::SmallVector, 0> &get_cmdline_targets(const char *cpu_target, F &&feature_cb) { static llvm::SmallVector, 0> targets = - parse_cmdline(jl_options.cpu_target, std::forward(feature_cb)); + parse_cmdline(cpu_target, std::forward(feature_cb)); return targets; } // Load sysimg, use the `callback` for dispatch and perform all relocations // for the selected target. template -static inline jl_image_t parse_sysimg(void *hdl, F &&callback) +static inline jl_image_t parse_sysimg(jl_image_buf_t image, F &&callback, void *ctx) { JL_TIMING(LOAD_IMAGE, LOAD_Processor); jl_image_t res{}; - const jl_image_pointers_t *pointers; - if (jl_system_image_size == 0) - jl_dlsym(hdl, "jl_image_pointers", (void**)&pointers, 1); - else - pointers = &jl_image_pointers; // libjulia-internal and sysimage statically linked + if (image.kind != JL_IMAGE_KIND_SO) + return res; + const jl_image_pointers_t *pointers = (const jl_image_pointers_t *)image.pointers; const void *ids = pointers->target_data; jl_value_t* rejection_reason = nullptr; JL_GC_PUSH1(&rejection_reason); - uint32_t target_idx = callback(ids, &rejection_reason); + uint32_t target_idx = callback(ctx, ids, &rejection_reason); if (target_idx == UINT32_MAX) { jl_error(jl_string_ptr(rejection_reason)); } @@ -794,17 +793,7 @@ static inline jl_image_t parse_sysimg(void *hdl, F &&callback) res.fptrs.nclones = clones.size(); } -#ifdef _OS_WINDOWS_ - res.base = (intptr_t)hdl; -#else - Dl_info dlinfo; - if (dladdr((void*)pointers, &dlinfo) != 0) { - res.base = (intptr_t)dlinfo.dli_fbase; - } - else { - res.base = 0; - } -#endif + res.base = image.base; { void *pgcstack_func_slot = pointers->ptls->pgcstack_func_slot; @@ -1020,7 +1009,7 @@ JL_DLLEXPORT jl_value_t *jl_get_cpu_features(void) } extern "C" JL_DLLEXPORT jl_value_t* jl_reflect_clone_targets() { - auto specs = jl_get_llvm_clone_targets(); + auto specs = jl_get_llvm_clone_targets(jl_options.cpu_target); const uint32_t base_flags = 0; llvm::SmallVector data; auto push_i32 = [&] (uint32_t v) { diff --git a/src/processor.h b/src/processor.h index 214f51845b12d..65b634fd0ba26 100644 --- a/src/processor.h +++ b/src/processor.h @@ -64,6 +64,7 @@ JL_DLLEXPORT int jl_test_cpu_feature(jl_cpu_feature_t feature); static const uint32_t jl_sysimg_tag_mask = 0x80000000u; static const uint32_t jl_sysimg_val_mask = ~((uint32_t)0x80000000u); +// A parsed image file typedef struct _jl_image_fptrs_t { // number of functions uint32_t nptrs; @@ -82,14 +83,14 @@ typedef struct _jl_image_fptrs_t { const uint32_t *clone_idxs; } jl_image_fptrs_t; -typedef struct { +struct _jl_image_t { uint64_t base; const char *gvars_base; const int32_t *gvars_offsets; uint32_t ngvars; jl_image_fptrs_t fptrs; void **jl_small_typeof; -} jl_image_t; +}; // The header for each image // Details important counts about the image @@ -206,8 +207,8 @@ typedef struct { * * Return the data about the function pointers selected. */ -jl_image_t jl_init_processor_sysimg(void *hdl); -jl_image_t jl_init_processor_pkgimg(void *hdl); +jl_image_t jl_init_processor_sysimg(jl_image_buf_t image, const char *cpu_target); +jl_image_t jl_init_processor_pkgimg(jl_image_buf_t image); // Return the name of the host CPU as a julia string. JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void); @@ -251,7 +252,7 @@ extern JL_DLLEXPORT bool jl_processor_print_help; * If the detected/specified CPU name is not available on the LLVM version specified, * a fallback CPU name will be used. Unsupported features will be ignored. */ -extern "C" JL_DLLEXPORT std::pair> jl_get_llvm_target(bool imaging, uint32_t &flags) JL_NOTSAFEPOINT; +extern "C" JL_DLLEXPORT std::pair> jl_get_llvm_target(const char *cpu_target, bool imaging, uint32_t &flags) JL_NOTSAFEPOINT; /** * Returns the CPU name and feature string to be used by LLVM disassembler. @@ -275,7 +276,7 @@ struct jl_target_spec_t { /** * Return the list of targets to clone */ -extern "C" JL_DLLEXPORT llvm::SmallVector jl_get_llvm_clone_targets(void) JL_NOTSAFEPOINT; +extern "C" JL_DLLEXPORT llvm::SmallVector jl_get_llvm_clone_targets(const char *cpu_target) JL_NOTSAFEPOINT; // NOLINTEND(clang-diagnostic-return-type-c-linkage) struct FeatureName { const char *name; diff --git a/src/processor_arm.cpp b/src/processor_arm.cpp index 0d9009afabec7..7df2b74773af4 100644 --- a/src/processor_arm.cpp +++ b/src/processor_arm.cpp @@ -1510,7 +1510,7 @@ static inline void disable_depends(FeatureList &features) ::disable_depends(features, Feature::deps, sizeof(Feature::deps) / sizeof(FeatureDep)); } -static const llvm::SmallVector, 0> &get_cmdline_targets(void) +static const llvm::SmallVector, 0> &get_cmdline_targets(const char *cpu_target) { auto feature_cb = [] (const char *str, size_t len, FeatureList &list) { #ifdef _CPU_AARCH64_ @@ -1527,7 +1527,7 @@ static const llvm::SmallVector, 0> &get_cmdline_targets(v set_bit(list, fbit, true); return true; }; - auto &targets = ::get_cmdline_targets(feature_cb); + auto &targets = ::get_cmdline_targets(cpu_target, feature_cb); for (auto &t: targets) { if (auto nname = normalize_cpu_name(t.name)) { t.name = nname; @@ -1590,10 +1590,11 @@ static int max_vector_size(const FeatureList &features) #endif } -static uint32_t sysimg_init_cb(const void *id, jl_value_t **rejection_reason) +static uint32_t sysimg_init_cb(void *ctx, const void *id, jl_value_t **rejection_reason) { // First see what target is requested for the JIT. - auto &cmdline = get_cmdline_targets(); + const char *cpu_target = (const char *)ctx; + auto &cmdline = get_cmdline_targets(cpu_target); TargetData target = arg_target_data(cmdline[0], true); // Then find the best match in the sysimg auto sysimg = deserialize_target_data((const uint8_t*)id); @@ -1617,7 +1618,7 @@ static uint32_t sysimg_init_cb(const void *id, jl_value_t **rejection_reason) return match.best_idx; } -static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason JL_REQUIRE_ROOTED_SLOT) +static uint32_t pkgimg_init_cb(void *ctx, const void *id, jl_value_t **rejection_reason JL_REQUIRE_ROOTED_SLOT) { TargetData target = jit_targets.front(); auto pkgimg = deserialize_target_data((const uint8_t*)id); @@ -1630,9 +1631,9 @@ static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason JL_ return match.best_idx; } -static void ensure_jit_target(bool imaging) +static void ensure_jit_target(const char *cpu_target, bool imaging) { - auto &cmdline = get_cmdline_targets(); + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, imaging); if (!jit_targets.empty()) return; @@ -1843,36 +1844,36 @@ JL_DLLEXPORT jl_value_t *jl_cpu_has_fma(int bits) #endif } -jl_image_t jl_init_processor_sysimg(void *hdl) +jl_image_t jl_init_processor_sysimg(jl_image_buf_t image, const char *cpu_target) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); - return parse_sysimg(hdl, sysimg_init_cb); + return parse_sysimg(image, sysimg_init_cb, (void *)cpu_target); } -jl_image_t jl_init_processor_pkgimg(void *hdl) +jl_image_t jl_init_processor_pkgimg(jl_image_buf_t image) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); if (jit_targets.size() > 1) jl_error("Expected only one JIT target"); - return parse_sysimg(hdl, pkgimg_init_cb); + return parse_sysimg(image, pkgimg_init_cb, NULL); } JL_DLLEXPORT jl_value_t* jl_check_pkgimage_clones(char *data) { jl_value_t *rejection_reason = NULL; JL_GC_PUSH1(&rejection_reason); - uint32_t match_idx = pkgimg_init_cb(data, &rejection_reason); + uint32_t match_idx = pkgimg_init_cb(NULL, data, &rejection_reason); JL_GC_POP(); if (match_idx == UINT32_MAX) return rejection_reason; return jl_nothing; } -std::pair> jl_get_llvm_target(bool imaging, uint32_t &flags) +std::pair> jl_get_llvm_target(const char *cpu_target, bool imaging, uint32_t &flags) { - ensure_jit_target(imaging); + ensure_jit_target(cpu_target, imaging); flags = jit_targets[0].en.flags; return get_llvm_target_vec(jit_targets[0]); } @@ -1891,10 +1892,10 @@ const std::pair &jl_get_llvm_disasm_target(void) } #ifndef __clang_gcanalyzer__ -llvm::SmallVector jl_get_llvm_clone_targets(void) +llvm::SmallVector jl_get_llvm_clone_targets(const char *cpu_target) { - auto &cmdline = get_cmdline_targets(); + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, true); llvm::SmallVector, 0> image_targets; for (auto &arg: cmdline) { diff --git a/src/processor_fallback.cpp b/src/processor_fallback.cpp index f8d9eb9fd9e73..c8c8feb072345 100644 --- a/src/processor_fallback.cpp +++ b/src/processor_fallback.cpp @@ -13,12 +13,12 @@ static inline const std::string &host_cpu_name() return name; } -static const llvm::SmallVector, 0> &get_cmdline_targets(void) +static const llvm::SmallVector, 0> &get_cmdline_targets(const char *cpu_target) { auto feature_cb = [] (const char*, size_t, FeatureList<1>&) { return false; }; - return ::get_cmdline_targets<1>(feature_cb); + return ::get_cmdline_targets<1>(cpu_target, feature_cb); } static llvm::SmallVector, 0> jit_targets; @@ -36,10 +36,11 @@ static TargetData<1> arg_target_data(const TargetData<1> &arg, bool require_host return res; } -static uint32_t sysimg_init_cb(const void *id, jl_value_t **rejection_reason) +static uint32_t sysimg_init_cb(void *ctx, const void *id, jl_value_t **rejection_reason) { // First see what target is requested for the JIT. - auto &cmdline = get_cmdline_targets(); + const char *cpu_target = (const char *)ctx; + auto &cmdline = get_cmdline_targets(cpu_target); TargetData<1> target = arg_target_data(cmdline[0], true); // Find the last name match or use the default one. uint32_t best_idx = 0; @@ -54,7 +55,7 @@ static uint32_t sysimg_init_cb(const void *id, jl_value_t **rejection_reason) return best_idx; } -static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason) +static uint32_t pkgimg_init_cb(void *ctx, const void *id, jl_value_t **rejection_reason) { TargetData<1> target = jit_targets.front(); // Find the last name match or use the default one. @@ -70,9 +71,9 @@ static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason) return best_idx; } -static void ensure_jit_target(bool imaging) +static void ensure_jit_target(const char *cpu_target, bool imaging) { - auto &cmdline = get_cmdline_targets(); + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, imaging); if (!jit_targets.empty()) return; @@ -115,25 +116,25 @@ get_llvm_target_str(const TargetData<1> &data) using namespace Fallback; -jl_image_t jl_init_processor_sysimg(void *hdl) +jl_image_t jl_init_processor_sysimg(jl_image_buf_t image, const char *cpu_target) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); - return parse_sysimg(hdl, sysimg_init_cb); + return parse_sysimg(image, sysimg_init_cb, (void *)cpu_target); } -jl_image_t jl_init_processor_pkgimg(void *hdl) +jl_image_t jl_init_processor_pkgimg(jl_image_buf_t image) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); if (jit_targets.size() > 1) jl_error("Expected only one JIT target"); - return parse_sysimg(hdl, pkgimg_init_cb); + return parse_sysimg(image, pkgimg_init_cb, NULL); } -std::pair> jl_get_llvm_target(bool imaging, uint32_t &flags) +std::pair> jl_get_llvm_target(const char *cpu_target, bool imaging, uint32_t &flags) { - ensure_jit_target(imaging); + ensure_jit_target(cpu_target, imaging); flags = jit_targets[0].en.flags; return get_llvm_target_vec(jit_targets[0]); } @@ -145,10 +146,10 @@ const std::pair &jl_get_llvm_disasm_target(void) return res; } #ifndef __clang_gcanalyzer__ -llvm::SmallVector jl_get_llvm_clone_targets(void) +llvm::SmallVector jl_get_llvm_clone_targets(const char *cpu_target) { - auto &cmdline = get_cmdline_targets(); + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, true); llvm::SmallVector, 0> image_targets; for (auto &arg: cmdline) { @@ -192,7 +193,7 @@ JL_DLLEXPORT jl_value_t* jl_check_pkgimage_clones(char *data) { jl_value_t *rejection_reason = NULL; JL_GC_PUSH1(&rejection_reason); - uint32_t match_idx = pkgimg_init_cb(data, &rejection_reason); + uint32_t match_idx = pkgimg_init_cb(NULL, data, &rejection_reason); JL_GC_POP(); if (match_idx == UINT32_MAX) return rejection_reason; diff --git a/src/processor_x86.cpp b/src/processor_x86.cpp index db954680289ea..dbd3de8bffa0c 100644 --- a/src/processor_x86.cpp +++ b/src/processor_x86.cpp @@ -782,7 +782,7 @@ static inline void disable_depends(FeatureList &features) ::disable_depends(features, Feature::deps, sizeof(Feature::deps) / sizeof(FeatureDep)); } -static const llvm::SmallVector, 0> &get_cmdline_targets(void) +static const llvm::SmallVector, 0> &get_cmdline_targets(const char *cpu_target) { auto feature_cb = [] (const char *str, size_t len, FeatureList &list) { auto fbit = find_feature_bit(feature_names, nfeature_names, str, len); @@ -791,7 +791,7 @@ static const llvm::SmallVector, 0> &get_cmdline_targets(v set_bit(list, fbit, true); return true; }; - auto &targets = ::get_cmdline_targets(feature_cb); + auto &targets = ::get_cmdline_targets(cpu_target, feature_cb); for (auto &t: targets) { if (auto nname = normalize_cpu_name(t.name)) { t.name = nname; @@ -851,10 +851,11 @@ static int max_vector_size(const FeatureList &features) return 16; } -static uint32_t sysimg_init_cb(const void *id, jl_value_t** rejection_reason) +static uint32_t sysimg_init_cb(void *ctx, const void *id, jl_value_t** rejection_reason) { // First see what target is requested for the JIT. - auto &cmdline = get_cmdline_targets(); + const char *cpu_target = (const char *)ctx; + auto &cmdline = get_cmdline_targets(cpu_target); TargetData target = arg_target_data(cmdline[0], true); // Then find the best match in the sysimg auto sysimg = deserialize_target_data((const uint8_t*)id); @@ -897,7 +898,7 @@ static uint32_t sysimg_init_cb(const void *id, jl_value_t** rejection_reason) return match.best_idx; } -static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason) +static uint32_t pkgimg_init_cb(void *ctx, const void *id, jl_value_t **rejection_reason) { TargetData target = jit_targets.front(); auto pkgimg = deserialize_target_data((const uint8_t*)id); @@ -912,9 +913,9 @@ static uint32_t pkgimg_init_cb(const void *id, jl_value_t **rejection_reason) //This function serves as a fallback during bootstrapping, at that point we don't have a sysimage with native code // so we won't call sysimg_init_cb, else this function shouldn't do anything. -static void ensure_jit_target(bool imaging) +static void ensure_jit_target(const char *cpu_target, bool imaging) { - auto &cmdline = get_cmdline_targets(); + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, imaging); if (!jit_targets.empty()) return; @@ -1058,7 +1059,7 @@ JL_DLLEXPORT jl_value_t* jl_check_pkgimage_clones(char *data) { jl_value_t *rejection_reason = NULL; JL_GC_PUSH1(&rejection_reason); - uint32_t match_idx = pkgimg_init_cb(data, &rejection_reason); + uint32_t match_idx = pkgimg_init_cb(NULL, data, &rejection_reason); JL_GC_POP(); if (match_idx == UINT32_MAX) return rejection_reason; @@ -1075,25 +1076,25 @@ JL_DLLEXPORT jl_value_t *jl_cpu_has_fma(int bits) return jl_false; } -jl_image_t jl_init_processor_sysimg(void *hdl) +jl_image_t jl_init_processor_sysimg(jl_image_buf_t image, const char *cpu_target) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); - return parse_sysimg(hdl, sysimg_init_cb); + return parse_sysimg(image, sysimg_init_cb, (void *)cpu_target); } -jl_image_t jl_init_processor_pkgimg(void *hdl) +jl_image_t jl_init_processor_pkgimg(jl_image_buf_t image) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); if (jit_targets.size() > 1) jl_error("Expected only one JIT target"); - return parse_sysimg(hdl, pkgimg_init_cb); + return parse_sysimg(image, pkgimg_init_cb, NULL); } -std::pair> jl_get_llvm_target(bool imaging, uint32_t &flags) +std::pair> jl_get_llvm_target(const char *cpu_target, bool imaging, uint32_t &flags) { - ensure_jit_target(imaging); + ensure_jit_target(cpu_target, imaging); flags = jit_targets[0].en.flags; return get_llvm_target_vec(jit_targets[0]); } @@ -1106,9 +1107,10 @@ const std::pair &jl_get_llvm_disasm_target(void) } //This function parses the -C command line to figure out which targets to multiversion to. #ifndef __clang_gcanalyzer__ -llvm::SmallVector jl_get_llvm_clone_targets(void) +llvm::SmallVector jl_get_llvm_clone_targets(const char *cpu_target) { - auto &cmdline = get_cmdline_targets(); + + auto &cmdline = get_cmdline_targets(cpu_target); check_cmdline(cmdline, true); llvm::SmallVector, 0> image_targets; for (auto &arg: cmdline) { diff --git a/src/staticdata.c b/src/staticdata.c index 793a2ccd19574..3376933eed038 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -607,8 +607,7 @@ typedef struct { } pkgcachesizes; // --- Static Compile --- -static void *jl_sysimg_handle = NULL; -static jl_image_t sysimage; +static jl_image_buf_t jl_sysimage_buf = { JL_IMAGE_KIND_NONE }; static inline uintptr_t *sysimg_gvars(const char *base, const int32_t *offsets, size_t idx) { @@ -620,29 +619,6 @@ JL_DLLEXPORT int jl_running_on_valgrind(void) return RUNNING_ON_VALGRIND; } -static void jl_load_sysimg_so(void) -{ - assert(sysimage.fptrs.ptrs); // jl_init_processor_sysimg should already be run - - size_t *plen; - const char *sysimg_data; - - if (jl_system_image_size == 0) { - // in the usual case, the sysimage was not statically linked to libjulia-internal - // look up the external sysimage symbols via the dynamic linker - jl_dlsym(jl_sysimg_handle, "jl_system_image_size", (void **)&plen, 1); - jl_dlsym(jl_sysimg_handle, "jl_system_image_data", (void **)&sysimg_data, 1); - } else { - // the sysimage was statically linked directly against libjulia-internal - // use the internal symbols - plen = &jl_system_image_size; - sysimg_data = &jl_system_image_data; - } - - jl_restore_system_image_data(sysimg_data, *plen); -} - - // --- serializer --- #define NBOX_C 1024 @@ -3058,33 +3034,111 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli JL_DLLEXPORT size_t ios_write_direct(ios_t *dest, ios_t *src); -// Takes in a path of the form "usr/lib/julia/sys.so" (jl_restore_system_image should be passed the same string) -JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname) +// Takes in a path of the form "usr/lib/julia/sys.so" +JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname) { - if (jl_sysimg_handle) - return; // embedded target already called jl_set_sysimg_so + if (jl_sysimage_buf.kind != JL_IMAGE_KIND_NONE) + return jl_sysimage_buf; char *dot = (char*) strrchr(fname, '.'); int is_ji = (dot && !strcmp(dot, ".ji")); - // Get handle to sys.so - if (!is_ji) // .ji extension => load .ji file only - jl_set_sysimg_so(jl_load_dynamic_library(fname, JL_RTLD_LOCAL | JL_RTLD_NOW, 1)); + if (is_ji) { + // .ji extension => load .ji file only + ios_t f; + + if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) + jl_errorf("System image file \"%s\" not found.", fname); + ios_bufmode(&f, bm_none); + + JL_SIGATOMIC_BEGIN(); + + ios_seek_end(&f); + size_t len = ios_pos(&f); + char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); + ios_seek(&f, 0); + + if (ios_readall(&f, sysimg, len) != len) + jl_errorf("Error reading system image file."); + + ios_close(&f); + + JL_SIGATOMIC_END(); + + jl_sysimage_buf = (jl_image_buf_t) { + .kind = JL_IMAGE_KIND_JI, + .handle = NULL, + .pointers = NULL, + .data = sysimg, + .size = len, + .base = 0, + }; + return jl_sysimage_buf; + } else { + // Get handle to sys.so + return jl_set_sysimg_so(jl_load_dynamic_library(fname, JL_RTLD_LOCAL | JL_RTLD_NOW, 1)); + } } -// Allow passing in a module handle directly, rather than a path -JL_DLLEXPORT void jl_set_sysimg_so(void *handle) +// From a shared library handle, verify consistency and return a jl_image_buf_t +static jl_image_buf_t get_image_buf(void *handle, int is_pkgimage) { + size_t *plen; + const char *data; + const void *pointers; + uint64_t base; + + // verify that the linker resolved the symbols in this image against ourselves (libjulia-internal) void** (*get_jl_RTLD_DEFAULT_handle_addr)(void) = NULL; if (handle != jl_RTLD_DEFAULT_handle) { int symbol_found = jl_dlsym(handle, "get_jl_RTLD_DEFAULT_handle_addr", (void **)&get_jl_RTLD_DEFAULT_handle_addr, 0); if (!symbol_found || (void*)&jl_RTLD_DEFAULT_handle != (get_jl_RTLD_DEFAULT_handle_addr())) - jl_error("System image file failed consistency check: maybe opened the wrong version?"); + jl_error("Image file failed consistency check: maybe opened the wrong version?"); } - if (jl_options.cpu_target == NULL) - jl_options.cpu_target = "native"; - jl_sysimg_handle = handle; - sysimage = jl_init_processor_sysimg(handle); + + // verification passed, lookup the buffer pointers + if (jl_system_image_size == 0 || is_pkgimage) { + // in the usual case, the sysimage was not statically linked to libjulia-internal + // look up the external sysimage symbols via the dynamic linker + jl_dlsym(handle, "jl_system_image_size", (void **)&plen, 1); + jl_dlsym(handle, "jl_system_image_data", (void **)&data, 1); + jl_dlsym(handle, "jl_image_pointers", (void**)&pointers, 1); + } else { + // the sysimage was statically linked directly against libjulia-internal + // use the internal symbols + plen = &jl_system_image_size; + pointers = &jl_image_pointers; + data = &jl_system_image_data; + } + +#ifdef _OS_WINDOWS_ + base = (intptr_t)handle; +#else + Dl_info dlinfo; + if (dladdr((void*)pointers, &dlinfo) != 0) + base = (intptr_t)dlinfo.dli_fbase; + else + base = 0; +#endif + + return (jl_image_buf_t) { + .kind = JL_IMAGE_KIND_SO, + .handle = handle, + .pointers = pointers, + .data = data, + .size = *plen, + .base = base, + }; +} + +// Allow passing in a module handle directly, rather than a path +JL_DLLEXPORT jl_image_buf_t jl_set_sysimg_so(void *handle) +{ + if (jl_sysimage_buf.kind != JL_IMAGE_KIND_NONE) + return jl_sysimage_buf; + + jl_sysimage_buf = get_image_buf(handle, /* is_pkgimage */ 0); + return jl_sysimage_buf; } #ifndef JL_NDEBUG @@ -3104,7 +3158,8 @@ extern void export_jl_small_typeof(void); // into the native code of the image. See https://github.com/JuliaLang/julia/pull/52123#issuecomment-1959965395. int IMAGE_NATIVE_CODE_TAINTED = 0; -static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_array_t *depmods, uint64_t checksum, +static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, void *image_handle, + jl_array_t *depmods, uint64_t checksum, /* outputs */ jl_array_t **restored, jl_array_t **init_order, jl_array_t **extext_methods, jl_array_t **internal_methods, jl_array_t **new_ext_cis, jl_array_t **method_roots_list, @@ -3569,8 +3624,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl s.s = &sysimg; jl_update_all_fptrs(&s, image); // fptr relocs and registration if (!ccallable_list) { - // TODO: jl_sysimg_handle or img_handle? - jl_reinit_ccallable(&s.ccallable_list, image_base, jl_sysimg_handle); + jl_reinit_ccallable(&s.ccallable_list, image_base, image_handle); arraylist_free(&s.ccallable_list); } s.s = NULL; @@ -3668,7 +3722,7 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i ios_close(f); ios_static_buffer(f, sysimg, len); pkgcachesizes cachesizes; - jl_restore_system_image_from_stream_(f, image, depmods, checksum, (jl_array_t**)&restored, &init_order, &extext_methods, &internal_methods, &new_ext_cis, &method_roots_list, + jl_restore_system_image_from_stream_(f, image, pkgimage_handle, depmods, checksum, (jl_array_t**)&restored, &init_order, &extext_methods, &internal_methods, &new_ext_cis, &method_roots_list, &ext_targets, &edges, &base, &ccallable_list, &cachesizes); JL_SIGATOMIC_END(); @@ -3717,16 +3771,16 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i return restored; } -static void jl_restore_system_image_from_stream(ios_t *f, jl_image_t *image, uint32_t checksum) +static void jl_restore_system_image_from_stream(ios_t *f, jl_image_t *image, void *image_handle, uint32_t checksum) { JL_TIMING(LOAD_IMAGE, LOAD_Sysimg); - jl_restore_system_image_from_stream_(f, image, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + jl_restore_system_image_from_stream_(f, image, image_handle, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } -JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(void* pkgimage_handle, const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int completeinfo, const char *pkgname, int needs_permalloc) +JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(void* pkgimage_handle, jl_image_buf_t buf, jl_image_t *image, jl_array_t *depmods, int completeinfo, const char *pkgname, int needs_permalloc) { ios_t f; - ios_static_buffer(&f, (char*)buf, sz); + ios_static_buffer(&f, (char*)buf.data, buf.size); jl_value_t *ret = jl_restore_package_image_from_stream(pkgimage_handle, &f, image, depmods, completeinfo, pkgname, needs_permalloc); ios_close(&f); return ret; @@ -3745,47 +3799,22 @@ JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *d return ret; } -// TODO: need to enforce that the alignment of the buffer is suitable for vectors -JL_DLLEXPORT void jl_restore_system_image(const char *fname) +JL_DLLEXPORT void jl_restore_system_image(jl_image_t *image, jl_image_buf_t buf) { -#ifndef JL_NDEBUG - char *dot = fname ? (char*)strrchr(fname, '.') : NULL; - int is_ji = (dot && !strcmp(dot, ".ji")); - assert((is_ji || jl_sysimg_handle) && "System image file not preloaded"); -#endif + ios_t f; - if (jl_sysimg_handle) { - // load the pre-compiled sysimage from jl_sysimg_handle - jl_load_sysimg_so(); - } - else { - ios_t f; - if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) - jl_errorf("System image file \"%s\" not found.", fname); - ios_bufmode(&f, bm_none); - JL_SIGATOMIC_BEGIN(); - ios_seek_end(&f); - size_t len = ios_pos(&f); - char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); - ios_seek(&f, 0); - if (ios_readall(&f, sysimg, len) != len) - jl_errorf("Error reading system image file."); - ios_close(&f); - uint32_t checksum = jl_crc32c(0, sysimg, len); - ios_static_buffer(&f, sysimg, len); - jl_restore_system_image_from_stream(&f, &sysimage, checksum); - ios_close(&f); - JL_SIGATOMIC_END(); - } -} + if (buf.kind == JL_IMAGE_KIND_NONE) + return; + + if (buf.kind == JL_IMAGE_KIND_SO) + assert(image->fptrs.ptrs); // jl_init_processor_sysimg should already be run -JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len) -{ - ios_t f; JL_SIGATOMIC_BEGIN(); - ios_static_buffer(&f, (char*)buf, len); - uint32_t checksum = jl_crc32c(0, buf, len); - jl_restore_system_image_from_stream(&f, &sysimage, checksum); + ios_static_buffer(&f, (char *)buf.data, buf.size); + + uint32_t checksum = jl_crc32c(0, buf.data, buf.size); + jl_restore_system_image_from_stream(&f, image, buf.handle, checksum); + ios_close(&f); JL_SIGATOMIC_END(); } @@ -3804,12 +3833,11 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j #endif jl_errorf("Error opening package file %s: %s\n", fname, reason); } - const char *pkgimg_data; - jl_dlsym(pkgimg_handle, "jl_system_image_data", (void **)&pkgimg_data, 1); - size_t *plen; - jl_dlsym(pkgimg_handle, "jl_system_image_size", (void **)&plen, 1); - jl_image_t pkgimage = jl_init_processor_pkgimg(pkgimg_handle); + jl_image_buf_t buf = get_image_buf(pkgimg_handle, /* is_pkgimage */ 1); + + // Despite the name, this function actually parses the pkgimage + jl_image_t pkgimage = jl_init_processor_pkgimg(buf); if (ignore_native) { // Must disable using native code in possible downstream users of this code: @@ -3818,7 +3846,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j IMAGE_NATIVE_CODE_TAINTED = 1; } - jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_handle, pkgimg_data, &pkgimage, *plen, depmods, completeinfo, pkgname, 0); + jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_handle, buf, &pkgimage, depmods, completeinfo, pkgname, 0); return mod; } diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 43cdf5e2696a6..1c70c5d3f2cab 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -979,7 +979,7 @@ let exename = `$(Base.julia_cmd().exec[1]) -t 1` p = run(pipeline(`$exename --sysimage=$libjulia`, stderr=err), wait=false) close(err.in) let s = read(err, String) - @test s == "ERROR: System image file failed consistency check: maybe opened the wrong version?\n" + @test s == "ERROR: Image file failed consistency check: maybe opened the wrong version?\n" end @test errors_not_signals(p) @test p.exitcode == 1 From 84f1db5746b82ccfafebf805d3a725e65f572e90 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 20 May 2025 16:54:18 -0400 Subject: [PATCH 03/18] [deps] enable zstd support (backport, no LLVM) (#58344) --- Make.inc | 11 ++++- Makefile | 1 + THIRDPARTY.md | 1 + contrib/refresh_checksums.mk | 2 +- deps/Makefile | 9 +++- deps/checksums/zstd | 38 +++++++++++++++++ deps/zstd.mk | 60 ++++++++++++++++++++++++++ deps/zstd.version | 8 ++++ julia.spdx.json | 17 ++++++++ src/Makefile | 5 ++- stdlib/Makefile | 2 +- stdlib/Manifest.toml | 5 +++ stdlib/Zstd_jll/Project.toml | 15 +++++++ stdlib/Zstd_jll/src/Zstd_jll.jl | 73 ++++++++++++++++++++++++++++++++ stdlib/Zstd_jll/test/runtests.jl | 7 +++ stdlib/stdlib.mk | 2 +- 16 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 deps/checksums/zstd create mode 100644 deps/zstd.mk create mode 100644 deps/zstd.version create mode 100644 stdlib/Zstd_jll/Project.toml create mode 100644 stdlib/Zstd_jll/src/Zstd_jll.jl create mode 100644 stdlib/Zstd_jll/test/runtests.jl diff --git a/Make.inc b/Make.inc index e0bc8ba224a6d..b45886c2af963 100644 --- a/Make.inc +++ b/Make.inc @@ -65,6 +65,7 @@ USE_SYSTEM_LIBGIT2:=0 USE_SYSTEM_PATCHELF:=0 USE_SYSTEM_LIBWHICH:=0 USE_SYSTEM_ZLIB:=0 +USE_SYSTEM_ZSTD:=0 USE_SYSTEM_P7ZIP:=0 USE_SYSTEM_LLD:=0 @@ -1201,6 +1202,14 @@ else UTF8PROC_INC := $(build_includedir) endif +ifeq ($(USE_SYSTEM_ZSTD), 1) + LIBZSTD := -lzstd + ZSTD_INC := $(LOCALBASE)/include +else + LIBZSTD := -L$(build_shlibdir) -lzstd + ZSTD_INC := $(build_includedir) +endif + # We need python for things like BB triplet recognition. We don't really care # about version, generally, so just find something that works: PYTHON := $(shell which python 2>/dev/null || which python3 2>/dev/null || which python2 2>/dev/null || echo not found) @@ -1268,7 +1277,7 @@ CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.33|GLIBCXX_3\.5\.|GLIBCXX_4\. # Note: we explicitly _do not_ define `CSL` here, since it requires some more # advanced techniques to decide whether it should be installed from a BB source # or not. See `deps/csl.mk` for more detail. -BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP LLD LIBTRACYCLIENT +BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB ZSTD P7ZIP LLD LIBTRACYCLIENT define SET_BB_DEFAULT # First, check to see if BB is disabled on a global setting ifeq ($$(USE_BINARYBUILDER),0) diff --git a/Makefile b/Makefile index 05683fda0a004..bdc05d75f9a2c 100644 --- a/Makefile +++ b/Makefile @@ -225,6 +225,7 @@ JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += zlib else JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libz endif +JL_PRIVATE_LIBS-$(USE_SYSTEM_ZSTD) += libzstd ifeq ($(USE_LLVM_SHLIB),1) JL_PRIVATE_LIBS-$(USE_SYSTEM_LLVM) += libLLVM $(LLVM_SHARED_LIB_NAME) endif diff --git a/THIRDPARTY.md b/THIRDPARTY.md index 412b84b688758..0b2982043d431 100644 --- a/THIRDPARTY.md +++ b/THIRDPARTY.md @@ -55,6 +55,7 @@ Julia bundles the following external programs and libraries: - [7-Zip](https://www.7-zip.org/license.txt) - [ZLIB](https://zlib.net/zlib_license.html) +- [ZSTD](https://github.com/facebook/zstd/blob/v1.5.7/LICENSE) On some platforms, distributions of Julia contain SSL certificate authority certificates, released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). diff --git a/contrib/refresh_checksums.mk b/contrib/refresh_checksums.mk index f67088141ccd4..506f524062538 100644 --- a/contrib/refresh_checksums.mk +++ b/contrib/refresh_checksums.mk @@ -24,7 +24,7 @@ CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS)) NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS)) # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded: -BB_PROJECTS=mbedtls libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient +BB_PROJECTS=mbedtls libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib zstd libsuitesparse openlibm blastrampoline libtracyclient BB_GCC_EXPANDED_PROJECTS=openblas csl BB_CXX_EXPANDED_PROJECTS=gmp llvm clang llvm-tools lld # These are non-BB source-only deps diff --git a/deps/Makefile b/deps/Makefile index 6a1b03eec919e..e942262cf6f0e 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -70,10 +70,12 @@ endif endif endif +PATCHELF_MANIFEST := ifneq (,$(findstring $(OS),Linux FreeBSD)) ifeq ($(USE_SYSTEM_PATCHELF), 0) DEP_LIBS += patchelf PATCHELF:=$(build_depsbindir)/patchelf +PATCHELF_MANIFEST:=$(build_prefix)/manifest/patchelf else PATCHELF:=patchelf endif @@ -155,6 +157,10 @@ ifeq ($(USE_SYSTEM_ZLIB), 0) DEP_LIBS += zlib endif +ifeq ($(USE_SYSTEM_ZSTD), 0) +DEP_LIBS += zstd +endif + ifeq ($(USE_SYSTEM_P7ZIP), 0) DEP_LIBS += p7zip endif @@ -195,7 +201,7 @@ DEP_LIBS_STAGED := $(DEP_LIBS) # list all targets DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \ openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \ - objconv mbedtls libssh2 nghttp2 curl libgit2 libwhich zlib p7zip csl \ + objconv mbedtls libssh2 nghttp2 curl libgit2 libwhich zlib zstd p7zip csl \ sanitizers libsuitesparse lld libtracyclient ittapi JuliaSyntax terminfo DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL) @@ -244,6 +250,7 @@ include $(SRCDIR)/openblas.mk include $(SRCDIR)/utf8proc.mk include $(SRCDIR)/libsuitesparse.mk include $(SRCDIR)/zlib.mk +include $(SRCDIR)/zstd.mk include $(SRCDIR)/unwind.mk include $(SRCDIR)/gmp.mk include $(SRCDIR)/mpfr.mk diff --git a/deps/checksums/zstd b/deps/checksums/zstd new file mode 100644 index 0000000000000..aea151b266966 --- /dev/null +++ b/deps/checksums/zstd @@ -0,0 +1,38 @@ +Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/md5/d6b2fb32d705078dbc369986ac8b056b +Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/sha512/5dfcf36087ce8540b1f6a04181adee962e2164a763e758ac5cc256c332756774b381ca58e26641a15ce555d59641690a6da72a67bf935d8611734f2006bde504 +Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/md5/0c627ec83e426383c25eb4bc297f3548 +Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/sha512/1fdcf77e877f0676fc26a05e0cc20a1d6e1df731d81e0bba9a5657131116bbea75da4d38953969d8d07dce0bf2d7654075dbb285ebe5f4588c446e88774336c8 +Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/md5/cc9ada74a19db50d7dd6edd05866c902 +Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/sha512/0b33c0df144bb1e95290685f01695b26da834a70a365c0362314cb001ba611962a0876bc5baac31f19c80bcb110e030fb9840a56761b4d29a7893ca65f95b111 +Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/md5/5daa5b2bf2b856c448feaa8329d0de1b +Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/sha512/b39d025463b4bf21295fd5bbff91ba501506b3480363cdcfe6dd2f11d2e0afaf130f6c74d962e503fccb7a55bfcad0504ebb19f18b6b5c8b8103e7b9919df536 +Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/md5/f4218e8b4f8d415df49aeba9d43f0ba0 +Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/sha512/878d4f90160c6b0c341c61ecafbf5f5cb89c73db3175f272adc666bc25c88b127145d78946bc0fcb992489b54fbb48089bfcacf768397fc5d54d7cae4aeae9f9 +Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/md5/3c2e132ca47e6d1d23c149fdde9d8bd5 +Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/sha512/3745d99c9ca0ce9f98ff9393e405e8b382d05573a972067d57e800e282a9544fff7bc3d49b91eccc98d7736acdc3faa4c637911d79fab10f5a691d33ae775574 +Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/md5/926d765281bef388ecc25d04cbb66102 +Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/sha512/2d2c14587e2e7b2b147cb6423720cc30ed6aa57ed07372a1aa54e7f2e6badb5aa640b116e83371561d6f8f3a1b3f7fff7f6df137f8c7be788ee889bb30273eae +Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/md5/c25420561ce254e57d74e30c88fc53dd +Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/sha512/2f924e2089589057e8713d04db9a1cb2f2d571ad9e7eeda3b7f898c9a75f8fecf0647f2185d3c01fc3b399d3662ff3b1acb13429c8a953f0394a3ed9ca30b877 +Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/md5/3314bf1b52f2295555fb4ae44b1d9331 +Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/sha512/91502910a0c9b786d91499477fee2445b8f6de6bcb71af7d79c738ea2430c67cb1957866383ee3921ed1a23c53a80be19aea6abcf0e76056ffee69583728c3ed +Zstd.v1.5.7+1.i686-linux-musl.tar.gz/md5/845eddc06527a4c4b196666f7ac64ba3 +Zstd.v1.5.7+1.i686-linux-musl.tar.gz/sha512/bb15b4327cef32be38c2fd68afedb3245c7db881ad66d3ece2198ff3034be9c12efa3d62bcba2b8e6056e7d8cb5f1b3e33726f7d1e1bead235c38f8fa985b557 +Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/md5/9bc0b3c951f5e66393fd5433bf60a2c8 +Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/sha512/550b0189097e569f98404aa836b76a5cbdc36428292214c4af8916dea2713440cf3ba94125b3e5fa0c65b2bcb916733094fdef906ad19f923d90dabfc961c75a +Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/md5/468d930de7a27af961996e7c6ed35298 +Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/sha512/d680715b1ac9ff07d5662c499fbab67757509599335f861158b9dba32fe9b22da6e52d0db6b402dd4542799621ad3dccf254dfd9d3c8748bbd22f7446681539a +Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/md5/b93fef8db2b0b4417f7836d73c5fbe86 +Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/sha512/9f3ee42c7952aba2d2c26252f058bb7ab96828fafc978c9273b500ef15ccd271c51399d4b93eebd4c832b087ab5ed8a4847104ce9c83c9483aaa13c22df681bb +Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/md5/29a260789fae6f6b6df0e5cebdafd615 +Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/sha512/015045a1b7a477504057cb4c87428d42386218e48af38f83739dbe6b93961ca2c8dd4d794377a2d54b8cc284f5a467e3358d4f534cf8bcbcad886ef8cea038e9 +Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/md5/06656befb6ef9a8cc7f56e7152c2acc5 +Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/sha512/16aea0d95432a87d21d9a6f55d84e45df85caf1fda77c75b7e9a8bba519605168585f21a812773ddf1075d9bad68412e63b8cad1a143420e25ae4405bb41842e +Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/md5/da13dd1cc0d20ba9a06e9e79a588cda4 +Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/sha512/cd4218fa92dcf8772390788d5654ca12132af7829fb0ada016f3c663e2045e29e7d7587f2f5a4f057020cacca17c188c8537f284b1456100d57e84bb47c40e77 +Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/md5/bce5f37e53e330bfe4df4a28cf5c223b +Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/sha512/8f6bd7664efea537ac7815db0604ca1a07bcfb71b5152c22dc7f0a11b57643f059c341fa71d315407e2333e4c97e43e214471c73eed8b977680785302c7c2b3e +Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/md5/7cf3a740fa174004b94125e8754f4a19 +Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/sha512/faac37ad4dacb0f083364c593cd3bd1c0b592947341a631bd2fbc4081361d97ef89482f4459c46ad37ae030aa900c62305a8525e64a2ad8e91204d76dda89db1 +zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/md5/a679d9aa86549b5851100ac5d4044c68 +zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/sha512/27c6fff165abea694d91311a6657a939433ba1d707147ed9072b5e4ecce259b929970306788e0c3e95db38ce85e894e5025936b1faa81cf67741b8464e24fc4e diff --git a/deps/zstd.mk b/deps/zstd.mk new file mode 100644 index 0000000000000..5ead77641858a --- /dev/null +++ b/deps/zstd.mk @@ -0,0 +1,60 @@ +## Zstd ## +ifneq ($(USE_BINARYBUILDER_ZSTD), 1) +ZSTD_GIT_URL := https://github.com/facebook/zstd.git +ZSTD_TAR_URL = https://api.github.com/repos/facebook/zstd/tarball/$1 +$(eval $(call git-external,zstd,ZSTD,,,$(BUILDDIR))) + +ZSTD_BUILD_OPTS := MOREFLAGS="-DZSTD_MULTITHREAD $(fPIC)" bindir=$(build_private_libexecdir) + +$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted + echo 1 > $@ + +$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured + $(MAKE) -C $(dir $<) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) + echo 1 > $@ + +$(eval $(call staged-install, \ + zstd,$(ZSTD_SRC_DIR), \ + MAKE_INSTALL,$(ZSTD_BUILD_OPTS) MT=1,, \ + $(INSTALL_NAME_CMD)libzstd.$(SHLIB_EXT) $(build_private_libexecdir)/libzstd.$(SHLIB_EXT))) + +clean-zstd: + -rm -f $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled + -$(MAKE) -C $(BUILDDIR)/$(ZSTD_SRC_DIR) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) clean + +get-zstd: $(ZSTD_SRC_FILE) +extract-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted +configure-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured +compile-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled +fastcheck-zstd: check-zstd +check-zstd: compile-zstd + +else # USE_BINARYBUILDER_ZSTD + +$(eval $(call bb-install,zstd,ZSTD,false)) +# move from bindir to shlibdir, where we expect to install it +install-zstd: post-install-zstd +uninstall-zstd: pre-uninstall-zstd +post-install-zstd: $(build_prefix)/manifest/zstd $(PATCHELF_MANIFEST) + mkdir -p $(build_private_libexecdir)/ + [ ! -e $(build_bindir)/zstdmt$(EXE) ] || mv $(build_bindir)/zstdmt$(EXE) $(build_private_libexecdir)/zstdmt$(EXE) + [ ! -e $(build_bindir)/zstd$(EXE) ] || mv $(build_bindir)/zstd$(EXE) $(build_private_libexecdir)/zstd$(EXE) + [ -e $(build_private_libexecdir)/zstd$(EXE) ] + [ -e $(build_private_libexecdir)/zstdmt$(EXE) ] +ifeq ($(OS), Darwin) + for j in zstd zstdmt ; do \ + [ -L $(build_private_libexecdir)/$$j ] && continue; \ + install_name_tool -rpath @executable_path/$(reverse_build_private_libexecdir_rel) @loader_path/$(build_libdir_rel) $(build_private_libexecdir)/$$j 2>/dev/null || true; \ + install_name_tool -rpath @loader_path/$(build_libdir_rel) @executable_path/$(reverse_build_private_libexecdir_rel) $(build_private_libexecdir)/$$j || exit 1; \ + done +else ifneq (,$(findstring $(OS),Linux FreeBSD)) + for j in zstd zstdmt ; do \ + [ -L $(build_private_libexecdir)/$$j ] && continue; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_build_private_libexecdir_rel)' $(build_private_libexecdir)/$$j || exit 1; \ + done +endif + +pre-uninstall-zstd: + -rm -f $(build_private_libexecdir)/zstd$(EXE) $(build_private_libexecdir)/zstdmt$(EXE) + +endif # USE_BINARYBUILDER_ZSTD diff --git a/deps/zstd.version b/deps/zstd.version new file mode 100644 index 0000000000000..d4d960aa6f04b --- /dev/null +++ b/deps/zstd.version @@ -0,0 +1,8 @@ +# -*- makefile -*- +## jll artifact +ZSTD_JLL_NAME := Zstd + +## source build +ZSTD_VER := 1.5.7 +ZSTD_BRANCH=v1.5.7 +ZSTD_SHA1=f8745da6ff1ad1e7bab384bd1f9d742439278e99 diff --git a/julia.spdx.json b/julia.spdx.json index 63683dd302a39..21d07c68560a0 100644 --- a/julia.spdx.json +++ b/julia.spdx.json @@ -420,6 +420,18 @@ "copyrightText": "Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler", "summary": "A massively spiffy yet delicately unobtrusive compression library." }, + { + "name": "zstd", + "SPDXID": "SPDXRef-zstd", + "downloadLocation": "git+https://github.com/facebook/zstd.git", + "filesAnalyzed": false, + "homepage": "https://www.zstd.net", + "sourceInfo": "The git hash of the version in use can be found in the file deps/zstd.version", + "licenseConcluded": "BSD-3-Clause", + "licenseDeclared": "GPL-2.0+ OR BSD-3-Clause", + "copyrightText": "Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.", + "summary": "Zstandard, or zstd as short version, is a fast lossless compression algorithm." + }, { "name": "patchelf", "SPDXID": "SPDXRef-patchelf", @@ -627,6 +639,11 @@ "relationshipType": "BUILD_DEPENDENCY_OF", "relatedSpdxElement": "SPDXRef-JuliaMain" }, + { + "spdxElementId": "SPDXRef-zstd", + "relationshipType": "BUILD_DEPENDENCY_OF", + "relatedSpdxElement": "SPDXRef-JuliaMain" + }, { "spdxElementId": "SPDXRef-patchelf", "relationshipType": "BUILD_TOOL_OF", diff --git a/src/Makefile b/src/Makefile index c25649d2cab7c..c2ff0b0f6b296 100644 --- a/src/Makefile +++ b/src/Makefile @@ -17,7 +17,8 @@ FLAGS := \ -D_GNU_SOURCE -I$(BUILDDIR) -I$(SRCDIR) \ -I$(SRCDIR)/flisp -I$(SRCDIR)/support \ -I$(LIBUV_INC) -I$(build_includedir) \ - -I$(JULIAHOME)/deps/valgrind + -I$(JULIAHOME)/deps/valgrind \ + -I$(ZSTD_INC) FLAGS += -Wall -Wno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden -fno-common \ -Wno-comment -Wpointer-arith -Wundef ifeq ($(USEGCC),1) # GCC bug #25509 (void)__attribute__((warn_unused_result)) @@ -169,7 +170,7 @@ LIBJULIA_PATH_REL := libjulia endif COMMON_LIBPATHS := -L$(build_libdir) -L$(build_shlibdir) -RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) +RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) $(LIBZSTD) CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) RT_DEBUG_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp-debug.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport-debug.a -ljulia-debug $(RT_LIBS) CG_DEBUG_LIBS := $(COMMON_LIBPATHS) $(CG_LIBS) -ljulia-debug -ljulia-internal-debug diff --git a/stdlib/Makefile b/stdlib/Makefile index d478d2ad4c188..eb08979987f75 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -19,7 +19,7 @@ $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) JLLS = DSFMT GMP CURL LIBGIT2 LLVM LIBSSH2 LIBUV MBEDTLS MPFR NGHTTP2 \ BLASTRAMPOLINE OPENBLAS OPENLIBM P7ZIP PCRE LIBSUITESPARSE ZLIB \ - LLVMUNWIND CSL UNWIND LLD + ZSTD LLVMUNWIND CSL UNWIND LLD # Initialize this with JLLs that aren't in "deps/$(LibName).version" JLL_NAMES := MozillaCACerts_jll diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index 5ad675ac21510..489baa7b9732d 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -268,6 +268,11 @@ deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" version = "1.2.13+1" +[[deps.Zstd_jll]] +deps = ["Libdl"] +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + [[deps.dSFMT_jll]] deps = ["Artifacts", "Libdl"] uuid = "05ff407c-b0c1-5878-9df8-858cc2e60c36" diff --git a/stdlib/Zstd_jll/Project.toml b/stdlib/Zstd_jll/Project.toml new file mode 100644 index 0000000000000..467516843390a --- /dev/null +++ b/stdlib/Zstd_jll/Project.toml @@ -0,0 +1,15 @@ +name = "Zstd_jll" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[deps] +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[compat] +julia = "1.6" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/stdlib/Zstd_jll/src/Zstd_jll.jl b/stdlib/Zstd_jll/src/Zstd_jll.jl new file mode 100644 index 0000000000000..c16413f963d0b --- /dev/null +++ b/stdlib/Zstd_jll/src/Zstd_jll.jl @@ -0,0 +1,73 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## dummy stub for https://github.com/JuliaBinaryWrappers/Zstd_jll.j: +# +baremodule Zstd_jll +using Base, Libdl + +export libzstd, zstd, zstdmt + +# These get calculated in __init__() +libzstd_handle::Ptr{Cvoid} = C_NULL + +if Sys.iswindows() + const libzstd = "libzstd-1.dll" +elseif Sys.isapple() + const libzstd = "@rpath/libzstd.1.dylib" +else + const libzstd = "libzstd.so.1" +end + +if Sys.iswindows() + const zstd_exe = "zstd.exe" + const zstdmt_exe = "zstdmt.exe" +else + const zstd_exe = "zstd" + const zstdmt_exe = "zstdmt" +end + +if Sys.iswindows() + const pathsep = ';' +elseif Sys.isapple() + const pathsep = ':' +else + const pathsep = ':' +end + +if Sys.iswindows() +function adjust_ENV(cmd::Cmd) + dllPATH = Sys.BINDIR + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? dllPATH : "$dllPATH$pathsep$oldPATH" + return addenv(cmd, "PATH"=>newPATH) +end +else +adjust_ENV(cmd::Cmd) = cmd +end + +function adjust_ENV() + addPATH = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR) + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? addPATH : "$addPATH$pathsep$oldPATH" + return ("PATH"=>newPATH,) +end + +function zstd(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstd()) + end +end +function zstdmt(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstdmt()) + end +end +zstd() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstd_exe))`) +zstdmt() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstdmt_exe))`) + +function __init__() + global libzstd_handle = dlopen(libzstd) + nothing +end + +end # module Zstd_jll diff --git a/stdlib/Zstd_jll/test/runtests.jl b/stdlib/Zstd_jll/test/runtests.jl new file mode 100644 index 0000000000000..5cfa2a1375c73 --- /dev/null +++ b/stdlib/Zstd_jll/test/runtests.jl @@ -0,0 +1,7 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test, Zstd_jll + +@testset "Zstd_jll" begin + @test ccall((:ZSTD_versionNumber, libzstd), Cuint, ()) == 1_05_07 +end diff --git a/stdlib/stdlib.mk b/stdlib/stdlib.mk index 021639ec0bad0..9670fe49bc0fe 100644 --- a/stdlib/stdlib.mk +++ b/stdlib/stdlib.mk @@ -9,7 +9,7 @@ INDEPENDENT_STDLIBS := \ SparseArrays Statistics StyledStrings SuiteSparse_jll Tar Test TOML Unicode UUIDs \ dSFMT_jll GMP_jll libLLVM_jll LLD_jll LLVMLibUnwind_jll LibUnwind_jll LibUV_jll \ LibCURL_jll LibSSH2_jll LibGit2_jll nghttp2_jll MozillaCACerts_jll MbedTLS_jll \ - MPFR_jll OpenLibm_jll PCRE2_jll p7zip_jll Zlib_jll + MPFR_jll OpenLibm_jll PCRE2_jll p7zip_jll Zlib_jll Zstd_jll STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS) VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) From 5c227a2484ab96c0ab310fd45529872928b08da1 Mon Sep 17 00:00:00 2001 From: Sam Schweigel Date: Wed, 6 Aug 2025 16:35:34 -0700 Subject: [PATCH 04/18] System image compression with zstd Co-authored-by: Gabriel Baraldi madvise with MADV_WILLNEED before accessing sysimage pages MADV_WILLNEED sysimage pages before checksumming or decompressing Even without compression, this gives about an 8% improvement in load times. Fix using jl_page_size before it is initialized Add zstd to ANALYSIS_DEPS now that it is used in staticdata.c Fix jl_page_size being uninitialized in jl_prefetch_system_image Add --compress-sysimage flag; use zstd-15 when enabled --- Make.inc | 2 +- base/options.jl | 1 + base/util.jl | 3 ++ src/Makefile | 4 +- src/aotcompile.cpp | 45 +++++++++++++---- src/jloptions.c | 22 +++++++-- src/jloptions.h | 1 + src/staticdata.c | 117 +++++++++++++++++++++++++++++++++++---------- 8 files changed, 154 insertions(+), 41 deletions(-) diff --git a/Make.inc b/Make.inc index b45886c2af963..1685142e691a8 100644 --- a/Make.inc +++ b/Make.inc @@ -1406,7 +1406,7 @@ JLDFLAGS += -Wl,--stack,8388608 --disable-auto-import --disable-runtime-pseudo-r ifeq ($(ARCH),i686) JLDFLAGS += -Wl,--large-address-aware endif -JCPPFLAGS += -D_WIN32_WINNT=0x0502 +JCPPFLAGS += -D_WIN32_WINNT=_WIN32_WINNT_WIN8 UNTRUSTED_SYSTEM_LIBM := 1 # Use hard links for files on windows, rather than soft links # https://stackoverflow.com/questions/3648819/how-to-make-a-symbolic-link-with-cygwin-in-windows-7 diff --git a/base/options.jl b/base/options.jl index 18fa2ad92654d..0465947801223 100644 --- a/base/options.jl +++ b/base/options.jl @@ -58,6 +58,7 @@ struct JLOptions strip_ir::Int8 permalloc_pkgimg::Int8 heap_size_hint::UInt64 + compress_sysimage::Int8 end # This runs early in the sysimage != is not defined yet diff --git a/base/util.jl b/base/util.jl index 3a621211162ec..f233e6f1d234d 100644 --- a/base/util.jl +++ b/base/util.jl @@ -245,6 +245,9 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Unio if opts.use_sysimage_native_code == 0 push!(addflags, "--sysimage-native-code=no") end + if opts.compress_sysimage == 1 + push!(addflags, "--compress-sysimage=yes") + end return `$julia -C $cpu_target -J$image_file $addflags` end diff --git a/src/Makefile b/src/Makefile index c2ff0b0f6b296..d1cfe867431fc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -171,7 +171,7 @@ endif COMMON_LIBPATHS := -L$(build_libdir) -L$(build_shlibdir) RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) $(LIBZSTD) -CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) +CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) $(LIBZSTD) RT_DEBUG_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp-debug.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport-debug.a -ljulia-debug $(RT_LIBS) CG_DEBUG_LIBS := $(COMMON_LIBPATHS) $(CG_LIBS) -ljulia-debug -ljulia-internal-debug RT_RELEASE_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport.a -ljulia $(RT_LIBS) @@ -474,7 +474,7 @@ $(build_shlibdir)/lib%Plugin.$(SHLIB_EXT): $(SRCDIR)/clangsa/%.cpp $(LLVM_CONFIG # before attempting this static analysis, so that all necessary headers # and dependencies are properly installed: # make -C src install-analysis-deps -ANALYSIS_DEPS := llvm clang llvm-tools libuv utf8proc +ANALYSIS_DEPS := llvm clang llvm-tools libuv utf8proc zstd ifeq ($(OS),Darwin) ANALYSIS_DEPS += llvmunwind else ifneq ($(OS),WINNT) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 05853828d36ce..8132628b5d370 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -57,9 +57,10 @@ #include #include - using namespace llvm; +#include + #include "jitlayers.h" #include "serialize.h" #include "julia_assert.h" @@ -1659,12 +1660,25 @@ void jl_dump_native_impl(void *native_code, sysimgM.setDataLayout(DL); sysimgM.setStackProtectorGuard(StackProtectorGuard); sysimgM.setOverrideStackAlignment(OverrideStackAlignment); - Constant *data = ConstantDataArray::get(Context, - ArrayRef((const unsigned char*)z->buf, z->size)); + + int compression = jl_options.compress_sysimage ? 15 : 0; + ArrayRef sysimg_data{z->buf, (size_t)z->size}; + SmallVector compressed_data; + if (compression) { + compressed_data.resize(ZSTD_compressBound(z->size)); + size_t comp_size = ZSTD_compress(compressed_data.data(), compressed_data.size(), + z->buf, z->size, compression); + compressed_data.resize(comp_size); + sysimg_data = compressed_data; + ios_close(z); + free(z); + } + + Constant *data = ConstantDataArray::get(Context, sysimg_data); auto sysdata = new GlobalVariable(sysimgM, data->getType(), false, GlobalVariable::ExternalLinkage, data, "jl_system_image_data"); - sysdata->setAlignment(Align(64)); + sysdata->setAlignment(Align(jl_page_size)); #if JL_LLVM_VERSION >= 180000 sysdata->setCodeModel(CodeModel::Large); #else @@ -1672,14 +1686,27 @@ void jl_dump_native_impl(void *native_code, sysdata->setSection(".ldata"); #endif addComdat(sysdata, TheTriple); - Constant *len = ConstantInt::get(sysimgM.getDataLayout().getIntPtrType(Context), z->size); + Constant *len = ConstantInt::get(sysimgM.getDataLayout().getIntPtrType(Context), sysimg_data.size()); addComdat(new GlobalVariable(sysimgM, len->getType(), true, GlobalVariable::ExternalLinkage, len, "jl_system_image_size"), TheTriple); - // Free z here, since we've copied out everything into data - // Results in serious memory savings - ios_close(z); - free(z); + + const char *unpack_func = compression ? "jl_image_unpack_zstd" : "jl_image_unpack_uncomp"; + auto unpack = new GlobalVariable(sysimgM, DL.getIntPtrType(Context), true, + GlobalVariable::ExternalLinkage, nullptr, + unpack_func); + addComdat(new GlobalVariable(sysimgM, PointerType::getUnqual(Context), true, + GlobalVariable::ExternalLinkage, unpack, + "jl_image_unpack"), + TheTriple); + + if (!compression) { + // Free z here, since we've copied out everything into data + // Results in serious memory savings + ios_close(z); + free(z); + } + compressed_data.clear(); // Note that we don't set z to null, this allows the check in WRITE_ARCHIVE // to function as expected // no need to free the module/context, destructor handles that diff --git a/src/jloptions.c b/src/jloptions.c index ad13151703531..2b1c0d40005e6 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -101,6 +101,7 @@ JL_DLLEXPORT void jl_init_options(void) 0, // strip-ir 0, // permalloc_pkgimg 0, // heap-size-hint + 0, // compress_sysimage }; jl_options_initialized = 1; } @@ -222,10 +223,13 @@ static const char opts_hidden[] = " Enable or disable JIT compiler, or request exhaustive or minimal compilation\n\n" // compiler output options - " --output-o Generate an object file (including system image data)\n" - " --output-ji Generate a system image data file (.ji)\n" - " --strip-metadata Remove docstrings and source location info from system image\n" - " --strip-ir Remove IR (intermediate representation) of compiled functions\n\n" + " --output-o Generate an object file (including system image data)\n" + " --output-ji Generate a system image data file (.ji)\n" + " --strip-metadata Remove docstrings and source location info from system image\n" + " --strip-ir Remove IR (intermediate representation) of compiled functions\n" + " --compress-sysimage={yes|no*} Compress the sys/pkgimage heap at the expense of\n" + " slightly increased load time.\n" + "\n" // compiler debugging (see the devdocs for tips on using these options) " --output-unopt-bc Generate unoptimized LLVM bitcode (.bc)\n" @@ -282,7 +286,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_strip_ir, opt_heap_size_hint, opt_gc_threads, - opt_permalloc_pkgimg + opt_permalloc_pkgimg, + opt_compress_sysimage, }; static const char* const shortopts = "+vhqH:e:E:L:J:C:it:p:O:g:"; static const struct option longopts[] = { @@ -344,6 +349,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "strip-ir", no_argument, 0, opt_strip_ir }, { "permalloc-pkgimg",required_argument, 0, opt_permalloc_pkgimg }, { "heap-size-hint", required_argument, 0, opt_heap_size_hint }, + { "compress-sysimage", required_argument, 0, opt_compress_sysimage }, { 0, 0, 0, 0 } }; @@ -895,6 +901,12 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) else jl_errorf("julia: invalid argument to --permalloc-pkgimg={yes|no} (%s)", optarg); break; + case opt_compress_sysimage: + if (!strcmp(optarg,"yes")) + jl_options.compress_sysimage = 1; + else if (!strcmp(optarg,"no")) + jl_options.compress_sysimage = 0; + break; default: jl_errorf("julia: unhandled option -- %c\n" "This is a bug, please report it.", c); diff --git a/src/jloptions.h b/src/jloptions.h index f72e72b462066..ca938aa3fb503 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -62,6 +62,7 @@ typedef struct { int8_t strip_ir; int8_t permalloc_pkgimg; uint64_t heap_size_hint; + int8_t compress_sysimage; } jl_options_t; #endif diff --git a/src/staticdata.c b/src/staticdata.c index 3376933eed038..10790422f0539 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -74,6 +74,8 @@ External links: #include // printf #include // PRIxPTR +#include + #include "julia.h" #include "julia_internal.h" #include "julia_gcext.h" @@ -81,8 +83,11 @@ External links: #include "processor.h" #include "serialize.h" -#ifndef _OS_WINDOWS_ +#ifdef _OS_WINDOWS_ +#include +#else #include +#include #endif #include "valgrind.h" @@ -3080,14 +3085,75 @@ JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname) } } -// From a shared library handle, verify consistency and return a jl_image_buf_t -static jl_image_buf_t get_image_buf(void *handle, int is_pkgimage) +typedef void jl_image_unpack_func_t(void *handle, jl_image_buf_t *image); + +static void jl_prefetch_system_image(const char *data, size_t size) +{ + size_t page_size = jl_getpagesize(); /* jl_page_size is not set yet when loading sysimg */ + void *start = (void *)((uintptr_t)data & ~(page_size - 1)); + size_t size_aligned = LLT_ALIGN(size, page_size); +#ifdef _OS_WINDOWS_ + WIN32_MEMORY_RANGE_ENTRY entry = {start, size_aligned}; + PrefetchVirtualMemory(GetCurrentProcess(), 1, &entry, 0); +#else + madvise(start, size_aligned, MADV_WILLNEED); +#endif +} + +JL_DLLEXPORT void jl_image_unpack_uncomp(void *handle, jl_image_buf_t *image) +{ + size_t *plen; + jl_dlsym(handle, "jl_system_image_size", (void **)&plen, 1); + jl_dlsym(handle, "jl_system_image_data", (void **)&image->data, 1); + jl_dlsym(handle, "jl_image_pointers", (void**)&image->pointers, 1); + image->size = *plen; + jl_prefetch_system_image(image->data, image->size); +} + +JL_DLLEXPORT void jl_image_unpack_zstd(void *handle, jl_image_buf_t *image) { size_t *plen; const char *data; - const void *pointers; - uint64_t base; + jl_dlsym(handle, "jl_system_image_size", (void **)&plen, 1); + jl_dlsym(handle, "jl_system_image_data", (void **)&data, 1); + jl_dlsym(handle, "jl_image_pointers", (void **)&image->pointers, 1); + jl_prefetch_system_image(data, *plen); + image->size = ZSTD_getFrameContentSize(data, *plen); + size_t page_size = jl_getpagesize(); /* jl_page_size is not set yet when loading sysimg */ + size_t aligned_size = LLT_ALIGN(image->size, page_size); +#if defined(_OS_WINDOWS_) + size_t large_page_size = GetLargePageMinimum(); + if (image->size > 4 * large_page_size) { + size_t aligned_size = LLT_ALIGN(image->size, large_page_size); + image->data = (char *)VirtualAlloc( + NULL, aligned_size, MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES, PAGE_READWRITE); + } + else { + image->data = (char *)VirtualAlloc(NULL, aligned_size, MEM_COMMIT | MEM_RESERVE, + PAGE_READWRITE); + } +#else + image->data = + (char *)mmap(NULL, aligned_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); +#endif + if (!image->data || image->data == (void *)-1) { + jl_printf(JL_STDERR, "ERROR: failed to allocate space for system image\n"); + jl_exit(1); + } + + ZSTD_decompress((void *)image->data, image->size, data, *plen); + size_t len = (*plen) & ~(page_size - 1); +#ifdef _OS_WINDOWS_ + if (len) + VirtualFree((void *)data, len, MEM_RELEASE); +#else + munmap((void *)data, len); +#endif +} +// From a shared library handle, verify consistency and return a jl_image_buf_t +static jl_image_buf_t get_image_buf(void *handle, int is_pkgimage) +{ // verify that the linker resolved the symbols in this image against ourselves (libjulia-internal) void** (*get_jl_RTLD_DEFAULT_handle_addr)(void) = NULL; if (handle != jl_RTLD_DEFAULT_handle) { @@ -3096,39 +3162,42 @@ static jl_image_buf_t get_image_buf(void *handle, int is_pkgimage) jl_error("Image file failed consistency check: maybe opened the wrong version?"); } + jl_image_unpack_func_t **unpack; + jl_image_buf_t image = { + .kind = JL_IMAGE_KIND_SO, + .handle = handle, + .pointers = NULL, + .data = NULL, + .size = 0, + .base = 0, + }; + // verification passed, lookup the buffer pointers if (jl_system_image_size == 0 || is_pkgimage) { // in the usual case, the sysimage was not statically linked to libjulia-internal // look up the external sysimage symbols via the dynamic linker - jl_dlsym(handle, "jl_system_image_size", (void **)&plen, 1); - jl_dlsym(handle, "jl_system_image_data", (void **)&data, 1); - jl_dlsym(handle, "jl_image_pointers", (void**)&pointers, 1); - } else { + jl_dlsym(handle, "jl_image_unpack", (void **)&unpack, 1); + (*unpack)(handle, &image); + } + else { // the sysimage was statically linked directly against libjulia-internal // use the internal symbols - plen = &jl_system_image_size; - pointers = &jl_image_pointers; - data = &jl_system_image_data; + image.size = jl_system_image_size; + image.pointers = &jl_image_pointers; + image.data = &jl_system_image_data; } #ifdef _OS_WINDOWS_ - base = (intptr_t)handle; + image.base = (intptr_t)handle; #else Dl_info dlinfo; - if (dladdr((void*)pointers, &dlinfo) != 0) - base = (intptr_t)dlinfo.dli_fbase; + if (dladdr((void*)image.pointers, &dlinfo) != 0) + image.base = (intptr_t)dlinfo.dli_fbase; else - base = 0; + image.base = 0; #endif - return (jl_image_buf_t) { - .kind = JL_IMAGE_KIND_SO, - .handle = handle, - .pointers = pointers, - .data = data, - .size = *plen, - .base = base, - }; + return image; } // Allow passing in a module handle directly, rather than a path From b01a5eaef1357cc572c04452e7af7ff060a5d150 Mon Sep 17 00:00:00 2001 From: Sam Schweigel Date: Fri, 5 Sep 2025 17:28:26 -0700 Subject: [PATCH 05/18] Allow linking a system image with jl_image_unpack with libjulia --- src/null_sysimage.c | 6 ++---- src/processor.h | 5 ++--- src/staticdata.c | 9 +++------ 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/null_sysimage.c b/src/null_sysimage.c index 386842f0c4e77..da50640d11b18 100644 --- a/src/null_sysimage.c +++ b/src/null_sysimage.c @@ -7,9 +7,7 @@ * These symbols support statically linking the sysimage with libjulia-internal. * * Here we provide dummy definitions that are used when these are not linked - * together (the default build configuration). The 0 value of jl_system_image_size + * together (the default build configuration). The 0 value of jl_image_unpack * is used as a sentinel to indicate that the sysimage should be loaded externally. **/ -char jl_system_image_data = 0; -size_t jl_system_image_size = 0; -jl_image_pointers_t jl_image_pointers = { 0 }; +jl_image_unpack_func_t *jl_image_unpack = NULL; diff --git a/src/processor.h b/src/processor.h index 65b634fd0ba26..965e020429598 100644 --- a/src/processor.h +++ b/src/processor.h @@ -233,9 +233,8 @@ JL_DLLEXPORT int32_t jl_get_default_nans(void); * libjulia-* and the sysimage together (see null_sysimage.c), in which * case they allow accessing the local copy of the sysimage. **/ -extern char jl_system_image_data; -extern size_t jl_system_image_size; -extern jl_image_pointers_t jl_image_pointers; +typedef void jl_image_unpack_func_t(void *handle, jl_image_buf_t *image); +extern jl_image_unpack_func_t *jl_image_unpack; #ifdef __cplusplus } diff --git a/src/staticdata.c b/src/staticdata.c index 10790422f0539..339e79f91f76e 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3085,7 +3085,6 @@ JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname) } } -typedef void jl_image_unpack_func_t(void *handle, jl_image_buf_t *image); static void jl_prefetch_system_image(const char *data, size_t size) { @@ -3173,19 +3172,17 @@ static jl_image_buf_t get_image_buf(void *handle, int is_pkgimage) }; // verification passed, lookup the buffer pointers - if (jl_system_image_size == 0 || is_pkgimage) { + if (jl_image_unpack == NULL || is_pkgimage) { // in the usual case, the sysimage was not statically linked to libjulia-internal // look up the external sysimage symbols via the dynamic linker jl_dlsym(handle, "jl_image_unpack", (void **)&unpack, 1); - (*unpack)(handle, &image); } else { // the sysimage was statically linked directly against libjulia-internal // use the internal symbols - image.size = jl_system_image_size; - image.pointers = &jl_image_pointers; - image.data = &jl_system_image_data; + unpack = &jl_image_unpack; } + (*unpack)(handle, &image); #ifdef _OS_WINDOWS_ image.base = (intptr_t)handle; From f49d3304981967c88202eba62feb26c085d3149c Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 30 Sep 2025 11:07:59 -0300 Subject: [PATCH 06/18] No GPL 1.11 + compression --- Make.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Make.inc b/Make.inc index 1685142e691a8..1e1e11457ab57 100644 --- a/Make.inc +++ b/Make.inc @@ -214,7 +214,7 @@ JULIA_COMMIT := $(JULIA_VERSION) endif # Whether to use GPL libraries or not. -USE_GPL_LIBS ?= 1 +USE_GPL_LIBS ?= 0 # Whether to install Julia as a framework on Darwin (Apple) platforms. DARWIN_FRAMEWORK ?= 0 From 44101d12ec2f37ba97040d3473a6467b386943b7 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Mon, 22 Sep 2025 14:33:31 -0400 Subject: [PATCH 07/18] Remove GPL libraries from the Julia build for binary-dist target (#59627) Currently we support removing GPL dependencies in the full source build. This will also remove the GPL dependencies from the binary-dist target when built with JLLs. I almost feel like it would be simpler to have a new SuiteSparse_NOGPL_jll package. Then in the default build, things stay as they are. In the no gpl build use the new JLL. In the no GPL build, if someone then tries to use a GPL SuiteSparse library, a warning can be printed asking them to get a different build of Julia. @DilumAluthge @andreasnoack @giordano Thoughts? Co-authored-by: Viral B. Shah --- stdlib/Makefile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/stdlib/Makefile b/stdlib/Makefile index eb08979987f75..bdb6f8a733200 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -15,6 +15,8 @@ include $(JULIAHOME)/deps/*.version VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) DIRS := $(build_datarootdir)/julia/stdlib/$(VERSDIR) $(build_prefix)/manifest/$(VERSDIR) +LIBDIR := $(build_datarootdir)/lib/julia + $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) JLLS = DSFMT GMP CURL LIBGIT2 LLVM LIBSSH2 LIBUV MBEDTLS MPFR NGHTTP2 \ @@ -61,8 +63,19 @@ $(foreach module, $(STDLIBS), $(eval $(call symlink_target,$$(JULIAHOME)/stdlib/ STDLIBS_LINK_TARGETS := $(addprefix $(build_datarootdir)/julia/stdlib/$(VERSDIR)/,$(STDLIBS)) +remove-gpl-libs: +ifeq ($(USE_GPL_LIBS),0) + @echo Removing GPL libs... + -rm -f $(LIBDIR)/libcholmod* + -rm -f $(LIBDIR)/libklu_cholmod* + -rm -f $(LIBDIR)/librbio* + -rm -f $(LIBDIR)/libspqr* + -rm -f $(LIBDIR)/libumfpack* +endif + getall get: $(addprefix get-, $(STDLIBS_EXT) $(JLL_NAMES)) -install: version-check $(addprefix install-, $(STDLIBS_EXT) $(JLL_NAMES)) $(STDLIBS_LINK_TARGETS) + +install: version-check $(addprefix install-, $(STDLIBS_EXT) $(JLL_NAMES)) $(STDLIBS_LINK_TARGETS) remove-gpl-libs version-check: $(addprefix version-check-, $(STDLIBS_EXT)) uninstall: $(addprefix uninstall-, $(STDLIBS_EXT)) extstdlibclean: From 686244ec9ffd0fa32003121889af43f4c2341d67 Mon Sep 17 00:00:00 2001 From: inky Date: Sat, 8 Jun 2024 09:18:14 -0500 Subject: [PATCH 08/18] build: Remove GPL libs when set `USE_GPL_LIBS=0` (#54240) Fix #53211 --- Makefile | 7 +++- THIRDPARTY.md | 14 ++++++- deps/Makefile | 4 +- deps/libsuitesparse.mk | 15 +++++++- stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl | 37 +++++++++++-------- 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index bdc05d75f9a2c..f96671f8d1cd2 100644 --- a/Makefile +++ b/Makefile @@ -206,8 +206,13 @@ JL_PRIVATE_LIBS-0 += libjulia-internal libjulia-codegen else ifeq ($(JULIA_BUILD_MODE),debug) JL_PRIVATE_LIBS-0 += libjulia-internal-debug libjulia-codegen-debug endif +# BSD-3-Clause +JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSUITESPARSE) += libamd libcamd libccolamd libcolamd libsuitesparseconfig +# LGPL-2.1+ +JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSUITESPARSE) += libbtf libklu libldl ifeq ($(USE_GPL_LIBS), 1) -JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSUITESPARSE) += libamd libbtf libcamd libccolamd libcholmod libcolamd libklu libldl librbio libspqr libsuitesparseconfig libumfpack +# GPL-2.0+ +JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSUITESPARSE) += libcholmod librbio libspqr libumfpack endif JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBBLASTRAMPOLINE) += libblastrampoline JL_PRIVATE_LIBS-$(USE_SYSTEM_PCRE) += libpcre2-8 diff --git a/THIRDPARTY.md b/THIRDPARTY.md index 0b2982043d431..ba0a4b9379eb9 100644 --- a/THIRDPARTY.md +++ b/THIRDPARTY.md @@ -41,7 +41,19 @@ Julia's `stdlib` uses the following external libraries, which have their own lic - [OPENBLAS](https://raw.github.com/xianyi/OpenBLAS/master/LICENSE) [BSD-3] - [LAPACK](https://netlib.org/lapack/LICENSE.txt) [BSD-3] - [PCRE](https://www.pcre.org/licence.txt) [BSD-3] -- [SUITESPARSE](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/master/LICENSE.txt) [mix of LGPL2+ and GPL2+; see individual module licenses] +- [SUITESPARSE](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/master/LICENSE.txt) [mix of BSD-3-Clause, LGPL2.1+ and GPL2+; see individual module licenses] + - [`libamd`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/AMD/Doc/License.txt) [BSD-3-Clause] + - [`libcamd`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/CAMD/Doc/License.txt) [BSD-3-Clause] + - [`libccolamd`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/CCOLAMD/Doc/License.txt) [BSD-3-Clause] + - [`libcolamd`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/COLAMD/Doc/License.txt) [BSD-3-Clause] + - [`libsuitesparseconfig`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/SuiteSparse_config/README.txt) [BSD-3-Clause] + - [`libbtf`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/BTF/Doc/License.txt) [LGPL-2.1+] + - [`libklu`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/KLU/Doc/License.txt) [LGPL-2.1+] + - [`libldl`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/LDL/Doc/License.txt) [LGPL-2.1+] + - [`libcholmod`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/CHOLMOD/Doc/License.txt) [LGPL-2.1+ and GPL-2.0+] + - [`librbio`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/RBio/Doc/License.txt) [GPL-2.0+] + - [`libspqr`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/SPQR/Doc/License.txt) [GPL-2.0+] + - [`libumfpack`](https://github.com/DrTimothyAldenDavis/SuiteSparse/blob/dev/UMFPACK/Doc/License.txt) [GPL-2.0+] - [LIBBLASTRAMPOLINE](https://github.com/staticfloat/libblastrampoline/blob/main/LICENSE) [MIT] - [NGHTTP2](https://github.com/nghttp2/nghttp2/blob/master/COPYING) [MIT] diff --git a/deps/Makefile b/deps/Makefile index e942262cf6f0e..403c09aa50072 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -143,11 +143,11 @@ ifeq ($(USE_SYSTEM_MPFR), 0) DEP_LIBS += mpfr endif -ifeq ($(USE_GPL_LIBS), 1) +# Only some of the modules in SuiteSparse are GPL. +# xref: `remove-libsuitesparse-gpl-lib` in libsuitesparse.mk ifeq ($(USE_SYSTEM_LIBSUITESPARSE), 0) DEP_LIBS += libsuitesparse endif -endif ifeq ($(USE_SYSTEM_UTF8PROC), 0) DEP_LIBS += utf8proc diff --git a/deps/libsuitesparse.mk b/deps/libsuitesparse.mk index 7e36bce8f4f9d..7f3a848d0dc23 100644 --- a/deps/libsuitesparse.mk +++ b/deps/libsuitesparse.mk @@ -95,7 +95,7 @@ configure-libsuitesparse: extract-libsuitesparse compile-libsuitesparse: $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/build-compiled fastcheck-libsuitesparse: #none check-libsuitesparse: $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/build-checked -install-libsuitesparse: $(build_prefix)/manifest/libsuitesparse +install-libsuitesparse: $(build_prefix)/manifest/libsuitesparse remove-libsuitesparse-gpl-lib else # USE_BINARYBUILDER_LIBSUITESPARSE @@ -103,6 +103,7 @@ $(eval $(call bb-install,libsuitesparse,LIBSUITESPARSE,false)) # libsuitesparse depends on blastrampoline compile-libsuitesparse: | $(build_prefix)/manifest/blastrampoline +install-libsuitesparse: | remove-libsuitesparse-gpl-lib endif define manual_libsuitesparse @@ -110,3 +111,15 @@ uninstall-libsuitesparse: -rm -f $(build_prefix)/manifest/libsuitesparse -rm -f $(addprefix $(build_shlibdir)/lib,$3) endef + +remove-libsuitesparse-gpl-lib: +ifeq ($(USE_GPL_LIBS),1) + @echo This build contains [GPL-2.0+] libs: libcholmod librbio libspqr libumfpack +else + @echo Removing GPL libs... + -rm -f $(build_bindir)/libcholmod* + -rm -f $(build_bindir)/libklu_cholmod* + -rm -f $(build_bindir)/librbio* + -rm -f $(build_bindir)/libspqr* + -rm -f $(build_bindir)/libumfpack* +endif diff --git a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl index e5edfa76997e1..9e03033c4e3fa 100644 --- a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl +++ b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl @@ -81,29 +81,34 @@ else end function __init__() + # BSD-3-Clause + global libamd_handle = dlopen(libamd) + global libamd_path = dlpath(libamd_handle) + global libcamd_handle = dlopen(libcamd) + global libcamd_path = dlpath(libcamd_handle) + global libccolamd_handle = dlopen(libccolamd) + global libccolamd_path = dlpath(libccolamd_handle) + global libcolamd_handle = dlopen(libcolamd) + global libcolamd_path = dlpath(libcolamd_handle) + global libsuitesparseconfig_handle = dlopen(libsuitesparseconfig) + global libsuitesparseconfig_path = dlpath(libsuitesparseconfig_handle) + + # LGPL-2.1+ + global libbtf_handle = dlopen(libbtf) + global libbtf_path = dlpath(libbtf_handle) + global libklu_handle = dlopen(libklu) + global libklu_path = dlpath(libklu_handle) + global libldl_handle = dlopen(libldl) + global libldl_path = dlpath(libldl_handle) + + # GPL-2.0+ if Base.USE_GPL_LIBS - global libamd_handle = dlopen(libamd) - global libamd_path = dlpath(libamd_handle) - global libbtf_handle = dlopen(libbtf) - global libbtf_path = dlpath(libbtf_handle) - global libcamd_handle = dlopen(libcamd) - global libcamd_path = dlpath(libcamd_handle) - global libccolamd_handle = dlopen(libccolamd) - global libccolamd_path = dlpath(libccolamd_handle) global libcholmod_handle = dlopen(libcholmod) global libcholmod_path = dlpath(libcholmod_handle) - global libcolamd_handle = dlopen(libcolamd) - global libcolamd_path = dlpath(libcolamd_handle) - global libklu_handle = dlopen(libklu) - global libklu_path = dlpath(libklu_handle) - global libldl_handle = dlopen(libldl) - global libldl_path = dlpath(libldl_handle) global librbio_handle = dlopen(librbio) global librbio_path = dlpath(librbio_handle) global libspqr_handle = dlopen(libspqr) global libspqr_path = dlpath(libspqr_handle) - global libsuitesparseconfig_handle = dlopen(libsuitesparseconfig) - global libsuitesparseconfig_path = dlpath(libsuitesparseconfig_handle) global libumfpack_handle = dlopen(libumfpack) global libumfpack_path = dlpath(libumfpack_handle) end From bf568bbb4d153012b66911dcf7a8c1ee7f6f98cb Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Tue, 6 Aug 2024 11:14:58 -0400 Subject: [PATCH 09/18] Disable printing of message about including GPL libs in libsuitesparse.mk (#55387) https://github.com/JuliaLang/julia/pull/54240/files#r1704655126 Co-authored-by: Viral B. Shah --- deps/libsuitesparse.mk | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deps/libsuitesparse.mk b/deps/libsuitesparse.mk index 7f3a848d0dc23..006d1ea51f2c5 100644 --- a/deps/libsuitesparse.mk +++ b/deps/libsuitesparse.mk @@ -113,9 +113,7 @@ uninstall-libsuitesparse: endef remove-libsuitesparse-gpl-lib: -ifeq ($(USE_GPL_LIBS),1) - @echo This build contains [GPL-2.0+] libs: libcholmod librbio libspqr libumfpack -else +ifeq ($(USE_GPL_LIBS),0) @echo Removing GPL libs... -rm -f $(build_bindir)/libcholmod* -rm -f $(build_bindir)/libklu_cholmod* From b3cf0264ea0804486fef942f34ddc8e5f14949e9 Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 30 Sep 2025 16:46:57 -0300 Subject: [PATCH 10/18] Disable Sparse arrays doc --- doc/make.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/make.jl b/doc/make.jl index e8ccbad85c468..4208a900c9136 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -306,12 +306,12 @@ for stdlib in STDLIB_DOCS end # A few standard libraries need more than just the module itself in the DocTestSetup. # This overwrites the existing ones from above though, hence the warn=false. -DocMeta.setdocmeta!( - SparseArrays, - :DocTestSetup, - maybe_revise(:(using SparseArrays, LinearAlgebra)); - recursive=true, warn=false, -) +# DocMeta.setdocmeta!( +# SparseArrays, +# :DocTestSetup, +# maybe_revise(:(using SparseArrays, LinearAlgebra)); +# recursive=true, warn=false, +# ) DocMeta.setdocmeta!( UUIDs, :DocTestSetup, From ad879ee485475a82250fb3109c17725e9849626c Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 30 Sep 2025 17:02:10 -0300 Subject: [PATCH 11/18] Do it differently. --- doc/make.jl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/make.jl b/doc/make.jl index 4208a900c9136..af58cc1b30279 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -26,6 +26,9 @@ cd(joinpath(@__DIR__, "src")) do Base.rm("stdlib"; recursive=true, force=true) mkdir("stdlib") for dir in readdir(STDLIB_DIR) + if dir == "SparseArrays" # Disable Sparse arrays doc + continue + end sourcefile = joinpath(STDLIB_DIR, dir, "docs", "src") if dir in EXT_STDLIB_DOCS sourcefile = joinpath(sourcefile, "basedocs.md") @@ -44,6 +47,7 @@ cd(joinpath(@__DIR__, "src")) do end end + # Because we have standard libraries that are hosted outside of the julia repo, # but their docs are included in the manual, we need to populate the remotes argument # of makedocs(), to make sure that Documenter knows how to resolve the directories @@ -306,12 +310,12 @@ for stdlib in STDLIB_DOCS end # A few standard libraries need more than just the module itself in the DocTestSetup. # This overwrites the existing ones from above though, hence the warn=false. -# DocMeta.setdocmeta!( -# SparseArrays, -# :DocTestSetup, -# maybe_revise(:(using SparseArrays, LinearAlgebra)); -# recursive=true, warn=false, -# ) +DocMeta.setdocmeta!( + SparseArrays, + :DocTestSetup, + maybe_revise(:(using SparseArrays, LinearAlgebra)); + recursive=true, warn=false, +) DocMeta.setdocmeta!( UUIDs, :DocTestSetup, From 0166f10c3b9ad7467a5a4bee262c6caa3d09873e Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 30 Sep 2025 17:12:23 -0300 Subject: [PATCH 12/18] oops --- doc/make.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/make.jl b/doc/make.jl index af58cc1b30279..0a3b7b939b0b8 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -310,12 +310,12 @@ for stdlib in STDLIB_DOCS end # A few standard libraries need more than just the module itself in the DocTestSetup. # This overwrites the existing ones from above though, hence the warn=false. -DocMeta.setdocmeta!( - SparseArrays, - :DocTestSetup, - maybe_revise(:(using SparseArrays, LinearAlgebra)); - recursive=true, warn=false, -) +# DocMeta.setdocmeta!( +# SparseArrays, +# :DocTestSetup, +# maybe_revise(:(using SparseArrays, LinearAlgebra)); +# recursive=true, warn=false, +# ) DocMeta.setdocmeta!( UUIDs, :DocTestSetup, From a998fe12b53d61bce7beb320586e13f6b0910093 Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 30 Sep 2025 17:30:44 -0300 Subject: [PATCH 13/18] Try other things --- doc/Manifest.toml | 2 +- doc/make.jl | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/doc/Manifest.toml b/doc/Manifest.toml index 76bdc332ff36f..f1d4283eff7b9 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -47,7 +47,7 @@ version = "0.9.3" deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] git-tree-sha1 = "d0ea2c044963ed6f37703cead7e29f70cba13d7e" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.8.0" +version = "1.8.1" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] diff --git a/doc/make.jl b/doc/make.jl index 0a3b7b939b0b8..e8ccbad85c468 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -26,9 +26,6 @@ cd(joinpath(@__DIR__, "src")) do Base.rm("stdlib"; recursive=true, force=true) mkdir("stdlib") for dir in readdir(STDLIB_DIR) - if dir == "SparseArrays" # Disable Sparse arrays doc - continue - end sourcefile = joinpath(STDLIB_DIR, dir, "docs", "src") if dir in EXT_STDLIB_DOCS sourcefile = joinpath(sourcefile, "basedocs.md") @@ -47,7 +44,6 @@ cd(joinpath(@__DIR__, "src")) do end end - # Because we have standard libraries that are hosted outside of the julia repo, # but their docs are included in the manual, we need to populate the remotes argument # of makedocs(), to make sure that Documenter knows how to resolve the directories @@ -310,12 +306,12 @@ for stdlib in STDLIB_DOCS end # A few standard libraries need more than just the module itself in the DocTestSetup. # This overwrites the existing ones from above though, hence the warn=false. -# DocMeta.setdocmeta!( -# SparseArrays, -# :DocTestSetup, -# maybe_revise(:(using SparseArrays, LinearAlgebra)); -# recursive=true, warn=false, -# ) +DocMeta.setdocmeta!( + SparseArrays, + :DocTestSetup, + maybe_revise(:(using SparseArrays, LinearAlgebra)); + recursive=true, warn=false, +) DocMeta.setdocmeta!( UUIDs, :DocTestSetup, From 4fe23e995b3718e2e79ceab12bac89aac0e2fc89 Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 30 Sep 2025 17:43:35 -0300 Subject: [PATCH 14/18] one more try --- doc/make.jl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/make.jl b/doc/make.jl index e8ccbad85c468..bd0bbf370aea3 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -26,6 +26,9 @@ cd(joinpath(@__DIR__, "src")) do Base.rm("stdlib"; recursive=true, force=true) mkdir("stdlib") for dir in readdir(STDLIB_DIR) + if dir == "SparseArrays" # Disable Sparse arrays doc + continue + end sourcefile = joinpath(STDLIB_DIR, dir, "docs", "src") if dir in EXT_STDLIB_DOCS sourcefile = joinpath(sourcefile, "basedocs.md") @@ -140,7 +143,7 @@ Manual = [ "manual/methods.md", "manual/constructors.md", "manual/conversion-and-promotion.md", - "manual/interfaces.md", + # "manual/interfaces.md", "manual/modules.md", "manual/documentation.md", "manual/metaprogramming.md", @@ -306,12 +309,12 @@ for stdlib in STDLIB_DOCS end # A few standard libraries need more than just the module itself in the DocTestSetup. # This overwrites the existing ones from above though, hence the warn=false. -DocMeta.setdocmeta!( - SparseArrays, - :DocTestSetup, - maybe_revise(:(using SparseArrays, LinearAlgebra)); - recursive=true, warn=false, -) +# DocMeta.setdocmeta!( +# SparseArrays, +# :DocTestSetup, +# maybe_revise(:(using SparseArrays, LinearAlgebra)); +# recursive=true, warn=false, +# ) DocMeta.setdocmeta!( UUIDs, :DocTestSetup, From 945e8728f6c38dd4e572d1ac3fa09d09d4aa24d7 Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 30 Sep 2025 18:08:56 -0300 Subject: [PATCH 15/18] Disable docs --- doc/make.jl | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/doc/make.jl b/doc/make.jl index bd0bbf370aea3..a6fee16dcd889 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -26,9 +26,6 @@ cd(joinpath(@__DIR__, "src")) do Base.rm("stdlib"; recursive=true, force=true) mkdir("stdlib") for dir in readdir(STDLIB_DIR) - if dir == "SparseArrays" # Disable Sparse arrays doc - continue - end sourcefile = joinpath(STDLIB_DIR, dir, "docs", "src") if dir in EXT_STDLIB_DOCS sourcefile = joinpath(sourcefile, "basedocs.md") @@ -143,7 +140,7 @@ Manual = [ "manual/methods.md", "manual/constructors.md", "manual/conversion-and-promotion.md", - # "manual/interfaces.md", + "manual/interfaces.md", "manual/modules.md", "manual/documentation.md", "manual/metaprogramming.md", @@ -309,12 +306,12 @@ for stdlib in STDLIB_DOCS end # A few standard libraries need more than just the module itself in the DocTestSetup. # This overwrites the existing ones from above though, hence the warn=false. -# DocMeta.setdocmeta!( -# SparseArrays, -# :DocTestSetup, -# maybe_revise(:(using SparseArrays, LinearAlgebra)); -# recursive=true, warn=false, -# ) +DocMeta.setdocmeta!( + SparseArrays, + :DocTestSetup, + maybe_revise(:(using SparseArrays, LinearAlgebra)); + recursive=true, warn=false, +) DocMeta.setdocmeta!( UUIDs, :DocTestSetup, @@ -367,20 +364,21 @@ else end const output_path = joinpath(buildroot, "doc", "_build", (render_pdf ? "pdf" : "html"), "en") -makedocs( - build = output_path, - modules = [Main, Base, Core, [Base.root_module(Base, stdlib.stdlib) for stdlib in STDLIB_DOCS]...], - clean = true, - doctest = ("doctest=fix" in ARGS) ? (:fix) : ("doctest=only" in ARGS) ? (:only) : ("doctest=true" in ARGS) ? true : false, - linkcheck = "linkcheck=true" in ARGS, - linkcheck_ignore = ["https://bugs.kde.org/show_bug.cgi?id=136779"], # fails to load from nanosoldier? - checkdocs = :none, - format = format, - sitename = "The Julia Language", - authors = "The Julia Project", - pages = PAGES, - remotes = documenter_stdlib_remotes, -) +# makedocs( +# build = output_path, +# modules = [Main, Base, Core, [Base.root_module(Base, stdlib.stdlib) for stdlib in STDLIB_DOCS]...], +# clean = true, +# doctest = ("doctest=fix" in ARGS) ? (:fix) : ("doctest=only" in ARGS) ? (:only) : ("doctest=true" in ARGS) ? true : false, +# linkcheck = "linkcheck=true" in ARGS, +# linkcheck_ignore = ["https://bugs.kde.org/show_bug.cgi?id=136779"], # fails to load from nanosoldier? +# checkdocs = :none, +# format = format, +# sitename = "The Julia Language", +# authors = "The Julia Project", +# pages = PAGES, +# warnonly = Documenter.except(:cross_references), +# remotes = documenter_stdlib_remotes, +# ) # Update URLs to external stdlibs (JuliaLang/julia#43199) for (root, _, files) in walkdir(output_path), file in joinpath.(root, files) From 55fef680981473e7d18eec54a1ca95fd3ed77fd4 Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Wed, 1 Oct 2025 10:31:54 -0300 Subject: [PATCH 16/18] Make it work --- doc/make.jl | 46 +- doc/src/manual/interfaces.md | 886 ----------------------------------- 2 files changed, 24 insertions(+), 908 deletions(-) delete mode 100644 doc/src/manual/interfaces.md diff --git a/doc/make.jl b/doc/make.jl index a6fee16dcd889..bac4adfd736e9 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -27,6 +27,9 @@ cd(joinpath(@__DIR__, "src")) do mkdir("stdlib") for dir in readdir(STDLIB_DIR) sourcefile = joinpath(STDLIB_DIR, dir, "docs", "src") + if dir == "SparseArrays" + continue + end if dir in EXT_STDLIB_DOCS sourcefile = joinpath(sourcefile, "basedocs.md") else @@ -140,7 +143,6 @@ Manual = [ "manual/methods.md", "manual/constructors.md", "manual/conversion-and-promotion.md", - "manual/interfaces.md", "manual/modules.md", "manual/documentation.md", "manual/metaprogramming.md", @@ -306,12 +308,12 @@ for stdlib in STDLIB_DOCS end # A few standard libraries need more than just the module itself in the DocTestSetup. # This overwrites the existing ones from above though, hence the warn=false. -DocMeta.setdocmeta!( - SparseArrays, - :DocTestSetup, - maybe_revise(:(using SparseArrays, LinearAlgebra)); - recursive=true, warn=false, -) +# DocMeta.setdocmeta!( +# SparseArrays, +# :DocTestSetup, +# maybe_revise(:(using SparseArrays, LinearAlgebra)); +# recursive=true, warn=false, +# ) DocMeta.setdocmeta!( UUIDs, :DocTestSetup, @@ -364,21 +366,21 @@ else end const output_path = joinpath(buildroot, "doc", "_build", (render_pdf ? "pdf" : "html"), "en") -# makedocs( -# build = output_path, -# modules = [Main, Base, Core, [Base.root_module(Base, stdlib.stdlib) for stdlib in STDLIB_DOCS]...], -# clean = true, -# doctest = ("doctest=fix" in ARGS) ? (:fix) : ("doctest=only" in ARGS) ? (:only) : ("doctest=true" in ARGS) ? true : false, -# linkcheck = "linkcheck=true" in ARGS, -# linkcheck_ignore = ["https://bugs.kde.org/show_bug.cgi?id=136779"], # fails to load from nanosoldier? -# checkdocs = :none, -# format = format, -# sitename = "The Julia Language", -# authors = "The Julia Project", -# pages = PAGES, -# warnonly = Documenter.except(:cross_references), -# remotes = documenter_stdlib_remotes, -# ) +makedocs( + build = output_path, + modules = [Main, Base, Core, [Base.root_module(Base, stdlib.stdlib) for stdlib in STDLIB_DOCS]...], + clean = true, + doctest = ("doctest=fix" in ARGS) ? (:fix) : ("doctest=only" in ARGS) ? (:only) : ("doctest=true" in ARGS) ? true : false, + linkcheck = "linkcheck=true" in ARGS, + linkcheck_ignore = ["https://bugs.kde.org/show_bug.cgi?id=136779"], # fails to load from nanosoldier? + checkdocs = :none, + format = format, + sitename = "The Julia Language", + authors = "The Julia Project", + pages = PAGES, + warnonly = Documenter.except(:cross_references), + remotes = documenter_stdlib_remotes, +) # Update URLs to external stdlibs (JuliaLang/julia#43199) for (root, _, files) in walkdir(output_path), file in joinpath.(root, files) diff --git a/doc/src/manual/interfaces.md b/doc/src/manual/interfaces.md deleted file mode 100644 index d158fb86575a2..0000000000000 --- a/doc/src/manual/interfaces.md +++ /dev/null @@ -1,886 +0,0 @@ -# Interfaces - -A lot of the power and extensibility in Julia comes from a collection of informal interfaces. - By extending a few specific methods to work for a custom type, objects of that type not only -receive those functionalities, but they are also able to be used in other methods that are written -to generically build upon those behaviors. - -## [Iteration](@id man-interface-iteration) - -There are two methods that are always required: - -| Required method | Brief description | -|:----------------------- |:---------------------------------------------------------------------------------------- | -| [`iterate(iter)`](@ref) | Returns either a tuple of the first item and initial state or [`nothing`](@ref) if empty | -| `iterate(iter, state)` | Returns either a tuple of the next item and next state or `nothing` if no items remain | - -There are several more methods that should be defined in some circumstances. -Please note that you should always define at least one of `Base.IteratorSize(IterType)` and `length(iter)` because the default definition of `Base.IteratorSize(IterType)` is `Base.HasLength()`. - -| Method | When should this method be defined? | Default definition | Brief description | -|:--- |:--- |:--- |:--- | -| [`Base.IteratorSize(IterType)`](@ref) | If default is not appropriate | `Base.HasLength()` | One of `Base.HasLength()`, `Base.HasShape{N}()`, `Base.IsInfinite()`, or `Base.SizeUnknown()` as appropriate | -| [`length(iter)`](@ref) | If `Base.IteratorSize()` returns `Base.HasLength()` or `Base.HasShape{N}()` | (*undefined*) | The number of items, if known | -| [`size(iter, [dim])`](@ref) | If `Base.IteratorSize()` returns `Base.HasShape{N}()` | (*undefined*) | The number of items in each dimension, if known | -| [`Base.IteratorEltype(IterType)`](@ref) | If default is not appropriate | `Base.HasEltype()` | Either `Base.EltypeUnknown()` or `Base.HasEltype()` as appropriate | -| [`eltype(IterType)`](@ref) | If default is not appropriate | `Any` | The type of the first entry of the tuple returned by `iterate()` | -| [`Base.isdone(iter, [state])`](@ref) | **Must** be defined if iterator is stateful | `missing` | Fast-path hint for iterator completion. If not defined for a stateful iterator then functions that check for done-ness, like `isempty()` and `zip()`, may mutate the iterator and cause buggy behaviour! | - -Sequential iteration is implemented by the [`iterate`](@ref) function. Instead -of mutating objects as they are iterated over, Julia iterators may keep track -of the iteration state externally from the object. The return value from iterate -is always either a tuple of a value and a state, or `nothing` if no elements remain. -The state object will be passed back to the iterate function on the next iteration -and is generally considered an implementation detail private to the iterable object. - -Any object that defines this function is iterable and can be used in the [many functions that rely upon iteration](@ref lib-collections-iteration). -It can also be used directly in a [`for`](@ref) loop since the syntax: - -```julia -for item in iter # or "for item = iter" - # body -end -``` - -is translated into: - -```julia -next = iterate(iter) -while next !== nothing - (item, state) = next - # body - next = iterate(iter, state) -end -``` - -A simple example is an iterable sequence of square numbers with a defined length: - -```jldoctest squaretype -julia> struct Squares - count::Int - end - -julia> Base.iterate(S::Squares, state=1) = state > S.count ? nothing : (state*state, state+1) -``` - -With only [`iterate`](@ref) definition, the `Squares` type is already pretty powerful. -We can iterate over all the elements: - -```jldoctest squaretype -julia> for item in Squares(7) - println(item) - end -1 -4 -9 -16 -25 -36 -49 -``` - -We can use many of the builtin methods that work with iterables, -like [`in`](@ref) or [`sum`](@ref): - -```jldoctest squaretype -julia> 25 in Squares(10) -true - -julia> sum(Squares(100)) -338350 -``` - -There are a few more methods we can extend to give Julia more information about this iterable -collection. We know that the elements in a `Squares` sequence will always be `Int`. By extending -the [`eltype`](@ref) method, we can give that information to Julia and help it make more specialized -code in the more complicated methods. We also know the number of elements in our sequence, so -we can extend [`length`](@ref), too: - -```jldoctest squaretype -julia> Base.eltype(::Type{Squares}) = Int # Note that this is defined for the type - -julia> Base.length(S::Squares) = S.count -``` - -Now, when we ask Julia to [`collect`](@ref) all the elements into an array it can preallocate a `Vector{Int}` -of the right size instead of naively [`push!`](@ref)ing each element into a `Vector{Any}`: - -```jldoctest squaretype -julia> collect(Squares(4)) -4-element Vector{Int64}: - 1 - 4 - 9 - 16 -``` - -While we can rely upon generic implementations, we can also extend specific methods where we know -there is a simpler algorithm. For example, there's a formula to compute the sum of squares, so -we can override the generic iterative version with a more performant solution: - -```jldoctest squaretype -julia> Base.sum(S::Squares) = (n = S.count; return n*(n+1)*(2n+1)÷6) - -julia> sum(Squares(1803)) -1955361914 -``` - -This is a very common pattern throughout Julia Base: a small set of required methods -define an informal interface that enable many fancier behaviors. In some cases, types will want -to additionally specialize those extra behaviors when they know a more efficient algorithm can -be used in their specific case. - -It is also often useful to allow iteration over a collection in *reverse order* -by iterating over [`Iterators.reverse(iterator)`](@ref). To actually support -reverse-order iteration, however, an iterator -type `T` needs to implement `iterate` for `Iterators.Reverse{T}`. -(Given `r::Iterators.Reverse{T}`, the underling iterator of type `T` is `r.itr`.) -In our `Squares` example, we would implement `Iterators.Reverse{Squares}` methods: - -```jldoctest squaretype -julia> Base.iterate(rS::Iterators.Reverse{Squares}, state=rS.itr.count) = state < 1 ? nothing : (state*state, state-1) - -julia> collect(Iterators.reverse(Squares(4))) -4-element Vector{Int64}: - 16 - 9 - 4 - 1 -``` - -## Indexing - -| Methods to implement | Brief description | -|:-------------------- |:-------------------------------- | -| `getindex(X, i)` | `X[i]`, indexed access, non-scalar `i` should allocate a copy | -| `setindex!(X, v, i)` | `X[i] = v`, indexed assignment | -| `firstindex(X)` | The first index, used in `X[begin]` | -| `lastindex(X)` | The last index, used in `X[end]` | - -For the `Squares` iterable above, we can easily compute the `i`th element of the sequence by squaring -it. We can expose this as an indexing expression `S[i]`. To opt into this behavior, `Squares` -simply needs to define [`getindex`](@ref): - -```jldoctest squaretype -julia> function Base.getindex(S::Squares, i::Int) - 1 <= i <= S.count || throw(BoundsError(S, i)) - return i*i - end - -julia> Squares(100)[23] -529 -``` - -Additionally, to support the syntax `S[begin]` and `S[end]`, we must define [`firstindex`](@ref) and -[`lastindex`](@ref) to specify the first and last valid indices, respectively: - -```jldoctest squaretype -julia> Base.firstindex(S::Squares) = 1 - -julia> Base.lastindex(S::Squares) = length(S) - -julia> Squares(23)[end] -529 -``` - -For multi-dimensional `begin`/`end` indexing as in `a[3, begin, 7]`, for example, -you should define `firstindex(a, dim)` and `lastindex(a, dim)` -(which default to calling `first` and `last` on `axes(a, dim)`, respectively). - -Note, though, that the above *only* defines [`getindex`](@ref) with one integer index. Indexing with -anything other than an `Int` will throw a [`MethodError`](@ref) saying that there was no matching method. -In order to support indexing with ranges or vectors of `Int`s, separate methods must be written: - -```jldoctest squaretype -julia> Base.getindex(S::Squares, i::Number) = S[convert(Int, i)] - -julia> Base.getindex(S::Squares, I) = [S[i] for i in I] - -julia> Squares(10)[[3,4.,5]] -3-element Vector{Int64}: - 9 - 16 - 25 -``` - -While this is starting to support more of the [indexing operations supported by some of the builtin types](@ref man-array-indexing), -there's still quite a number of behaviors missing. This `Squares` sequence is starting to look -more and more like a vector as we've added behaviors to it. Instead of defining all these behaviors -ourselves, we can officially define it as a subtype of an [`AbstractArray`](@ref). - -## [Abstract Arrays](@id man-interface-array) - -| Methods to implement | | Brief description | -|:----------------------------------------------- |:-------------------------------------- |:------------------------------------------------------------------------------------- | -| `size(A)` | | Returns a tuple containing the dimensions of `A` | -| `getindex(A, i::Int)` | | (if `IndexLinear`) Linear scalar indexing | -| `getindex(A, I::Vararg{Int, N})` | | (if `IndexCartesian`, where `N = ndims(A)`) N-dimensional scalar indexing | -| **Optional methods** | **Default definition** | **Brief description** | -| `IndexStyle(::Type)` | `IndexCartesian()` | Returns either `IndexLinear()` or `IndexCartesian()`. See the description below. | -| `setindex!(A, v, i::Int)` | | (if `IndexLinear`) Scalar indexed assignment | -| `setindex!(A, v, I::Vararg{Int, N})` | | (if `IndexCartesian`, where `N = ndims(A)`) N-dimensional scalar indexed assignment | -| `getindex(A, I...)` | defined in terms of scalar `getindex` | [Multidimensional and nonscalar indexing](@ref man-array-indexing) | -| `setindex!(A, X, I...)` | defined in terms of scalar `setindex!` | [Multidimensional and nonscalar indexed assignment](@ref man-array-indexing) | -| `iterate` | defined in terms of scalar `getindex` | Iteration | -| `length(A)` | `prod(size(A))` | Number of elements | -| `similar(A)` | `similar(A, eltype(A), size(A))` | Return a mutable array with the same shape and element type | -| `similar(A, ::Type{S})` | `similar(A, S, size(A))` | Return a mutable array with the same shape and the specified element type | -| `similar(A, dims::Dims)` | `similar(A, eltype(A), dims)` | Return a mutable array with the same element type and size *dims* | -| `similar(A, ::Type{S}, dims::Dims)` | `Array{S}(undef, dims)` | Return a mutable array with the specified element type and size | -| **Non-traditional indices** | **Default definition** | **Brief description** | -| `axes(A)` | `map(OneTo, size(A))` | Return a tuple of `AbstractUnitRange{<:Integer}` of valid indices. The axes should be their own axes, that is `axes.(axes(A),1) == axes(A)` should be satisfied. | -| `similar(A, ::Type{S}, inds)` | `similar(A, S, Base.to_shape(inds))` | Return a mutable array with the specified indices `inds` (see below) | -| `similar(T::Union{Type,Function}, inds)` | `T(Base.to_shape(inds))` | Return an array similar to `T` with the specified indices `inds` (see below) | - -If a type is defined as a subtype of `AbstractArray`, it inherits a very large set of rich behaviors -including iteration and multidimensional indexing built on top of single-element access. See -the [arrays manual page](@ref man-multi-dim-arrays) and the [Julia Base section](@ref lib-arrays) for more supported methods. - -A key part in defining an `AbstractArray` subtype is [`IndexStyle`](@ref). Since indexing is -such an important part of an array and often occurs in hot loops, it's important to make both -indexing and indexed assignment as efficient as possible. Array data structures are typically -defined in one of two ways: either it most efficiently accesses its elements using just one index -(linear indexing) or it intrinsically accesses the elements with indices specified for every dimension. - These two modalities are identified by Julia as `IndexLinear()` and `IndexCartesian()`. - Converting a linear index to multiple indexing subscripts is typically very expensive, so this -provides a traits-based mechanism to enable efficient generic code for all array types. - -This distinction determines which scalar indexing methods the type must define. `IndexLinear()` -arrays are simple: just define `getindex(A::ArrayType, i::Int)`. When the array is subsequently -indexed with a multidimensional set of indices, the fallback `getindex(A::AbstractArray, I...)` -efficiently converts the indices into one linear index and then calls the above method. `IndexCartesian()` -arrays, on the other hand, require methods to be defined for each supported dimensionality with -`ndims(A)` `Int` indices. For example, [`SparseMatrixCSC`](@ref) from the `SparseArrays` standard -library module, only supports two dimensions, so it just defines -`getindex(A::SparseMatrixCSC, i::Int, j::Int)`. The same holds for [`setindex!`](@ref). - -Returning to the sequence of squares from above, we could instead define it as a subtype of an -`AbstractArray{Int, 1}`: - -```jldoctest squarevectype -julia> struct SquaresVector <: AbstractArray{Int, 1} - count::Int - end - -julia> Base.size(S::SquaresVector) = (S.count,) - -julia> Base.IndexStyle(::Type{<:SquaresVector}) = IndexLinear() - -julia> Base.getindex(S::SquaresVector, i::Int) = i*i -``` - -Note that it's very important to specify the two parameters of the `AbstractArray`; the first -defines the [`eltype`](@ref), and the second defines the [`ndims`](@ref). That supertype and those three -methods are all it takes for `SquaresVector` to be an iterable, indexable, and completely functional -array: - -```jldoctest squarevectype -julia> s = SquaresVector(4) -4-element SquaresVector: - 1 - 4 - 9 - 16 - -julia> s[s .> 8] -2-element Vector{Int64}: - 9 - 16 - -julia> s + s -4-element Vector{Int64}: - 2 - 8 - 18 - 32 - -julia> sin.(s) -4-element Vector{Float64}: - 0.8414709848078965 - -0.7568024953079282 - 0.4121184852417566 - -0.2879033166650653 -``` - -As a more complicated example, let's define our own toy N-dimensional sparse-like array type built -on top of [`Dict`](@ref): - -```jldoctest squarevectype -julia> struct SparseArray{T,N} <: AbstractArray{T,N} - data::Dict{NTuple{N,Int}, T} - dims::NTuple{N,Int} - end - -julia> SparseArray(::Type{T}, dims::Int...) where {T} = SparseArray(T, dims); - -julia> SparseArray(::Type{T}, dims::NTuple{N,Int}) where {T,N} = SparseArray{T,N}(Dict{NTuple{N,Int}, T}(), dims); - -julia> Base.size(A::SparseArray) = A.dims - -julia> Base.similar(A::SparseArray, ::Type{T}, dims::Dims) where {T} = SparseArray(T, dims) - -julia> Base.getindex(A::SparseArray{T,N}, I::Vararg{Int,N}) where {T,N} = get(A.data, I, zero(T)) - -julia> Base.setindex!(A::SparseArray{T,N}, v, I::Vararg{Int,N}) where {T,N} = (A.data[I] = v) -``` - -Notice that this is an `IndexCartesian` array, so we must manually define [`getindex`](@ref) and [`setindex!`](@ref) -at the dimensionality of the array. Unlike the `SquaresVector`, we are able to define [`setindex!`](@ref), -and so we can mutate the array: - -```jldoctest squarevectype -julia> A = SparseArray(Float64, 3, 3) -3×3 SparseArray{Float64, 2}: - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - -julia> fill!(A, 2) -3×3 SparseArray{Float64, 2}: - 2.0 2.0 2.0 - 2.0 2.0 2.0 - 2.0 2.0 2.0 - -julia> A[:] = 1:length(A); A -3×3 SparseArray{Float64, 2}: - 1.0 4.0 7.0 - 2.0 5.0 8.0 - 3.0 6.0 9.0 -``` - -The result of indexing an `AbstractArray` can itself be an array (for instance when indexing by -an `AbstractRange`). The `AbstractArray` fallback methods use [`similar`](@ref) to allocate an `Array` -of the appropriate size and element type, which is filled in using the basic indexing method described -above. However, when implementing an array wrapper you often want the result to be wrapped as -well: - -```jldoctest squarevectype -julia> A[1:2,:] -2×3 SparseArray{Float64, 2}: - 1.0 4.0 7.0 - 2.0 5.0 8.0 -``` - -In this example it is accomplished by defining `Base.similar(A::SparseArray, ::Type{T}, dims::Dims) where T` -to create the appropriate wrapped array. (Note that while `similar` supports 1- and 2-argument -forms, in most case you only need to specialize the 3-argument form.) For this to work it's important -that `SparseArray` is mutable (supports `setindex!`). Defining `similar`, `getindex` and -`setindex!` for `SparseArray` also makes it possible to [`copy`](@ref) the array: - -```jldoctest squarevectype -julia> copy(A) -3×3 SparseArray{Float64, 2}: - 1.0 4.0 7.0 - 2.0 5.0 8.0 - 3.0 6.0 9.0 -``` - -In addition to all the iterable and indexable methods from above, these types can also interact -with each other and use most of the methods defined in Julia Base for `AbstractArrays`: - -```jldoctest squarevectype -julia> A[SquaresVector(3)] -3-element SparseArray{Float64, 1}: - 1.0 - 4.0 - 9.0 - -julia> sum(A) -45.0 -``` - -If you are defining an array type that allows non-traditional indexing (indices that start at -something other than 1), you should specialize [`axes`](@ref). You should also specialize [`similar`](@ref) -so that the `dims` argument (ordinarily a `Dims` size-tuple) can accept `AbstractUnitRange` objects, -perhaps range-types `Ind` of your own design. For more information, see -[Arrays with custom indices](@ref man-custom-indices). - -## [Strided Arrays](@id man-interface-strided-arrays) - -| Methods to implement | | Brief description | -|:----------------------------------------------- |:-------------------------------------- |:------------------------------------------------------------------------------------- | -| `strides(A)` | | Return the distance in memory (in number of elements) between adjacent elements in each dimension as a tuple. If `A` is an `AbstractArray{T,0}`, this should return an empty tuple. | -| `Base.unsafe_convert(::Type{Ptr{T}}, A)` | | Return the native address of an array. | -| `Base.elsize(::Type{<:A})` | | Return the stride between consecutive elements in the array. | -| **Optional methods** | **Default definition** | **Brief description** | -| `stride(A, i::Int)` | `strides(A)[i]` | Return the distance in memory (in number of elements) between adjacent elements in dimension k. | - -A strided array is a subtype of `AbstractArray` whose entries are stored in memory with fixed strides. -Provided the element type of the array is compatible with BLAS, a strided array can utilize BLAS and LAPACK routines -for more efficient linear algebra routines. A typical example of a user-defined strided array is one -that wraps a standard `Array` with additional structure. - -Warning: do not implement these methods if the underlying storage is not actually strided, as it -may lead to incorrect results or segmentation faults. - -Here are some examples to demonstrate which type of arrays are strided and which are not: -```julia -1:5 # not strided (there is no storage associated with this array.) -Vector(1:5) # is strided with strides (1,) -A = [1 5; 2 6; 3 7; 4 8] # is strided with strides (1,4) -V = view(A, 1:2, :) # is strided with strides (1,4) -V = view(A, 1:2:3, 1:2) # is strided with strides (2,4) -V = view(A, [1,2,4], :) # is not strided, as the spacing between rows is not fixed. -``` - - - - - -## [Customizing broadcasting](@id man-interfaces-broadcasting) - -| Methods to implement | Brief description | -|:-------------------- |:----------------- | -| `Base.BroadcastStyle(::Type{SrcType}) = SrcStyle()` | Broadcasting behavior of `SrcType` | -| `Base.similar(bc::Broadcasted{DestStyle}, ::Type{ElType})` | Allocation of output container | -| **Optional methods** | | | -| `Base.BroadcastStyle(::Style1, ::Style2) = Style12()` | Precedence rules for mixing styles | -| `Base.axes(x)` | Declaration of the indices of `x`, as per [`axes(x)`](@ref). | -| `Base.broadcastable(x)` | Convert `x` to an object that has `axes` and supports indexing | -| **Bypassing default machinery** | | -| `Base.copy(bc::Broadcasted{DestStyle})` | Custom implementation of `broadcast` | -| `Base.copyto!(dest, bc::Broadcasted{DestStyle})` | Custom implementation of `broadcast!`, specializing on `DestStyle` | -| `Base.copyto!(dest::DestType, bc::Broadcasted{Nothing})` | Custom implementation of `broadcast!`, specializing on `DestType` | -| `Base.Broadcast.broadcasted(f, args...)` | Override the default lazy behavior within a fused expression | -| `Base.Broadcast.instantiate(bc::Broadcasted{DestStyle})` | Override the computation of the lazy broadcast's axes | - -[Broadcasting](@ref) is triggered by an explicit call to `broadcast` or `broadcast!`, or implicitly by -"dot" operations like `A .+ b` or `f.(x, y)`. Any object that has [`axes`](@ref) and supports -indexing can participate as an argument in broadcasting, and by default the result is stored -in an `Array`. This basic framework is extensible in three major ways: - -* Ensuring that all arguments support broadcast -* Selecting an appropriate output array for the given set of arguments -* Selecting an efficient implementation for the given set of arguments - -Not all types support `axes` and indexing, but many are convenient to allow in broadcast. -The [`Base.broadcastable`](@ref) function is called on each argument to broadcast, allowing -it to return something different that supports `axes` and indexing. By -default, this is the identity function for all `AbstractArray`s and `Number`s — they already -support `axes` and indexing. - -If a type is intended to act like a "0-dimensional scalar" (a single object) rather than as a -container for broadcasting, then the following method should be defined: -```julia -Base.broadcastable(o::MyType) = Ref(o) -``` -that returns the argument wrapped in a 0-dimensional [`Ref`](@ref) container. For example, such a wrapper -method is defined for types themselves, functions, special singletons like [`missing`](@ref) and [`nothing`](@ref), and dates. - -Custom array-like types can specialize -`Base.broadcastable` to define their shape, but they should follow the convention that -`collect(Base.broadcastable(x)) == collect(x)`. A notable exception is `AbstractString`; -strings are special-cased to behave as scalars for the purposes of broadcast even though -they are iterable collections of their characters (see [Strings](@ref) for more). - -The next two steps (selecting the output array and implementation) are dependent upon -determining a single answer for a given set of arguments. Broadcast must take all the varied -types of its arguments and collapse them down to just one output array and one -implementation. Broadcast calls this single answer a "style". Every broadcastable object -each has its own preferred style, and a promotion-like system is used to combine these -styles into a single answer — the "destination style". - -### Broadcast Styles - -`Base.BroadcastStyle` is the abstract type from which all broadcast styles are derived. When used as a -function it has two possible forms, unary (single-argument) and binary. The unary variant states -that you intend to implement specific broadcasting behavior and/or output type, and do not wish to -rely on the default fallback [`Broadcast.DefaultArrayStyle`](@ref). - -To override these defaults, you can define a custom `BroadcastStyle` for your object: - -```julia -struct MyStyle <: Broadcast.BroadcastStyle end -Base.BroadcastStyle(::Type{<:MyType}) = MyStyle() -``` - -In some cases it might be convenient not to have to define `MyStyle`, in which case you can -leverage one of the general broadcast wrappers: - - - `Base.BroadcastStyle(::Type{<:MyType}) = Broadcast.Style{MyType}()` can be - used for arbitrary types. - - `Base.BroadcastStyle(::Type{<:MyType}) = Broadcast.ArrayStyle{MyType}()` is preferred - if `MyType` is an `AbstractArray`. - - For `AbstractArrays` that only support a certain dimensionality, create a subtype of `Broadcast.AbstractArrayStyle{N}` (see below). - -When your broadcast operation involves several arguments, individual argument styles get -combined to determine a single `DestStyle` that controls the type of the output container. -For more details, see [below](@ref writing-binary-broadcasting-rules). - -### Selecting an appropriate output array - -The broadcast style is computed for every broadcasting operation to allow for -dispatch and specialization. The actual allocation of the result array is -handled by `similar`, using the Broadcasted object as its first argument. - -```julia -Base.similar(bc::Broadcasted{DestStyle}, ::Type{ElType}) -``` - -The fallback definition is - -```julia -similar(bc::Broadcasted{DefaultArrayStyle{N}}, ::Type{ElType}) where {N,ElType} = - similar(Array{ElType}, axes(bc)) -``` - -However, if needed you can specialize on any or all of these arguments. The final argument -`bc` is a lazy representation of a (potentially fused) broadcast operation, a `Broadcasted` -object. For these purposes, the most important fields of the wrapper are -`f` and `args`, describing the function and argument list, respectively. Note that the argument -list can — and often does — include other nested `Broadcasted` wrappers. - -For a complete example, let's say you have created a type, `ArrayAndChar`, that stores an -array and a single character: - -```jldoctest ArrayAndChar; output = false -struct ArrayAndChar{T,N} <: AbstractArray{T,N} - data::Array{T,N} - char::Char -end -Base.size(A::ArrayAndChar) = size(A.data) -Base.getindex(A::ArrayAndChar{T,N}, inds::Vararg{Int,N}) where {T,N} = A.data[inds...] -Base.setindex!(A::ArrayAndChar{T,N}, val, inds::Vararg{Int,N}) where {T,N} = A.data[inds...] = val -Base.showarg(io::IO, A::ArrayAndChar, toplevel) = print(io, typeof(A), " with char '", A.char, "'") -# output - -``` - -You might want broadcasting to preserve the `char` "metadata". First we define - -```jldoctest ArrayAndChar; output = false -Base.BroadcastStyle(::Type{<:ArrayAndChar}) = Broadcast.ArrayStyle{ArrayAndChar}() -# output - -``` - -This means we must also define a corresponding `similar` method: -```jldoctest ArrayAndChar; output = false -function Base.similar(bc::Broadcast.Broadcasted{Broadcast.ArrayStyle{ArrayAndChar}}, ::Type{ElType}) where ElType - # Scan the inputs for the ArrayAndChar: - A = find_aac(bc) - # Use the char field of A to create the output - ArrayAndChar(similar(Array{ElType}, axes(bc)), A.char) -end - -"`A = find_aac(As)` returns the first ArrayAndChar among the arguments." -find_aac(bc::Base.Broadcast.Broadcasted) = find_aac(bc.args) -find_aac(args::Tuple) = find_aac(find_aac(args[1]), Base.tail(args)) -find_aac(x) = x -find_aac(::Tuple{}) = nothing -find_aac(a::ArrayAndChar, rest) = a -find_aac(::Any, rest) = find_aac(rest) -# output -find_aac (generic function with 6 methods) -``` - -From these definitions, one obtains the following behavior: -```jldoctest ArrayAndChar -julia> a = ArrayAndChar([1 2; 3 4], 'x') -2×2 ArrayAndChar{Int64, 2} with char 'x': - 1 2 - 3 4 - -julia> a .+ 1 -2×2 ArrayAndChar{Int64, 2} with char 'x': - 2 3 - 4 5 - -julia> a .+ [5,10] -2×2 ArrayAndChar{Int64, 2} with char 'x': - 6 7 - 13 14 -``` - -### [Extending broadcast with custom implementations](@id extending-in-place-broadcast) - -In general, a broadcast operation is represented by a lazy `Broadcasted` container that holds onto -the function to be applied alongside its arguments. Those arguments may themselves be more nested -`Broadcasted` containers, forming a large expression tree to be evaluated. A nested tree of -`Broadcasted` containers is directly constructed by the implicit dot syntax; `5 .+ 2.*x` is -transiently represented by `Broadcasted(+, 5, Broadcasted(*, 2, x))`, for example. This is -invisible to users as it is immediately realized through a call to `copy`, but it is this container -that provides the basis for broadcast's extensibility for authors of custom types. The built-in -broadcast machinery will then determine the result type and size based upon the arguments, allocate -it, and then finally copy the realization of the `Broadcasted` object into it with a default -`copyto!(::AbstractArray, ::Broadcasted)` method. The built-in fallback `broadcast` and -`broadcast!` methods similarly construct a transient `Broadcasted` representation of the operation -so they can follow the same codepath. This allows custom array implementations to -provide their own `copyto!` specialization to customize and -optimize broadcasting. This is again determined by the computed broadcast style. This is such -an important part of the operation that it is stored as the first type parameter of the -`Broadcasted` type, allowing for dispatch and specialization. - -For some types, the machinery to "fuse" operations across nested levels of broadcasting -is not available or could be done more efficiently incrementally. In such cases, you may -need or want to evaluate `x .* (x .+ 1)` as if it had been -written `broadcast(*, x, broadcast(+, x, 1))`, where the inner operation is evaluated before -tackling the outer operation. This sort of eager operation is directly supported by a bit -of indirection; instead of directly constructing `Broadcasted` objects, Julia lowers the -fused expression `x .* (x .+ 1)` to `Broadcast.broadcasted(*, x, Broadcast.broadcasted(+, x, 1))`. Now, -by default, `broadcasted` just calls the `Broadcasted` constructor to create the lazy representation -of the fused expression tree, but you can choose to override it for a particular combination -of function and arguments. - -As an example, the builtin `AbstractRange` objects use this machinery to optimize pieces -of broadcasted expressions that can be eagerly evaluated purely in terms of the start, -step, and length (or stop) instead of computing every single element. Just like all the -other machinery, `broadcasted` also computes and exposes the combined broadcast style of its -arguments, so instead of specializing on `broadcasted(f, args...)`, you can specialize on -`broadcasted(::DestStyle, f, args...)` for any combination of style, function, and arguments. - -For example, the following definition supports the negation of ranges: - -```julia -broadcasted(::DefaultArrayStyle{1}, ::typeof(-), r::OrdinalRange) = range(-first(r), step=-step(r), length=length(r)) -``` - -### [Extending in-place broadcasting](@id extending-in-place-broadcast) - -In-place broadcasting can be supported by defining the appropriate `copyto!(dest, bc::Broadcasted)` -method. Because you might want to specialize either on `dest` or the specific subtype of `bc`, -to avoid ambiguities between packages we recommend the following convention. - -If you wish to specialize on a particular style `DestStyle`, define a method for -```julia -copyto!(dest, bc::Broadcasted{DestStyle}) -``` -Optionally, with this form you can also specialize on the type of `dest`. - -If instead you want to specialize on the destination type `DestType` without specializing -on `DestStyle`, then you should define a method with the following signature: - -```julia -copyto!(dest::DestType, bc::Broadcasted{Nothing}) -``` - -This leverages a fallback implementation of `copyto!` that converts the wrapper into a -`Broadcasted{Nothing}`. Consequently, specializing on `DestType` has lower precedence than -methods that specialize on `DestStyle`. - -Similarly, you can completely override out-of-place broadcasting with a `copy(::Broadcasted)` -method. - -#### Working with `Broadcasted` objects - -In order to implement such a `copy` or `copyto!`, method, of course, you must -work with the `Broadcasted` wrapper to compute each element. There are two main -ways of doing so: - -* `Broadcast.flatten` recomputes the potentially nested operation into a single - function and flat list of arguments. You are responsible for implementing the - broadcasting shape rules yourself, but this may be helpful in limited situations. -* Iterating over the `CartesianIndices` of the `axes(::Broadcasted)` and using - indexing with the resulting `CartesianIndex` object to compute the result. - -### [Writing binary broadcasting rules](@id writing-binary-broadcasting-rules) - -The precedence rules are defined by binary `BroadcastStyle` calls: - -```julia -Base.BroadcastStyle(::Style1, ::Style2) = Style12() -``` - -where `Style12` is the `BroadcastStyle` you want to choose for outputs involving -arguments of `Style1` and `Style2`. For example, - -```julia -Base.BroadcastStyle(::Broadcast.Style{Tuple}, ::Broadcast.AbstractArrayStyle{0}) = Broadcast.Style{Tuple}() -``` - -indicates that `Tuple` "wins" over zero-dimensional arrays (the output container will be a tuple). -It is worth noting that you do not need to (and should not) define both argument orders -of this call; defining one is sufficient no matter what order the user supplies the arguments in. - -For `AbstractArray` types, defining a `BroadcastStyle` supersedes the fallback choice, -[`Broadcast.DefaultArrayStyle`](@ref). `DefaultArrayStyle` and the abstract supertype, `AbstractArrayStyle`, store the dimensionality as a type parameter to support specialized -array types that have fixed dimensionality requirements. - -`DefaultArrayStyle` "loses" to any other -`AbstractArrayStyle` that has been defined because of the following methods: - -```julia -BroadcastStyle(a::AbstractArrayStyle{Any}, ::DefaultArrayStyle) = a -BroadcastStyle(a::AbstractArrayStyle{N}, ::DefaultArrayStyle{N}) where N = a -BroadcastStyle(a::AbstractArrayStyle{M}, ::DefaultArrayStyle{N}) where {M,N} = - typeof(a)(Val(max(M, N))) -``` - -You do not need to write binary `BroadcastStyle` -rules unless you want to establish precedence for -two or more non-`DefaultArrayStyle` types. - -If your array type does have fixed dimensionality requirements, then you should -subtype `AbstractArrayStyle`. For example, the sparse array code has the following definitions: - -```julia -struct SparseVecStyle <: Broadcast.AbstractArrayStyle{1} end -struct SparseMatStyle <: Broadcast.AbstractArrayStyle{2} end -Base.BroadcastStyle(::Type{<:SparseVector}) = SparseVecStyle() -Base.BroadcastStyle(::Type{<:SparseMatrixCSC}) = SparseMatStyle() -``` - -Whenever you subtype `AbstractArrayStyle`, you also need to define rules for combining -dimensionalities, by creating a constructor for your style that takes a `Val(N)` argument. -For example: - -```julia -SparseVecStyle(::Val{0}) = SparseVecStyle() -SparseVecStyle(::Val{1}) = SparseVecStyle() -SparseVecStyle(::Val{2}) = SparseMatStyle() -SparseVecStyle(::Val{N}) where N = Broadcast.DefaultArrayStyle{N}() -``` - -These rules indicate that the combination of a `SparseVecStyle` with 0- or 1-dimensional arrays -yields another `SparseVecStyle`, that its combination with a 2-dimensional array -yields a `SparseMatStyle`, and anything of higher dimensionality falls back to the dense arbitrary-dimensional framework. -These rules allow broadcasting to keep the sparse representation for operations that result -in one or two dimensional outputs, but produce an `Array` for any other dimensionality. - -## [Instance Properties](@id man-instance-properties) - -| Methods to implement | Default definition | Brief description | -|:--------------------------------- |:---------------------------- |:------------------------------------------------------------------------------------- | -| `propertynames(x::ObjType, private::Bool=false)` | `fieldnames(typeof(x))` | Return a tuple of the properties (`x.property`) of an object `x`. If `private=true`, also return property names intended to be kept as private | -| `getproperty(x::ObjType, s::Symbol)` | `getfield(x, s)` | Return property `s` of `x`. `x.s` calls `getproperty(x, :s)`. | -| `setproperty!(x::ObjType, s::Symbol, v)` | `setfield!(x, s, v)` | Set property `s` of `x` to `v`. `x.s = v` calls `setproperty!(x, :s, v)`. Should return `v`.| - -Sometimes, it is desirable to change how the end-user interacts with the fields of an object. -Instead of granting direct access to type fields, an extra layer of abstraction between -the user and the code can be provided by overloading `object.field`. Properties are what the -user *sees of* the object, fields what the object *actually is*. - -By default, properties and fields are the same. However, this behavior can be changed. -For example, take this representation of a point in a plane in [polar coordinates](https://en.wikipedia.org/wiki/Polar_coordinate_system): - -```jldoctest polartype -julia> mutable struct Point - r::Float64 - ϕ::Float64 - end - -julia> p = Point(7.0, pi/4) -Point(7.0, 0.7853981633974483) -``` - -As described in the table above dot access `p.r` is the same as `getproperty(p, :r)` which is by default the same as `getfield(p, :r)`: - -```jldoctest polartype -julia> propertynames(p) -(:r, :ϕ) - -julia> getproperty(p, :r), getproperty(p, :ϕ) -(7.0, 0.7853981633974483) - -julia> p.r, p.ϕ -(7.0, 0.7853981633974483) - -julia> getfield(p, :r), getproperty(p, :ϕ) -(7.0, 0.7853981633974483) -``` - -However, we may want users to be unaware that `Point` stores the coordinates as `r` and `ϕ` (fields), -and instead interact with `x` and `y` (properties). The methods in the first column can be -defined to add new functionality: - -```jldoctest polartype -julia> Base.propertynames(::Point, private::Bool=false) = private ? (:x, :y, :r, :ϕ) : (:x, :y) - -julia> function Base.getproperty(p::Point, s::Symbol) - if s === :x - return getfield(p, :r) * cos(getfield(p, :ϕ)) - elseif s === :y - return getfield(p, :r) * sin(getfield(p, :ϕ)) - else - # This allows accessing fields with p.r and p.ϕ - return getfield(p, s) - end - end - -julia> function Base.setproperty!(p::Point, s::Symbol, f) - if s === :x - y = p.y - setfield!(p, :r, sqrt(f^2 + y^2)) - setfield!(p, :ϕ, atan(y, f)) - return f - elseif s === :y - x = p.x - setfield!(p, :r, sqrt(x^2 + f^2)) - setfield!(p, :ϕ, atan(f, x)) - return f - else - # This allow modifying fields with p.r and p.ϕ - return setfield!(p, s, f) - end - end -``` - -It is important that `getfield` and `setfield` are used inside `getproperty` and `setproperty!` instead of the dot syntax, -since the dot syntax would make the functions recursive which can lead to type inference issues. We can now -try out the new functionality: - -```jldoctest polartype -julia> propertynames(p) -(:x, :y) - -julia> p.x -4.949747468305833 - -julia> p.y = 4.0 -4.0 - -julia> p.r -6.363961030678928 -``` - -Finally, it is worth noting that adding instance properties like this is quite -rarely done in Julia and should in general only be done if there is a good -reason for doing so. - -## [Rounding](@id man-rounding-interface) - -| Methods to implement | Default definition | Brief description | -|:--------------------------------------------- |:------------------------- |:--------------------------------------------------------------------------------------------------- | -| `round(x::ObjType, r::RoundingMode)` | none | Round `x` and return the result. If possible, round should return an object of the same type as `x` | -| `round(T::Type, x::ObjType, r::RoundingMode)` | `convert(T, round(x, r))` | Round `x`, returning the result as a `T` | - -To support rounding on a new type it is typically sufficient to define the single method -`round(x::ObjType, r::RoundingMode)`. The passed rounding mode determines in which direction -the value should be rounded. The most commonly used rounding modes are `RoundNearest`, -`RoundToZero`, `RoundDown`, and `RoundUp`, as these rounding modes are used in the -definitions of the one argument `round`, method, and `trunc`, `floor`, and `ceil`, -respectively. - -In some cases, it is possible to define a three-argument `round` method that is more -accurate or performant than the two-argument method followed by conversion. In this case it -is acceptable to define the three argument method in addition to the two argument method. -If it is impossible to represent the rounded result as an object of the type `T`, -then the three argument method should throw an `InexactError`. - -For example, if we have an `Interval` type which represents a range of possible values -similar to https://github.com/JuliaPhysics/Measurements.jl, we may define rounding on that -type with the following - -```jldoctest -julia> struct Interval{T} - min::T - max::T - end - -julia> Base.round(x::Interval, r::RoundingMode) = Interval(round(x.min, r), round(x.max, r)) - -julia> x = Interval(1.7, 2.2) -Interval{Float64}(1.7, 2.2) - -julia> round(x) -Interval{Float64}(2.0, 2.0) - -julia> floor(x) -Interval{Float64}(1.0, 2.0) - -julia> ceil(x) -Interval{Float64}(2.0, 3.0) - -julia> trunc(x) -Interval{Float64}(1.0, 2.0) -``` From 3516305202a517077a44c2c75bde4ec8f6df51c3 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Fri, 3 Oct 2025 18:06:20 -0300 Subject: [PATCH 17/18] Disable failing tests --- test/choosetests.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/choosetests.jl b/test/choosetests.jl index 36da1ab39bc57..01eaf011121e9 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -165,6 +165,7 @@ function choosetests(choices = []) filtertests!(tests, "internet_required", INTERNET_REQUIRED_LIST) # do ambiguous first to avoid failing if ambiguities are introduced by other tests filtertests!(tests, "ambiguous") + filter!(!contains("SparseArrays/fixed"), tests) # temporarily disable SparseArrays/fixed if startswith(string(Sys.ARCH), "arm") # Remove profile from default tests on ARM since it currently segfaults From c96e1cc6c83b46b3373b51eb5696f3828fdcd20a Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Mon, 6 Oct 2025 11:02:00 -0300 Subject: [PATCH 18/18] Actually fix tests --- test/choosetests.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/choosetests.jl b/test/choosetests.jl index 01eaf011121e9..a5dff61a5ec9d 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -165,7 +165,6 @@ function choosetests(choices = []) filtertests!(tests, "internet_required", INTERNET_REQUIRED_LIST) # do ambiguous first to avoid failing if ambiguities are introduced by other tests filtertests!(tests, "ambiguous") - filter!(!contains("SparseArrays/fixed"), tests) # temporarily disable SparseArrays/fixed if startswith(string(Sys.ARCH), "arm") # Remove profile from default tests on ARM since it currently segfaults @@ -228,7 +227,7 @@ function choosetests(choices = []) # Filter out tests from the test groups in the stdlibs filter!(!in(tests), unhandled) filter!(!in(skip_tests), tests) - + filter!(!contains("SparseArrays/fixed"), tests) # temporarily disable SparseArrays/fixed if !isempty(unhandled) @warn "Not skipping tests: $(join(unhandled, ", "))" end