batch some binding changes (JuliaLang#57765)

For model simplicity (and eventually for better unspecialized
compilation), try to backdate initial module operations back to world 0
if nobody could have observed them before (no using statements depend
on the module yet), rather than putting each incremental operation in a
separate, but unobserved (and unobservable) world increment.

Author: vtjnash

base/Base_compiler.jl

@@ -1,9 +1,8 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-baremodule Base
+module Base
 
-using Core
-using Core.Intrinsics, Core.IR
+using .Core.Intrinsics, .Core.IR
 
 # to start, we're going to use a very simple definition of `include`
 # that doesn't require any function (except what we can get from the `Core` top-module)
@@ -375,5 +374,5 @@ Core._setparser!(fl_parse)
 
 # Further definition of Base will happen in Base.jl if loaded.
 
-end # baremodule Base
+end # module Base
 using .Base

base/sysimg.jl

@@ -1,19 +1,12 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-# Can be loaded on top of either an existing system image built from
-# `Base_compiler.jl` or standalone, in which case we will build it now.
-let had_compiler = isdefined(Main, :Base)
-if had_compiler; else
-include("Base_compiler.jl")
-end
-
-Core.include(Base, "Base.jl")
+Base.Core.include(Base, "Base.jl") # finish populating Base (currently just has the Compiler)
 
-had_compiler && ccall(:jl_init_restored_module, Cvoid, (Any,), Base)
-end
+# Set up Main module by importing from Base
+using .Base
+using .Base.MainInclude # ans, err, and sometimes Out
 
-# Set up Main module
-using Base.MainInclude # ans, err, and sometimes Out
+ccall(:jl_init_restored_module, Cvoid, (Any,), Base)
 
 # These definitions calls Base._include rather than Base.include to get
 # one-frame stacktraces for the common case of using include(fname) in Main.
@@ -59,7 +52,7 @@ definition of `eval`, which evaluates expressions in that module.
 const eval = Core.EvalInto(Main)
 
 # Ensure this file is also tracked
-pushfirst!(Base._included_files, (@__MODULE__, abspath(@__FILE__)))
+pushfirst!(Base._included_files, (Main, abspath(@__FILE__)))
 
 # set up depot & load paths to be able to find stdlib packages
 Base.init_depot_path()

doc/src/devdocs/init.md

@@ -63,7 +63,7 @@ the [LLVM library](https://llvm.org).
 If there is no sysimg file (`!jl_options.image_file`) then the `Core` and `Main` modules are
 created and `boot.jl` is evaluated:
 
-`jl_core_module = jl_new_module(jl_symbol("Core"))` creates the Julia `Core` module.
+`jl_core_module = jl_new_module(jl_symbol("Core"), NULL)` creates the Julia `Core` module.
 
 [`jl_init_intrinsic_functions()`](https://github.com/JuliaLang/julia/blob/master/src/intrinsics.cpp)
 creates a new Julia module `Intrinsics` containing constant `jl_intrinsic_type` symbols. These define

src/builtins.c

@@ -2396,8 +2396,7 @@ static void add_intrinsic(jl_module_t *inm, const char *name, enum intrinsic f)
 {
     jl_value_t *i = jl_permbox32(jl_intrinsic_type, 0, (int32_t)f);
     jl_sym_t *sym = jl_symbol(name);
-    jl_set_const(inm, sym, i);
-    jl_module_public(inm, sym, 1);
+    jl_set_initial_const(inm, sym, i, 1);
 }
 
 void jl_init_intrinsic_properties(void) JL_GC_DISABLED
@@ -2413,9 +2412,8 @@ void jl_init_intrinsic_properties(void) JL_GC_DISABLED
 
 void jl_init_intrinsic_functions(void) JL_GC_DISABLED
 {
-    jl_module_t *inm = jl_new_module(jl_symbol("Intrinsics"), NULL);
-    inm->parent = jl_core_module;
-    jl_set_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm);
+    jl_module_t *inm = jl_new_module_(jl_symbol("Intrinsics"), jl_core_module, 0, 1);
+    jl_set_initial_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm, 0);
     jl_mk_builtin_func(jl_intrinsic_type, "IntrinsicFunction", jl_f_intrinsic_call);
     jl_mk_builtin_func(
         (jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_opaque_closure_type),
@@ -2437,7 +2435,7 @@ void jl_init_intrinsic_functions(void) JL_GC_DISABLED
 
 static void add_builtin(const char *name, jl_value_t *v)
 {
-    jl_set_const(jl_core_module, jl_symbol(name), v);
+    jl_set_initial_const(jl_core_module, jl_symbol(name), v, 0);
 }
 
 jl_fptr_args_t jl_get_builtin_fptr(jl_datatype_t *dt)

src/gf.c

@@ -294,7 +294,7 @@ jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_a
     if (dt == NULL) {
         // Builtins are specially considered available from world 0
         jl_value_t *f = jl_new_generic_function_with_supertype(sname, jl_core_module, jl_builtin_type, 0);
-        jl_set_const(jl_core_module, sname, f);
+        jl_set_initial_const(jl_core_module, sname, f, 0);
         dt = (jl_datatype_t*)jl_typeof(f);
     }
 

src/jl_exported_funcs.inc

@@ -2,7 +2,6 @@
 
 #define JL_RUNTIME_EXPORTED_FUNCS(XX) \
     XX(jl_active_task_stack) \
-    XX(jl_add_standard_imports) \
     XX(jl_adopt_thread) \
     XX(jl_alignment) \
     XX(jl_alloc_array_1d) \
@@ -98,7 +97,6 @@
     XX(jl_cstr_to_string) \
     XX(jl_current_exception) \
     XX(jl_debug_method_invalidation) \
-    XX(jl_defines_or_exports_p) \
     XX(jl_deprecate_binding) \
     XX(jl_dlclose) \
     XX(jl_dlopen) \
@@ -315,7 +313,6 @@
     XX(jl_module_names) \
     XX(jl_module_parent) \
     XX(jl_module_getloc) \
-    XX(jl_module_public) \
     XX(jl_module_public_p) \
     XX(jl_module_use) \
     XX(jl_module_using) \

src/jltypes.c

@@ -3287,10 +3287,9 @@ void jl_init_types(void) JL_GC_DISABLED
                         jl_emptysvec, 0, 0, 3);
 
     core = jl_new_module(jl_symbol("Core"), NULL);
-    core->parent = core;
     jl_type_typename->mt->module = core;
     jl_core_module = core;
-    core = NULL; // not ready yet to use
+    core = NULL; // not actually ready yet to use
 
     tv = jl_svec1(tvar("Backend"));
     jl_addrspace_typename =
@@ -3381,9 +3380,8 @@ void jl_init_types(void) JL_GC_DISABLED
     core = jl_core_module;
     jl_atomic_store_relaxed(&core->bindingkeyset, (jl_genericmemory_t*)jl_an_empty_memory_any);
     // export own name, so "using Foo" makes "Foo" itself visible
-    jl_set_const(core, core->name, (jl_value_t*)core);
-    jl_module_public(core, core->name, 1);
-    jl_set_const(core, jl_symbol("CPU"), (jl_value_t*)cpumem);
+    jl_set_initial_const(core, core->name, (jl_value_t*)core, 1);
+    jl_set_initial_const(core, jl_symbol("CPU"), (jl_value_t*)cpumem, 0);
     core = NULL;
 
     jl_expr_type =

src/julia.h

@@ -2096,13 +2096,13 @@ JL_DLLEXPORT void jl_check_binding_currently_writable(jl_binding_t *b, jl_module
 JL_DLLEXPORT jl_binding_t *jl_get_binding_wr(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var);
 JL_DLLEXPORT jl_value_t *jl_get_existing_strong_gf(jl_binding_t *b JL_PROPAGATES_ROOT, size_t new_world);
 JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import);
-JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var);
 JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var);
 JL_DLLEXPORT int jl_globalref_is_const(jl_globalref_t *gr);
 JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr);
 JL_DLLEXPORT jl_value_t *jl_get_global(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var);
 JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT);
 JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT);
+void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT, int exported);
 JL_DLLEXPORT void jl_checked_assignment(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED);
 JL_DLLEXPORT jl_value_t *jl_checked_swap(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED);
 JL_DLLEXPORT jl_value_t *jl_checked_replace(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *expected, jl_value_t *rhs);
@@ -2115,10 +2115,9 @@ JL_DLLEXPORT void jl_module_use(jl_task_t *ct, jl_module_t *to, jl_module_t *fro
 JL_DLLEXPORT void jl_module_use_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname);
 JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s);
 JL_DLLEXPORT void jl_module_import_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname);
-JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported);
+int jl_module_public_(jl_module_t *from, jl_sym_t *s, int exported, size_t new_world);
 JL_DLLEXPORT int jl_is_imported(jl_module_t *m, jl_sym_t *s);
 JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var);
-JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m);
 
 // eq hash tables
 JL_DLLEXPORT jl_genericmemory_t *jl_eqtable_put(jl_genericmemory_t *h JL_ROOTING_ARGUMENT, jl_value_t *key, jl_value_t *val JL_ROOTED_ARGUMENT, int *inserted);

src/julia_internal.h

@@ -869,6 +869,7 @@ JL_DLLEXPORT void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t
 JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3(jl_binding_t *b JL_ROOTING_ARGUMENT, jl_module_t *mod, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, enum jl_partition_kind, size_t new_world) JL_GLOBALLY_ROOTED;
 JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *m, jl_value_t *e, int fast, int expanded, const char **toplevel_filename, int *toplevel_lineno);
 
+void jl_module_initial_using(jl_module_t *to, jl_module_t *from);
 STATIC_INLINE struct _jl_module_using *module_usings_getidx(jl_module_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT;
 STATIC_INLINE jl_module_t *module_usings_getmod(jl_module_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT;
 void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to);
@@ -941,7 +942,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b
 JL_DLLEXPORT void jl_update_loaded_bpart(jl_binding_t *b, jl_binding_partition_t *bpart);
 extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED;
 extern htable_t jl_current_modules JL_GLOBALLY_ROOTED;
-extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED;
+extern jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED;
 extern jl_genericmemory_t *jl_global_roots_list JL_GLOBALLY_ROOTED;
 extern jl_genericmemory_t *jl_global_roots_keyset JL_GLOBALLY_ROOTED;
 extern arraylist_t *jl_entrypoint_mis;
@@ -1275,9 +1276,8 @@ _Atomic(jl_value_t*) *jl_table_peek_bp(jl_genericmemory_t *a, jl_value_t *key) J
 
 JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t*);
 
-JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent);
-JL_DLLEXPORT jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name);
-JL_DLLEXPORT void jl_add_default_names(jl_module_t *m, uint8_t default_using_core, uint8_t self_name);
+jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name);
+jl_module_t *jl_add_standard_imports(jl_module_t *m);
 JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module);
 JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, int mt_cache);
 jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_svec_t *sp) JL_PROPAGATES_ROOT;

src/module.c

@@ -401,7 +401,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_leaf_partitions_value_if_const(jl_bindin
     return NULL;
 }
 
-JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent)
+static jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent)
 {
     jl_task_t *ct = jl_current_task;
     const jl_uuid_t uuid_zero = {0, 0};
@@ -410,7 +410,7 @@ JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent)
     jl_set_typetagof(m, jl_module_tag, 0);
     assert(jl_is_symbol(name));
     m->name = name;
-    m->parent = parent;
+    m->parent = parent ? parent : m;
     m->istopmod = 0;
     m->uuid = uuid_zero;
     static unsigned int mcounter; // simple counter backup, in case hrtime is not incrementing
@@ -437,23 +437,22 @@ JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent)
     return m;
 }
 
-JL_DLLEXPORT void jl_add_default_names(jl_module_t *m, uint8_t default_using_core, uint8_t self_name)
+static void jl_add_default_names(jl_module_t *m, uint8_t default_using_core, uint8_t self_name)
 {
     if (jl_core_module) {
         // Bootstrap: Before jl_core_module is defined, we don't have enough infrastructure
         // for bindings, so Core itself gets special handling in jltypes.c
         if (default_using_core) {
-            jl_module_using(m, jl_core_module);
+            jl_module_initial_using(m, jl_core_module);
         }
         if (self_name) {
             // export own name, so "using Foo" makes "Foo" itself visible
-            jl_set_const(m, m->name, (jl_value_t*)m);
-            jl_module_public(m, m->name, 1);
+            jl_set_initial_const(m, m->name, (jl_value_t*)m, 1);
         }
     }
 }
 
-JL_DLLEXPORT jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name)
+jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name)
 {
     jl_module_t *m = jl_new_module__(name, parent);
     JL_GC_PUSH1(&m);
@@ -1231,6 +1230,19 @@ void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to)
     JL_UNLOCK(&from->lock);
 }
 
+void jl_module_initial_using(jl_module_t *to, jl_module_t *from)
+{
+    struct _jl_module_using new_item = {
+        .mod = from,
+        .min_world = 0,
+        .max_world = ~(size_t)0
+    };
+    arraylist_grow(&to->usings, sizeof(struct _jl_module_using)/sizeof(void*));
+    memcpy(&to->usings.items[to->usings.len-3], &new_item, sizeof(struct _jl_module_using));
+    jl_gc_wb(to, from);
+    jl_add_usings_backedge(from, to);
+}
+
 JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from)
 {
     if (to == from)
@@ -1323,11 +1335,10 @@ JL_DLLEXPORT jl_value_t *jl_get_module_binding_or_nothing(jl_module_t *m, jl_sym
     return (jl_value_t*)b;
 }
 
-JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported)
+int jl_module_public_(jl_module_t *from, jl_sym_t *s, int exported, size_t new_world)
 {
+    // caller must hold world_counter_lock
     jl_binding_t *b = jl_get_module_binding(from, s, 1);
-    JL_LOCK(&world_counter_lock);
-    size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1;
     jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world);
     int was_exported = (bpart->kind & PARTITION_FLAG_EXPORTED) != 0;
     if (jl_atomic_load_relaxed(&b->flags) & BINDING_FLAG_PUBLICP) {
@@ -1342,9 +1353,9 @@ JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported)
     jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_PUBLICP);
     if (was_exported != exported) {
         jl_replace_binding_locked2(b, bpart, bpart->restriction, bpart->kind | PARTITION_FLAG_EXPORTED, new_world);
-        jl_atomic_store_release(&jl_world_counter, new_world);
+        return 1;
     }
-    JL_UNLOCK(&world_counter_lock);
+    return 0;
 }
 
 JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import) // unlike most queries here, this is currently seq_cst
@@ -1370,13 +1381,6 @@ JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import) // u
     return jl_atomic_load(&b->value) != NULL;
 }
 
-JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var)
-{
-    jl_binding_t *b = jl_get_module_binding(m, var, 0);
-    jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age);
-    return b && ((bpart->kind & PARTITION_FLAG_EXPORTED) || jl_binding_kind(bpart) == PARTITION_KIND_GLOBAL);
-}
-
 JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var)
 {
     jl_binding_t *b = jl_get_module_binding(m, var, 0);
@@ -1475,9 +1479,25 @@ JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *va
     jl_checked_assignment(bp, m, var, val);
 }
 
+JL_DLLEXPORT void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT, int exported)
+{
+    // this function is only valid during initialization, so there is no risk of data races her are not too important to use
+    int kind = PARTITION_KIND_CONST | (exported ? PARTITION_FLAG_EXPORTED : 0);
+    // jl_declare_constant_val3(NULL, m, var, (jl_value_t*)jl_any_type, kind, 0);
+    jl_binding_t *bp = jl_get_module_binding(m, var, 1);
+    jl_binding_partition_t *bpart = jl_get_binding_partition(bp, 0);
+    assert(bpart->min_world == 0);
+    jl_atomic_store_relaxed(&bpart->max_world, ~(size_t)0); // jl_check_new_binding_implicit likely incorrectly truncated it
+    if (exported)
+        jl_atomic_fetch_or_relaxed(&bp->flags, BINDING_FLAG_PUBLICP);
+    bpart->kind = kind | (bpart->kind & PARTITION_MASK_FLAG);
+    bpart->restriction = val;
+    jl_gc_wb(bpart, val);
+}
+
 JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT)
 {
-    // this function is mostly only used during initialization, so the data races here are not too important to us
+    // this function is dangerous and unsound. do not use.
     jl_binding_t *bp = jl_get_module_binding(m, var, 1);
     jl_binding_partition_t *bpart = jl_get_binding_partition(bp, jl_current_task->world_age);
     bpart->min_world = 0;