Merge branch 'master' into add-memoryref-addrspace-test

Author: rickwoi

src/datatype.c

@@ -354,7 +354,8 @@ STATIC_INLINE void jl_maybe_allocate_singleton_instance(jl_datatype_t *st) JL_NO
     if (st->instance)
         return;
     if (jl_is_datatype_make_singleton(st)) {
-        st->instance = jl_gc_permobj(0, st, 0);
+        jl_task_t *ct = jl_current_task;
+        st->instance = jl_gc_permobj(ct->ptls, 0, st, 0);
     }
 }
 
@@ -616,7 +617,8 @@ void jl_get_genericmemory_layout(jl_datatype_t *st)
 
     if (jl_is_addrspacecore(addrspace) && jl_unbox_uint8(addrspace) == 0) {
         if (kind == (jl_value_t*)jl_not_atomic_sym || kind == (jl_value_t*)jl_atomic_sym) {
-            jl_genericmemory_t *zeroinst = (jl_genericmemory_t*)jl_gc_permobj(LLT_ALIGN(sizeof(jl_genericmemory_t), JL_SMALL_BYTE_ALIGNMENT) + (elsz ? elsz : isunion), st, 0);
+            jl_task_t *ct = jl_current_task;
+            jl_genericmemory_t *zeroinst = (jl_genericmemory_t*)jl_gc_permobj(ct->ptls, LLT_ALIGN(sizeof(jl_genericmemory_t), JL_SMALL_BYTE_ALIGNMENT) + (elsz ? elsz : isunion), st, 0);
             zeroinst->length = 0;
             zeroinst->ptr = (char*)zeroinst + JL_SMALL_BYTE_ALIGNMENT;
             memset(zeroinst->ptr, 0, elsz ? elsz : isunion);
@@ -1458,12 +1460,13 @@ JL_DLLEXPORT int jl_atomic_storeonce_bits(jl_datatype_t *dt, char *dst, const jl
 
 #define PERMBOXN_FUNC(nb)                                                  \
     jl_value_t *jl_permbox##nb(jl_datatype_t *t, uintptr_t tag, uint##nb##_t x) JL_NOTSAFEPOINT \
-    {   /* n.b. t must be a concrete isbits datatype of the right size */  \
-        jl_value_t *v = jl_gc_permobj(LLT_ALIGN(nb, sizeof(void*)), t, 0); \
-        assert(tag);                                                       \
-        jl_set_typetagof(v, tag, GC_OLD_MARKED);                           \
-        *(uint##nb##_t*)jl_data_ptr(v) = x;                                \
-        return v;                                                          \
+    {   /* n.b. t must be a concrete isbits datatype of the right size */               \
+        jl_task_t *ct = jl_current_task;                                                \
+        jl_value_t *v = jl_gc_permobj(ct->ptls, LLT_ALIGN(nb, sizeof(void*)), t, 0);    \
+        assert(tag);                                                                    \
+        jl_set_typetagof(v, tag, GC_OLD_MARKED);                                        \
+        *(uint##nb##_t*)jl_data_ptr(v) = x;                                             \
+        return v;                                                                       \
     }
 PERMBOXN_FUNC(8)
 PERMBOXN_FUNC(16)

src/gc-interface.h

@@ -209,7 +209,7 @@ JL_DLLEXPORT void *jl_gc_perm_alloc(size_t sz, int zero, unsigned align,
 //              necessarily mark any objects that would have ordinarily been rooted by
 //              the allocated object. All objects stored in fields of this object
 //              must be either permanently allocated or have other roots.
-struct _jl_value_t *jl_gc_permobj(size_t sz, void *ty, unsigned align) JL_NOTSAFEPOINT;
+struct _jl_value_t *jl_gc_permobj(struct _jl_tls_states_t *ptls, size_t sz, void *ty, unsigned align) JL_NOTSAFEPOINT;
 // This function notifies the GC about memory addresses that are set when loading the boot image.
 // The GC may use that information to, for instance, determine that such objects should
 // be treated as marked and belonged to the old generation in nursery collections.

src/gc-mmtk.c

@@ -506,6 +506,9 @@ JL_DLLEXPORT void jl_gc_scan_vm_specific_roots(RootsWorkClosure* closure)
     // add module
     add_node_to_roots_buffer(closure, &buf, &len, jl_main_module);
 
+    // add global_method_table
+    add_node_to_roots_buffer(closure, &buf, &len, jl_method_table);
+
     // buildin values
     add_node_to_roots_buffer(closure, &buf, &len, jl_an_empty_vec_any);
     add_node_to_roots_buffer(closure, &buf, &len, jl_module_init_order);
@@ -1019,30 +1022,29 @@ JL_DLLEXPORT void *jl_gc_counted_realloc_with_old_size(void *p, size_t old, size
     return realloc(p, sz);
 }
 
-void *jl_gc_perm_alloc_nolock(size_t sz, int zero, unsigned align, unsigned offset)
+void *jl_gc_perm_alloc_nolock(jl_ptls_t ptls, size_t sz, int zero, unsigned align, unsigned offset)
 {
-    jl_ptls_t ptls = jl_current_task->ptls;
     size_t allocsz = mmtk_align_alloc_sz(sz);
     void* addr = mmtk_immortal_alloc_fast(&ptls->gc_tls.mmtk_mutator, allocsz, align, offset);
     return addr;
 }
 
 void *jl_gc_perm_alloc(size_t sz, int zero, unsigned align, unsigned offset)
 {
-    return jl_gc_perm_alloc_nolock(sz, zero, align, offset);
+    jl_ptls_t ptls = jl_current_task->ptls;
+    return jl_gc_perm_alloc_nolock(ptls, sz, zero, align, offset);
 }
 
-jl_value_t *jl_gc_permobj(size_t sz, void *ty, unsigned align) JL_NOTSAFEPOINT
+jl_value_t *jl_gc_permobj(jl_ptls_t ptls, size_t sz, void *ty, unsigned align) JL_NOTSAFEPOINT
 {
     const size_t allocsz = sz + sizeof(jl_taggedvalue_t);
     if (align == 0) {
         align = ((sz == 0) ? sizeof(void*) : (allocsz <= sizeof(void*) * 2 ?
                                                  sizeof(void*) * 2 : 16));
     }
-    jl_taggedvalue_t *o = (jl_taggedvalue_t*)jl_gc_perm_alloc(allocsz, 0, align,
+    jl_taggedvalue_t *o = (jl_taggedvalue_t*)jl_gc_perm_alloc_nolock(ptls, allocsz, 0, align,
                                                               sizeof(void*) % align);
 
-    jl_ptls_t ptls = jl_current_task->ptls;
     mmtk_immortal_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, jl_valueof(o), allocsz);
     o->header = (uintptr_t)ty;
     return jl_valueof(o);

src/gc-stock.c

@@ -3957,7 +3957,7 @@ void *jl_gc_perm_alloc(size_t sz, int zero, unsigned align, unsigned offset)
     return p;
 }
 
-jl_value_t *jl_gc_permobj(size_t sz, void *ty, unsigned align) JL_NOTSAFEPOINT
+jl_value_t *jl_gc_permobj(jl_ptls_t _ptls, size_t sz, void *ty, unsigned align) JL_NOTSAFEPOINT
 {
     const size_t allocsz = sz + sizeof(jl_taggedvalue_t);
     if (align == 0) {

src/init.c

@@ -776,12 +776,12 @@ JL_DLLEXPORT void jl_init_(jl_image_buf_t sysimage)
         jl_atomic_fetch_add(&jl_task_metrics_enabled, 1);
     }
     // Initialize constant objects
-    jl_nothing = jl_gc_permobj(0, jl_nothing_type, 0);
+    jl_nothing = jl_gc_permobj(ptls, 0, jl_nothing_type, 0);
     jl_set_typetagof(jl_nothing, jl_nothing_tag, GC_OLD_MARKED);
-    jl_init_box_caches();
-    jl_init_common_symbols();
     // warning: this changes `jl_current_task`, so be careful not to call that from this function
     jl_task_t *ct = jl_init_root_task(ptls, stack_lo, stack_hi);
+    jl_init_box_caches();
+    jl_init_common_symbols();
 #pragma GCC diagnostic pop
     JL_GC_PROMISE_ROOTED(ct);
     _finish_jl_init_(sysimage, ptls, ct);

src/jltypes.c

@@ -2972,6 +2972,7 @@ void export_jl_sysimg_globals(void)
 void jl_init_types(void) JL_GC_DISABLED
 {
     jl_module_t *core = NULL; // will need to be assigned later
+    jl_task_t *ct = jl_current_task;
 
     // create base objects
     jl_datatype_type = jl_new_uninitialized_datatype();
@@ -2985,7 +2986,7 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_methtable_type = jl_new_uninitialized_datatype();
     jl_method_table = jl_new_method_table(jl_symbol("methodtable"), core);
 
-    jl_emptysvec = (jl_svec_t*)jl_gc_permobj(sizeof(void*), jl_simplevector_type, 0);
+    jl_emptysvec = (jl_svec_t*)jl_gc_permobj(ct->ptls, sizeof(void*), jl_simplevector_type, 0);
     jl_set_typetagof(jl_emptysvec, jl_simplevector_tag, GC_OLD_MARKED);
     jl_svec_set_len_unsafe(jl_emptysvec, 0);
 
@@ -3115,7 +3116,7 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_typeofbottom_type = jl_new_datatype(jl_symbol("TypeofBottom"), core, type_type, jl_emptysvec,
                                            jl_emptysvec, jl_emptysvec, jl_emptysvec, 0, 0, 0);
     XX(typeofbottom);
-    jl_bottom_type = jl_gc_permobj(0, jl_typeofbottom_type, 0);
+    jl_bottom_type = jl_gc_permobj(ct->ptls, 0, jl_typeofbottom_type, 0);
     jl_set_typetagof(jl_bottom_type, jl_typeofbottom_tag, GC_OLD_MARKED);
     jl_typeofbottom_type->instance = jl_bottom_type;
 
@@ -3165,7 +3166,7 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_typeofbottom_type->super = jl_wrap_Type(jl_bottom_type);
     jl_typeofbottom_type->super->layout = jl_typeofbottom_type->layout; // the only abstract type with a layout
     jl_emptytuple_type = (jl_datatype_t*)jl_apply_tuple_type(jl_emptysvec, 0);
-    jl_emptytuple = jl_gc_permobj(0, jl_emptytuple_type, 0);
+    jl_emptytuple = jl_gc_permobj(ct->ptls, 0, jl_emptytuple_type, 0);
     jl_emptytuple_type->instance = jl_emptytuple;
 
     // non-primitive definitions follow

src/runtime_ccall.cpp

@@ -296,7 +296,8 @@ jl_value_t *jl_get_cfunction_trampoline(
                 permanent = true;
         }
         if (permanent) {
-            result = jl_gc_permobj(sizeof(jl_taggedvalue_t) + jl_datatype_size(result_type), result_type, 0);
+            jl_task_t *ct = jl_current_task;
+            result = jl_gc_permobj(ct->ptls, sizeof(jl_taggedvalue_t) + jl_datatype_size(result_type), result_type, 0);
             memset(result, 0, jl_datatype_size(result_type));
         }
         else {

src/simplevector.c

@@ -22,7 +22,8 @@ JL_DLLEXPORT jl_svec_t *(ijl_svec)(size_t n, ...)
 jl_svec_t *(jl_perm_symsvec)(size_t n, ...)
 {
     if (n == 0) return jl_emptysvec;
-    jl_svec_t *jv = (jl_svec_t*)jl_gc_permobj((n + 1) * sizeof(void*), jl_simplevector_type, 0);
+    jl_task_t *ct = jl_current_task;
+    jl_svec_t *jv = (jl_svec_t*)jl_gc_permobj(ct->ptls, (n + 1) * sizeof(void*), jl_simplevector_type, 0);
     jl_set_typetagof(jv, jl_simplevector_tag, jl_astaggedvalue(jv)->bits.gc);
     jl_svec_set_len_unsafe(jv, n);
     va_list args;

src/staticdata.c

@@ -3993,7 +3993,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image,
                     // and we overwrite the name field (field 0) now so preserve it too
                     if (dt->instance) {
                         if (dt->instance == jl_nothing)
-                            dt->instance = jl_gc_permobj(0, newdt, 0);
+                            dt->instance = jl_gc_permobj(ct->ptls, 0, newdt, 0);
                         newdt->instance = dt->instance;
                     }
                     static_assert(offsetof(jl_datatype_t, name) == 0, "");

src/symbol.c

@@ -35,10 +35,11 @@ static size_t symbol_nbytes(size_t len) JL_NOTSAFEPOINT
 static jl_sym_t *mk_symbol(const char *str, size_t len) JL_NOTSAFEPOINT
 {
     size_t nb = symbol_nbytes(len);
+    jl_task_t *ct = jl_current_task;
     // jl_sym_t is an object and needs to be allocated with jl_gc_permobj
     // but its type is set below with jl_set_typetagof since
     // jl_symbol_type might not have been initialized
-    jl_sym_t *sym = (jl_sym_t*)jl_gc_permobj(nb, NULL, sizeof(void*));
+    jl_sym_t *sym = (jl_sym_t*)jl_gc_permobj(ct->ptls, nb, NULL, sizeof(void*));
     jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED);
     jl_atomic_store_relaxed(&sym->left, NULL);
     jl_atomic_store_relaxed(&sym->right, NULL);