Final changes to support the binding

Author: udesou

src/gc-common.c

@@ -477,6 +477,20 @@ void jl_gc_track_malloced_genericmemory(jl_ptls_t ptls, jl_genericmemory_t *m, i
     ptls->gc_tls.heap.mallocarrays = ma;
 }
 
+JL_DLLEXPORT int64_t jl_gc_pool_live_bytes(void)
+{
+    int n_threads = jl_atomic_load_acquire(&jl_n_threads);
+    jl_ptls_t *all_tls_states = jl_atomic_load_relaxed(&jl_all_tls_states);
+    int64_t pool_live_bytes = 0;
+    for (int i = 0; i < n_threads; i++) {
+        jl_ptls_t ptls2 = all_tls_states[i];
+        if (ptls2 != NULL) {
+            pool_live_bytes += jl_atomic_load_relaxed(&ptls2->gc_tls.gc_num.pool_live_bytes);
+        }
+    }
+    return pool_live_bytes;
+}
+
 void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT
 {
     jl_ptls_t ptls = jl_current_task->ptls;

src/gc.c

@@ -2789,20 +2789,6 @@ static void sweep_finalizer_list(arraylist_t *list)
     list->len = j;
 }
 
-JL_DLLEXPORT int64_t jl_gc_pool_live_bytes(void)
-{
-    int n_threads = jl_atomic_load_acquire(&jl_n_threads);
-    jl_ptls_t *all_tls_states = jl_atomic_load_relaxed(&jl_all_tls_states);
-    int64_t pool_live_bytes = 0;
-    for (int i = 0; i < n_threads; i++) {
-        jl_ptls_t ptls2 = all_tls_states[i];
-        if (ptls2 != NULL) {
-            pool_live_bytes += jl_atomic_load_relaxed(&ptls2->gc_tls.gc_num.pool_live_bytes);
-        }
-    }
-    return pool_live_bytes;
-}
-
 uint64_t jl_gc_smooth(uint64_t old_val, uint64_t new_val, double factor)
 {
     double est = factor * old_val + (1 - factor) * new_val;

src/gc.h

@@ -47,7 +47,6 @@ extern jl_value_t *jl_gc_pool_alloc_inner(jl_ptls_t ptls, int pool_offset, int o
 extern void jl_rng_split(uint64_t to[JL_RNG_SIZE], uint64_t from[JL_RNG_SIZE]);
 extern void *gc_managed_realloc_(jl_ptls_t ptls, void *d, size_t sz, size_t oldsz,
                                  int isaligned, jl_value_t *owner, int8_t can_collect);
-extern size_t jl_array_nbytes(jl_array_t *a);
 extern void run_finalizers(jl_task_t *ct, int finalizers_thread);
 
 #define malloc_cache_align(sz) jl_malloc_aligned(sz, JL_CACHE_BYTE_ALIGNMENT)

src/genericmemory.c

@@ -54,6 +54,7 @@ jl_genericmemory_t *_new_genericmemory_(jl_value_t *mtype, size_t nel, int8_t is
         tot = sizeof(jl_genericmemory_t) + sizeof(void*);
     }
     m = (jl_genericmemory_t*)jl_gc_alloc(ct->ptls, tot, mtype);
+
     if (pooled) {
         data = (char*)m + JL_SMALL_BYTE_ALIGNMENT;
     }

src/julia.h

@@ -646,11 +646,12 @@ typedef struct _jl_binding_t {
     _Atomic(struct _jl_binding_t*) owner;  // for individual imported bindings (NULL until 'resolved')
     _Atomic(jl_value_t*) ty;  // binding type
     uint8_t constp:1;
-    uint8_t exportp:1;
+    uint8_t exportp:1; // `public foo` sets `publicp`, `export foo` sets both `publicp` and `exportp`
+    uint8_t publicp:1; // exportp without publicp is not allowed.
     uint8_t imported:1;
     uint8_t usingfailed:1;
     uint8_t deprecated:2; // 0=not deprecated, 1=renamed, 2=moved to another package
-    uint8_t padding:2;
+    uint8_t padding:1;
 } jl_binding_t;
 
 typedef struct {
@@ -809,7 +810,7 @@ static inline jl_value_t *jl_to_typeof(uintptr_t t)
     return (jl_value_t*)t;
 }
 #else
-extern JL_HIDDEN jl_datatype_t *ijl_small_typeof[(jl_max_tags << 4) / sizeof(jl_datatype_t*)];
+extern JL_DLLEXPORT jl_datatype_t *ijl_small_typeof[(jl_max_tags << 4) / sizeof(jl_datatype_t*)];
 static inline jl_value_t *jl_to_typeof(uintptr_t t)
 {
     if (t < (jl_max_tags << 4))

src/llvm-final-gc-lowering.cpp

@@ -103,7 +103,6 @@ void FinalLowerGC::lowerNewGCFrame(CallInst *target, Function &F)
     builder.CreateMemSet(gcframe, Constant::getNullValue(Type::getInt8Ty(F.getContext())), ptrsize * (nRoots + 2), Align(16), tbaa_gcframe);
 
     target->replaceAllUsesWith(gcframe);
-    target->eraseFromParent();
 }
 
 void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F)
@@ -131,7 +130,6 @@ void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F)
             gcframe,
             pgcstack,
             Align(sizeof(void*)));
-    target->eraseFromParent();
 }
 
 void FinalLowerGC::lowerPopGCFrame(CallInst *target, Function &F)
@@ -150,7 +148,6 @@ void FinalLowerGC::lowerPopGCFrame(CallInst *target, Function &F)
         pgcstack,
         Align(sizeof(void*)));
     inst->setMetadata(LLVMContext::MD_tbaa, tbaa_gcframe);
-    target->eraseFromParent();
 }
 
 void FinalLowerGC::lowerGetGCFrameSlot(CallInst *target, Function &F)
@@ -170,7 +167,6 @@ void FinalLowerGC::lowerGetGCFrameSlot(CallInst *target, Function &F)
     auto gep = builder.CreateInBoundsGEP(T_prjlvalue, gcframe, index);
     gep->takeName(target);
     target->replaceAllUsesWith(gep);
-    target->eraseFromParent();
 }
 
 void FinalLowerGC::lowerQueueGCRoot(CallInst *target, Function &F)
@@ -187,7 +183,6 @@ void FinalLowerGC::lowerSafepoint(CallInst *target, Function &F)
     IRBuilder<> builder(target);
     Value* signal_page = target->getOperand(0);
     builder.CreateLoad(T_size, signal_page, true);
-    target->eraseFromParent();
 }
 
 #ifdef MMTK_GC
@@ -252,7 +247,7 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
 
             // Should we generate fastpath allocation sequence here? We should always generate fastpath here for MMTk.
             // Setting this to false will increase allocation overhead a lot, and should only be used for debugging.
-            const bool INLINE_FASTPATH_ALLOCATION = true;
+            const bool INLINE_FASTPATH_ALLOCATION = false;
 
             if (INLINE_FASTPATH_ALLOCATION) {
                 // Assuming we use the first immix allocator.
@@ -307,12 +302,12 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
                 builder.CreateStore(new_cursor, cursor_ptr);
 
                 // ptls->gc_num.allocd += osize;
-                auto pool_alloc_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, gc_tls) + offsetof(jl_gc_tls_states_t, gc_num));
-                auto pool_alloc_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, pool_alloc_pos);
-                auto pool_alloc_tls = builder.CreateBitCast(pool_alloc_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "pool_alloc");
-                auto pool_allocd = builder.CreateLoad(Type::getInt64Ty(target->getContext()), pool_alloc_tls);
-                auto pool_allocd_total = builder.CreateAdd(pool_allocd, pool_osize);
-                builder.CreateStore(pool_allocd_total, pool_alloc_tls);
+                // auto pool_alloc_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, gc_tls) + offsetof(jl_gc_tls_states_t, gc_num));
+                // auto pool_alloc_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, pool_alloc_pos);
+                // auto pool_alloc_tls = builder.CreateBitCast(pool_alloc_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "pool_alloc");
+                // auto pool_allocd = builder.CreateLoad(Type::getInt64Ty(target->getContext()), pool_alloc_tls);
+                // auto pool_allocd_total = builder.CreateAdd(pool_allocd, pool_osize);
+                // builder.CreateStore(pool_allocd_total, pool_alloc_tls);
 
                 auto v_raw = builder.CreateNSWAdd(result, ConstantInt::get(Type::getInt64Ty(target->getContext()), sizeof(jl_taggedvalue_t)));
                 auto v_as_ptr = builder.CreateIntToPtr(v_raw, poolAllocFunc->getReturnType());
@@ -321,14 +316,14 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
                 phiNode->addIncoming(new_call, slowpath);
                 phiNode->addIncoming(v_as_ptr, fastpath);
                 phiNode->takeName(target);
-
+                
                 target->replaceAllUsesWith(phiNode);
-                target->eraseFromParent();
                 return;
             } else {
                 auto pool_offs = ConstantInt::get(Type::getInt32Ty(F.getContext()), 1);
                 newI = builder.CreateCall(poolAllocFunc, { ptls, pool_offs, pool_osize_i32, type });
-                derefBytes = sizeof(void*);
+                if (sz > 0)
+                    derefBytes = sz;
             }
         #endif // MMTK_GC
         }
@@ -346,7 +341,6 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
         newI->addDereferenceableRetAttr(derefBytes);
     newI->takeName(target);
     target->replaceAllUsesWith(newI);
-    target->eraseFromParent();
 }
 
 bool FinalLowerGC::runOnFunction(Function &F)
@@ -372,21 +366,23 @@ bool FinalLowerGC::runOnFunction(Function &F)
 
     // Lower all calls to supported intrinsics.
     for (auto &BB : F) {
-        for (auto &I : make_early_inc_range(BB)) {
-            auto *CI = dyn_cast<CallInst>(&I);
-            if (!CI)
+        for (auto it = BB.begin(); it != BB.end();) {
+            auto *CI = dyn_cast<CallInst>(&*it);
+            if (!CI) {
+                ++it;
                 continue;
+            }
 
             Value *callee = CI->getCalledOperand();
             assert(callee);
 
 #define LOWER_INTRINSIC(INTRINSIC, LOWER_INTRINSIC_FUNC) \
-            do { \
-                auto intrinsic = getOrNull(jl_intrinsics::INTRINSIC); \
-                if (intrinsic == callee) { \
-                    LOWER_INTRINSIC_FUNC(CI, F); \
-                } \
-            } while (0)
+            auto INTRINSIC = getOrNull(jl_intrinsics::INTRINSIC); \
+            if (INTRINSIC == callee) { \
+                LOWER_INTRINSIC_FUNC(CI, F); \
+                it = CI->eraseFromParent(); \
+                continue; \
+            } \
 
             LOWER_INTRINSIC(newGCFrame, lowerNewGCFrame);
             LOWER_INTRINSIC(pushGCFrame, lowerPushGCFrame);
@@ -396,14 +392,13 @@ bool FinalLowerGC::runOnFunction(Function &F)
             LOWER_INTRINSIC(queueGCRoot, lowerQueueGCRoot);
             LOWER_INTRINSIC(safepoint, lowerSafepoint);
 
-
 #ifdef MMTK_GC
             LOWER_INTRINSIC(writeBarrier1, lowerWriteBarrier1);
             LOWER_INTRINSIC(writeBarrier2, lowerWriteBarrier2);
             LOWER_INTRINSIC(writeBarrier1Slow, lowerWriteBarrier1Slow);
             LOWER_INTRINSIC(writeBarrier2Slow, lowerWriteBarrier2Slow);
 #endif
-
+            ++it;
 
 #undef LOWER_INTRINSIC
         }