Immix working; inlining fastpath allocation is not

Author: udesou

src/datatype.c

@@ -291,6 +291,10 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz,
     if ((void*)ret == HT_NOTFOUND) {
         if (!should_malloc) {
             char *perm_mem = (char *)jl_gc_perm_alloc(flddesc_sz, 0, 4, 0);
+#ifdef MMTK_GC
+    jl_ptls_t ptls = jl_current_task->ptls;
+    mmtk_immortal_post_alloc_fast(&ptls->mmtk_mutator, jl_valueof(perm_mem), flddesc_sz);
+#endif
             assert(perm_mem);
             ret = (jl_datatype_layout_t *)perm_mem;
             memcpy(perm_mem, flddesc, flddesc_sz);
@@ -968,6 +972,10 @@ JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name,
     jl_datatype_layout_t *layout = (jl_datatype_layout_t *)
       jl_gc_perm_alloc(sizeof(jl_datatype_layout_t) + sizeof(jl_fielddescdyn_t),
         0, 4, 0);
+#ifdef MMTK_GC
+    jl_ptls_t ptls = jl_current_task->ptls;
+    mmtk_immortal_post_alloc_fast(&ptls->mmtk_mutator, jl_valueof(layout), sizeof(jl_datatype_layout_t) + sizeof(jl_fielddescdyn_t));
+#endif
     layout->size = large ? GC_MAX_SZCLASS+1 : 0;
     layout->nfields = 0;
     layout->alignment = sizeof(void *);

src/gc-common.c

@@ -525,18 +525,6 @@ void jl_gc_free_memory(jl_value_t *v, int isaligned) JL_NOTSAFEPOINT
     gc_num.freecall++;
 }
 
-void jl_free_thread_gc_state(jl_ptls_t ptls)
-{
-    jl_gc_markqueue_t *mq = &ptls->gc_tls.mark_queue;
-    ws_queue_t *cq = &mq->chunk_queue;
-    free_ws_array(jl_atomic_load_relaxed(&cq->array));
-    jl_atomic_store_relaxed(&cq->array, NULL);
-    ws_queue_t *q = &mq->ptr_queue;
-    free_ws_array(jl_atomic_load_relaxed(&q->array));
-    jl_atomic_store_relaxed(&q->array, NULL);
-    arraylist_free(&mq->reclaim_set);
-}
-
 // GCNum, statistics manipulation
 // ---
 // Only safe to update the heap inside the GC
@@ -642,6 +630,16 @@ JL_DLLEXPORT int jl_gc_is_enabled(void)
     return !ptls->disable_gc;
 }
 
+int gc_logging_enabled = 0;
+
+JL_DLLEXPORT void jl_enable_gc_logging(int enable) {
+    gc_logging_enabled = enable;
+}
+
+JL_DLLEXPORT int jl_is_gc_logging_enabled(void) {
+    return gc_logging_enabled;
+}
+
 JL_DLLEXPORT void jl_gc_get_total_bytes(int64_t *bytes) JL_NOTSAFEPOINT
 {
     jl_gc_num_t num = gc_num;

src/gc-debug.c

@@ -1142,16 +1142,6 @@ NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, jl_gc_markqueue_t *mq, int off
 //     return (slot - start) / elsize;
 // }
 
-static int gc_logging_enabled = 0;
-
-JL_DLLEXPORT void jl_enable_gc_logging(int enable) {
-    gc_logging_enabled = enable;
-}
-
-JL_DLLEXPORT int jl_is_gc_logging_enabled(void) {
-    return gc_logging_enabled;
-}
-
 void _report_gc_finished(uint64_t pause, uint64_t freed, int full, int recollect, int64_t live_bytes) JL_NOTSAFEPOINT {
     if (!gc_logging_enabled) {
         return;

src/gc-page-profiler.c

@@ -180,4 +180,4 @@ JL_DLLEXPORT void jl_gc_take_page_profile(ios_t *stream)
 }
 #endif
 
-#endif // !MMTK_GC
+#endif // !MMTK_GC

src/gc.c

@@ -3345,6 +3345,18 @@ void jl_deinit_thread_heap(jl_ptls_t ptls)
     // Do nothing
 }
 
+void jl_free_thread_gc_state(jl_ptls_t ptls)
+{
+    jl_gc_markqueue_t *mq = &ptls->gc_tls.mark_queue;
+    ws_queue_t *cq = &mq->chunk_queue;
+    free_ws_array(jl_atomic_load_relaxed(&cq->array));
+    jl_atomic_store_relaxed(&cq->array, NULL);
+    ws_queue_t *q = &mq->ptr_queue;
+    free_ws_array(jl_atomic_load_relaxed(&q->array));
+    jl_atomic_store_relaxed(&q->array, NULL);
+    arraylist_free(&mq->reclaim_set);
+}
+
 // System-wide initializations
 void jl_gc_init(void)
 {

src/jitlayers.h

@@ -320,7 +320,12 @@ class MaxAlignedAllocImpl
     LLVM_ATTRIBUTE_RETURNS_NONNULL void *Allocate(size_t Size, Align Alignment) {
         Align MaxAlign = alignment(Size);
         assert(Alignment < MaxAlign); (void)Alignment;
-        return jl_gc_perm_alloc(Size, 0, MaxAlign.value(), offset);
+        void* result = jl_gc_perm_alloc(Size, 0, MaxAlign.value(), offset);
+#ifdef MMTK_GC
+        jl_ptls_t ptls = jl_current_task->ptls;
+        mmtk_immortal_post_alloc_fast(&ptls->mmtk_mutator, jl_valueof(result), Size);
+#endif
+        return result;
     }
 
     inline LLVM_ATTRIBUTE_RETURNS_NONNULL

src/llvm-final-gc-lowering.cpp

@@ -302,12 +302,12 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
                 builder.CreateStore(new_cursor, cursor_ptr);
 
                 // ptls->gc_num.allocd += osize;
-                // auto pool_alloc_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, gc_tls) + offsetof(jl_gc_tls_states_t, gc_num));
-                // auto pool_alloc_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, pool_alloc_pos);
-                // auto pool_alloc_tls = builder.CreateBitCast(pool_alloc_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "pool_alloc");
-                // auto pool_allocd = builder.CreateLoad(Type::getInt64Ty(target->getContext()), pool_alloc_tls);
-                // auto pool_allocd_total = builder.CreateAdd(pool_allocd, pool_osize);
-                // builder.CreateStore(pool_allocd_total, pool_alloc_tls);
+                auto pool_alloc_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, gc_tls) + offsetof(jl_gc_tls_states_t, gc_num));
+                auto pool_alloc_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, pool_alloc_pos);
+                auto pool_alloc_tls = builder.CreateBitCast(pool_alloc_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "pool_alloc");
+                auto pool_allocd = builder.CreateLoad(Type::getInt64Ty(target->getContext()), pool_alloc_tls);
+                auto pool_allocd_total = builder.CreateAdd(pool_allocd, pool_osize);
+                builder.CreateStore(pool_allocd_total, pool_alloc_tls);
 
                 auto v_raw = builder.CreateNSWAdd(result, ConstantInt::get(Type::getInt64Ty(target->getContext()), sizeof(jl_taggedvalue_t)));
                 auto v_as_ptr = builder.CreateIntToPtr(v_raw, poolAllocFunc->getReturnType());
@@ -316,7 +316,7 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
                 phiNode->addIncoming(new_call, slowpath);
                 phiNode->addIncoming(v_as_ptr, fastpath);
                 phiNode->takeName(target);
-                
+
                 target->replaceAllUsesWith(phiNode);
                 return;
             } else {
@@ -364,6 +364,13 @@ bool FinalLowerGC::runOnFunction(Function &F)
     allocTypedFunc = getOrDeclare(jl_well_known::GCAllocTyped);
     T_size = F.getParent()->getDataLayout().getIntPtrType(F.getContext());
 
+#ifdef MMTK_GC
+    writeBarrier1Func = getOrDeclare(jl_well_known::GCWriteBarrier1);
+    writeBarrier2Func = getOrDeclare(jl_well_known::GCWriteBarrier2);
+    writeBarrier1SlowFunc = getOrDeclare(jl_well_known::GCWriteBarrier1Slow);
+    writeBarrier2SlowFunc = getOrDeclare(jl_well_known::GCWriteBarrier2Slow);
+#endif
+
     // Lower all calls to supported intrinsics.
     for (auto &BB : F) {
         for (auto it = BB.begin(); it != BB.end();) {

src/llvm-late-gc-lowering.cpp

@@ -2617,7 +2617,7 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) {
         // FIXME: Currently we call write barrier with the src object (parent).
         // This works fine for object barrier for generational plans (such as stickyimmix), which does not use the target object at all.
         // But for other MMTk plans, we need to be careful.
-        const bool INLINE_WRITE_BARRIER = true;
+        const bool INLINE_WRITE_BARRIER = false;
         if (CI->getCalledOperand() == write_barrier_func) {
             if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) {
                 if (INLINE_WRITE_BARRIER) {

src/mmtk-gc.c

@@ -32,7 +32,23 @@ JL_DLLEXPORT void jl_gc_set_cb_notify_external_free(jl_gc_cb_notify_external_fre
 JL_DLLEXPORT void jl_gc_set_cb_notify_gc_pressure(jl_gc_cb_notify_gc_pressure_t cb, int enable)
 {
 }
+JL_DLLEXPORT void jl_gc_take_page_profile(ios_t *stream)
+{
+}
+
+JL_DLLEXPORT double jl_gc_page_utilization_stats[JL_GC_N_MAX_POOLS];
+
+STATIC_INLINE void gc_dump_page_utilization_data(void) JL_NOTSAFEPOINT
+{
+    // FIXME: MMTk would have to provide its own stats
+}
 
+#define MMTK_GC_PAGE_SZ (1 << 12) // MMTk's page size is defined in mmtk-core constants
+
+JL_DLLEXPORT uint64_t jl_get_pg_size(void)
+{
+    return MMTK_GC_PAGE_SZ;
+}
 
 inline void maybe_collect(jl_ptls_t ptls)
 {
@@ -271,6 +287,10 @@ void jl_init_thread_heap(jl_ptls_t ptls)
     mmtk_post_bind_mutator(&ptls->mmtk_mutator, mmtk_mutator);
 }
 
+void jl_free_thread_gc_state(jl_ptls_t ptls)
+{
+}
+
 void jl_deinit_thread_heap(jl_ptls_t ptls)
 {
     mmtk_destroy_mutator(&ptls->mmtk_mutator);
@@ -380,32 +400,34 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz)
 {
     jl_gcframe_t **pgcstack = jl_get_pgcstack();
     jl_task_t *ct = jl_current_task;
-    if (pgcstack && ct->world_age) {
+    void *data = malloc(sz);
+    if (data != NULL && pgcstack != NULL && ct->world_age) {
         jl_ptls_t ptls = ct->ptls;
         malloc_maybe_collect(ptls, sz);
         jl_atomic_fetch_add_relaxed(&JULIA_MALLOC_BYTES, sz);
     }
-    return malloc(sz);
+    return data;
 }
 
 JL_DLLEXPORT void *jl_gc_counted_calloc(size_t nm, size_t sz)
 {
     jl_gcframe_t **pgcstack = jl_get_pgcstack();
     jl_task_t *ct = jl_current_task;
-    if (pgcstack && ct->world_age) {
+    void *data = calloc(nm, sz);
+    if (data != NULL && pgcstack != NULL && ct->world_age) {
         jl_ptls_t ptls = ct->ptls;
         malloc_maybe_collect(ptls, nm * sz);
         jl_atomic_fetch_add_relaxed(&JULIA_MALLOC_BYTES, nm * sz);
     }
-    return calloc(nm, sz);
+    return data;
 }
 
 JL_DLLEXPORT void jl_gc_counted_free_with_size(void *p, size_t sz)
 {
     jl_gcframe_t **pgcstack = jl_get_pgcstack();
     jl_task_t *ct = jl_current_task;
     free(p);
-    if (pgcstack && ct->world_age) {
+    if (pgcstack != NULL && ct->world_age) {
         jl_atomic_fetch_add_relaxed(&JULIA_MALLOC_BYTES, -sz);
     }
 }