Enabling fastpath allocation

Author: udesou

src/llvm-final-gc-lowering.cpp

@@ -103,6 +103,7 @@ void FinalLowerGC::lowerNewGCFrame(CallInst *target, Function &F)
     builder.CreateMemSet(gcframe, Constant::getNullValue(Type::getInt8Ty(F.getContext())), ptrsize * (nRoots + 2), Align(16), tbaa_gcframe);
 
     target->replaceAllUsesWith(gcframe);
+    target->eraseFromParent();
 }
 
 void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F)
@@ -130,6 +131,7 @@ void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F)
             gcframe,
             pgcstack,
             Align(sizeof(void*)));
+    target->eraseFromParent();
 }
 
 void FinalLowerGC::lowerPopGCFrame(CallInst *target, Function &F)
@@ -148,6 +150,7 @@ void FinalLowerGC::lowerPopGCFrame(CallInst *target, Function &F)
         pgcstack,
         Align(sizeof(void*)));
     inst->setMetadata(LLVMContext::MD_tbaa, tbaa_gcframe);
+    target->eraseFromParent();
 }
 
 void FinalLowerGC::lowerGetGCFrameSlot(CallInst *target, Function &F)
@@ -167,6 +170,7 @@ void FinalLowerGC::lowerGetGCFrameSlot(CallInst *target, Function &F)
     auto gep = builder.CreateInBoundsGEP(T_prjlvalue, gcframe, index);
     gep->takeName(target);
     target->replaceAllUsesWith(gep);
+    target->eraseFromParent();
 }
 
 void FinalLowerGC::lowerQueueGCRoot(CallInst *target, Function &F)
@@ -183,6 +187,7 @@ void FinalLowerGC::lowerSafepoint(CallInst *target, Function &F)
     IRBuilder<> builder(target);
     Value* signal_page = target->getOperand(0);
     builder.CreateLoad(T_size, signal_page, true);
+    target->eraseFromParent();
 }
 
 #ifdef MMTK_GC
@@ -209,7 +214,6 @@ void FinalLowerGC::lowerWriteBarrier2Slow(CallInst *target, Function &F)
     assert(target->arg_size() == 2);
     target->setCalledFunction(writeBarrier2SlowFunc);
 }
-
 #endif
 
 void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
@@ -235,112 +239,26 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
                 derefBytes = sz;
         }
         else {
-        #ifndef MMTK_GC
             auto pool_offs = ConstantInt::get(Type::getInt32Ty(F.getContext()), offset);
             auto pool_osize = ConstantInt::get(Type::getInt32Ty(F.getContext()), osize);
             newI = builder.CreateCall(poolAllocFunc, { ptls, pool_offs, pool_osize, type });
             if (sz > 0)
                 derefBytes = sz;
-        #else // MMTK_GC
-            auto pool_osize_i32 = ConstantInt::get(Type::getInt32Ty(F.getContext()), osize);
-            auto pool_osize = ConstantInt::get(Type::getInt64Ty(F.getContext()), osize);
-
-            // Should we generate fastpath allocation sequence here? We should always generate fastpath here for MMTk.
-            // Setting this to false will increase allocation overhead a lot, and should only be used for debugging.
-            const bool INLINE_FASTPATH_ALLOCATION = true;
-
-            if (INLINE_FASTPATH_ALLOCATION) {
-                // Assuming we use the first immix allocator.
-                // FIXME: We should get the allocator index and type from MMTk.
-                auto allocator_offset = offsetof(jl_tls_states_t, mmtk_mutator) + offsetof(MMTkMutatorContext, allocators) + offsetof(Allocators, immix);
-
-                auto cursor_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), allocator_offset + offsetof(ImmixAllocator, cursor));
-                auto limit_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()),  allocator_offset + offsetof(ImmixAllocator, limit));
-
-                auto cursor_tls_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, cursor_pos);
-                auto cursor_ptr = builder.CreateBitCast(cursor_tls_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "cursor_ptr");
-                auto cursor = builder.CreateLoad(Type::getInt64Ty(target->getContext()), cursor_ptr, "cursor");
-
-                // offset = 8
-                auto delta_offset = builder.CreateNSWSub(ConstantInt::get(Type::getInt64Ty(target->getContext()), 0), ConstantInt::get(Type::getInt64Ty(target->getContext()), 8));
-                auto delta_cursor = builder.CreateNSWSub(ConstantInt::get(Type::getInt64Ty(target->getContext()), 0), cursor);
-                auto delta_op = builder.CreateNSWAdd(delta_offset, delta_cursor);
-                // alignment 16 (15 = 16 - 1)
-                auto delta = builder.CreateAnd(delta_op, ConstantInt::get(Type::getInt64Ty(target->getContext()), 15), "delta");
-                auto result = builder.CreateNSWAdd(cursor, delta, "result");
-
-                auto new_cursor = builder.CreateNSWAdd(result, pool_osize);
-
-                auto limit_tls_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, limit_pos);
-                auto limit_ptr = builder.CreateBitCast(limit_tls_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "limit_ptr");
-                auto limit = builder.CreateLoad(Type::getInt64Ty(target->getContext()), limit_ptr, "limit");
-
-                auto gt_limit = builder.CreateICmpSGT(new_cursor, limit);
-
-                auto current_block = target->getParent();
-                builder.SetInsertPoint(target->getNextNode());
-                auto phiNode = builder.CreatePHI(poolAllocFunc->getReturnType(), 2, "phi_fast_slow");
-                auto top_cont = current_block->splitBasicBlock(target->getNextNode(), "top_cont");
-
-                auto slowpath = BasicBlock::Create(target->getContext(), "slowpath", target->getFunction());
-                auto fastpath = BasicBlock::Create(target->getContext(), "fastpath", target->getFunction(), top_cont);
-
-                auto next_br = current_block->getTerminator();
-                next_br->eraseFromParent();
-                builder.SetInsertPoint(current_block);
-                builder.CreateCondBr(gt_limit, slowpath, fastpath);
-
-                // slowpath
-                builder.SetInsertPoint(slowpath);
-                auto pool_offs = ConstantInt::get(Type::getInt32Ty(F.getContext()), 1);
-                auto new_call = builder.CreateCall(poolAllocFunc, { ptls, pool_offs, pool_osize_i32, type });
-                new_call->setAttributes(new_call->getCalledFunction()->getAttributes());
-                builder.CreateBr(top_cont);
-
-                // // fastpath
-                builder.SetInsertPoint(fastpath);
-                builder.CreateStore(new_cursor, cursor_ptr);
-
-                // ptls->gc_num.allocd += osize;
-                auto pool_alloc_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, gc_tls) + offsetof(jl_gc_tls_states_t, gc_num));
-                auto pool_alloc_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, pool_alloc_pos);
-                auto pool_alloc_tls = builder.CreateBitCast(pool_alloc_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "pool_alloc");
-                auto pool_allocd = builder.CreateLoad(Type::getInt64Ty(target->getContext()), pool_alloc_tls);
-                auto pool_allocd_total = builder.CreateAdd(pool_allocd, pool_osize);
-                builder.CreateStore(pool_allocd_total, pool_alloc_tls);
-
-                auto v_raw = builder.CreateNSWAdd(result, ConstantInt::get(Type::getInt64Ty(target->getContext()), sizeof(jl_taggedvalue_t)));
-                auto v_as_ptr = builder.CreateIntToPtr(v_raw, poolAllocFunc->getReturnType());
-                builder.CreateBr(top_cont);
-
-                phiNode->addIncoming(new_call, slowpath);
-                phiNode->addIncoming(v_as_ptr, fastpath);
-                phiNode->takeName(target);
-
-                target->replaceAllUsesWith(phiNode);
-                return;
-            } else {
-                auto pool_offs = ConstantInt::get(Type::getInt32Ty(F.getContext()), 1);
-                newI = builder.CreateCall(poolAllocFunc, { ptls, pool_offs, pool_osize_i32, type });
-                if (sz > 0)
-                    derefBytes = sz;
-            }
-        #endif // MMTK_GC
         }
     } else {
         auto size = builder.CreateZExtOrTrunc(target->getArgOperand(1), T_size);
         // allocTypedFunc does not include the type tag in the allocation size!
         newI = builder.CreateCall(allocTypedFunc, { ptls, size, type });
         derefBytes = sizeof(void*);
     }
-
     newI->setAttributes(newI->getCalledFunction()->getAttributes());
     unsigned align = std::max((unsigned)target->getRetAlign().valueOrOne().value(), (unsigned)sizeof(void*));
     newI->addRetAttr(Attribute::getWithAlignment(F.getContext(), Align(align)));
     if (derefBytes > 0)
         newI->addDereferenceableRetAttr(derefBytes);
     newI->takeName(target);
     target->replaceAllUsesWith(newI);
+    target->eraseFromParent();
 }
 
 bool FinalLowerGC::runOnFunction(Function &F)
@@ -362,63 +280,48 @@ bool FinalLowerGC::runOnFunction(Function &F)
     poolAllocFunc = getOrDeclare(jl_well_known::GCPoolAlloc);
     bigAllocFunc = getOrDeclare(jl_well_known::GCBigAlloc);
     allocTypedFunc = getOrDeclare(jl_well_known::GCAllocTyped);
-    T_size = F.getParent()->getDataLayout().getIntPtrType(F.getContext());
-
 #ifdef MMTK_GC
     writeBarrier1Func = getOrDeclare(jl_well_known::GCWriteBarrier1);
     writeBarrier2Func = getOrDeclare(jl_well_known::GCWriteBarrier2);
     writeBarrier1SlowFunc = getOrDeclare(jl_well_known::GCWriteBarrier1Slow);
     writeBarrier2SlowFunc = getOrDeclare(jl_well_known::GCWriteBarrier2Slow);
 #endif
+    T_size = F.getParent()->getDataLayout().getIntPtrType(F.getContext());
 
     // Lower all calls to supported intrinsics.
     for (auto &BB : F) {
-        for (auto it = BB.begin(); it != BB.end();) {
-            auto *CI = dyn_cast<CallInst>(&*it);
-            if (!CI) {
-                ++it;
+        for (auto &I : make_early_inc_range(BB)) {
+            auto *CI = dyn_cast<CallInst>(&I);
+            if (!CI)
                 continue;
-            }
 
             Value *callee = CI->getCalledOperand();
             assert(callee);
 
 #define LOWER_INTRINSIC(INTRINSIC, LOWER_INTRINSIC_FUNC) \
-            auto INTRINSIC = getOrNull(jl_intrinsics::INTRINSIC); \
-            if (INTRINSIC == callee) { \
-                LOWER_INTRINSIC_FUNC(CI, F); \
-                it = CI->eraseFromParent(); \
-                continue; \
-            } \
+            do { \
+                auto intrinsic = getOrNull(jl_intrinsics::INTRINSIC); \
+                if (intrinsic == callee) { \
+                    LOWER_INTRINSIC_FUNC(CI, F); \
+                } \
+            } while (0)
 
             LOWER_INTRINSIC(newGCFrame, lowerNewGCFrame);
             LOWER_INTRINSIC(pushGCFrame, lowerPushGCFrame);
             LOWER_INTRINSIC(popGCFrame, lowerPopGCFrame);
             LOWER_INTRINSIC(getGCFrameSlot, lowerGetGCFrameSlot);
             LOWER_INTRINSIC(GCAllocBytes, lowerGCAllocBytes);
+            LOWER_INTRINSIC(queueGCRoot, lowerQueueGCRoot);
             LOWER_INTRINSIC(safepoint, lowerSafepoint);
 
-// These lowerings preserve the CI and do not erase them from the parent
-#define LOWER_WB_INTRINSIC(INTRINSIC, LOWER_INTRINSIC_FUNC) \
-            auto INTRINSIC = getOrNull(jl_intrinsics::INTRINSIC); \
-            if (INTRINSIC == callee) { \
-                LOWER_INTRINSIC_FUNC(CI, F); \
-                ++it; \
-                continue; \
-            } \
-
-            LOWER_WB_INTRINSIC(queueGCRoot, lowerQueueGCRoot);
-
 #ifdef MMTK_GC
-            LOWER_WB_INTRINSIC(writeBarrier1, lowerWriteBarrier1);
-            LOWER_WB_INTRINSIC(writeBarrier2, lowerWriteBarrier2);
-            LOWER_WB_INTRINSIC(writeBarrier1Slow, lowerWriteBarrier1Slow);
-            LOWER_WB_INTRINSIC(writeBarrier2Slow, lowerWriteBarrier2Slow);
+            LOWER_INTRINSIC(writeBarrier1, lowerNewGCFrame);
+            LOWER_INTRINSIC(writeBarrier2, lowerNewGCFrame);
+            LOWER_INTRINSIC(writeBarrier1Slow, lowerNewGCFrame);
+            LOWER_INTRINSIC(writeBarrier2Slow, lowerNewGCFrame);
 #endif
-            ++it;
 
 #undef LOWER_INTRINSIC
-#undef LOWER_WB_INTRINSIC
         }
     }