[ruby/mmtk] Implement fast path for bump pointer allocator

Adding a fast path for bump pointer allocator can improve allocation
performance.

For the following microbenchmark with MMTK_HEAP_MIN=100MiB:

    10_000_000.times { String.new }

Before:

    810.7 ms ±   8.3 ms    [User: 790.9 ms, System: 40.3 ms]

After:

    777.9 ms ±  10.4 ms    [User: 759.0 ms, System: 37.9 ms]

ruby/mmtk@0ff5c9f579

Author: peterzhu2118

gc/mmtk/cbindgen.toml

@@ -20,6 +20,11 @@ typedef void *MMTk_Address;
 typedef void *MMTk_ObjectReference;
 typedef void *MMTk_NullableObjectReference;
 typedef uint32_t MMTk_AllocationSemantics;
+
+typedef struct MMTk_BumpPointer {
+    uintptr_t cursor;
+    uintptr_t limit;
+} MMTk_BumpPointer;
 """
 
 [export]

gc/mmtk/mmtk.c

@@ -48,6 +48,8 @@ struct MMTk_ractor_cache {
 
     MMTk_Mutator *mutator;
     bool gc_mutator_p;
+
+    MMTk_BumpPointer *bump_pointer;
 };
 
 struct MMTk_final_job {
@@ -447,6 +449,7 @@ rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor)
     ccan_list_add(&objspace->ractor_caches, &cache->list_node);
 
     cache->mutator = mmtk_bind_mutator(cache);
+    cache->bump_pointer = mmtk_get_bump_pointer_allocator(cache->mutator);
 
     return cache;
 }
@@ -612,6 +615,24 @@ rb_gc_impl_config_set(void *objspace_ptr, VALUE hash)
 
 // Object allocation
 
+static VALUE
+rb_mmtk_alloc_fast_path(struct objspace *objspace, struct MMTk_ractor_cache *ractor_cache, size_t size)
+{
+    MMTk_BumpPointer *bump_pointer = ractor_cache->bump_pointer;
+    if (bump_pointer == NULL) return 0;
+
+    uintptr_t new_cursor = bump_pointer->cursor + size;
+
+    if (new_cursor > bump_pointer->limit) {
+        return 0;
+    }
+    else {
+        VALUE obj = (VALUE)bump_pointer->cursor;
+        bump_pointer->cursor = new_cursor;
+        return obj;
+    }
+}
+
 VALUE
 rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size)
 {
@@ -632,13 +653,20 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
         mmtk_handle_user_collection_request(ractor_cache, false, false);
     }
 
-    VALUE *alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size + 8, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+    alloc_size += sizeof(VALUE);
+
+    VALUE *alloc_obj = (VALUE *)rb_mmtk_alloc_fast_path(objspace, ractor_cache, alloc_size);
+    if (!alloc_obj) {
+        alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+    }
+
     alloc_obj++;
-    alloc_obj[-1] = alloc_size;
+    alloc_obj[-1] = alloc_size - sizeof(VALUE);
     alloc_obj[0] = flags;
     alloc_obj[1] = klass;
 
-    mmtk_post_alloc(ractor_cache->mutator, (void*)alloc_obj, alloc_size + 8, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+    // TODO: implement fast path for mmtk_post_alloc
+    mmtk_post_alloc(ractor_cache->mutator, (void*)alloc_obj, alloc_size, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
 
     // TODO: only add when object needs obj_free to be called
     mmtk_add_obj_free_candidate(alloc_obj);

gc/mmtk/mmtk.h

@@ -20,6 +20,11 @@ typedef void *MMTk_ObjectReference;
 typedef void *MMTk_NullableObjectReference;
 typedef uint32_t MMTk_AllocationSemantics;
 
+typedef struct MMTk_BumpPointer {
+    uintptr_t cursor;
+    uintptr_t limit;
+} MMTk_BumpPointer;
+
 
 #define MMTk_OBJREF_OFFSET 8
 
@@ -93,6 +98,8 @@ void mmtk_initialize_collection(MMTk_VMThread tls);
 
 MMTk_Mutator *mmtk_bind_mutator(MMTk_VMMutatorThread tls);
 
+MMTk_BumpPointer *mmtk_get_bump_pointer_allocator(MMTk_Mutator *m);
+
 void mmtk_destroy_mutator(MMTk_Mutator *mutator);
 
 void mmtk_handle_user_collection_request(MMTk_VMMutatorThread tls, bool force, bool exhaustive);

gc/mmtk/src/api.rs

@@ -2,6 +2,8 @@
 // They are called by C functions and they need to pass raw pointers to Rust.
 #![allow(clippy::missing_safety_doc)]
 
+use mmtk::util::alloc::BumpPointer;
+use mmtk::util::alloc::ImmixAllocator;
 use mmtk::util::options::PlanSelector;
 use std::str::FromStr;
 use std::sync::atomic::Ordering;
@@ -174,6 +176,24 @@ pub extern "C" fn mmtk_bind_mutator(tls: VMMutatorThread) -> *mut RubyMutator {
     Box::into_raw(memory_manager::bind_mutator(mmtk(), tls))
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn mmtk_get_bump_pointer_allocator(m: *mut RubyMutator) -> *mut BumpPointer {
+    match *crate::BINDING.get().unwrap().mmtk.get_options().plan {
+        PlanSelector::Immix => {
+            let mutator: &mut Mutator<Ruby> = unsafe { &mut *m };
+            let allocator =
+                unsafe { mutator.allocator_mut(mmtk::util::alloc::AllocatorSelector::Immix(0)) };
+
+            if let Some(immix_allocator) = allocator.downcast_mut::<ImmixAllocator<Ruby>>() {
+                &mut immix_allocator.bump_pointer as *mut BumpPointer
+            } else {
+                panic!("Failed to get bump pointer allocator");
+            }
+        }
+        _ => std::ptr::null_mut(),
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn mmtk_destroy_mutator(mutator: *mut RubyMutator) {
     // notify mmtk-core about destroyed mutator