Implement moving Immix in MMTk

This commit implements moving Immix in MMTk, which allows objects to move
in the GC.

The performance of this implementation is not yet amazing. It is very
similar to non-moving Immix in many of them and slightly slower in others.
The benchmark results is shown below.

    --------------  -----------------  ----------  ---------
    bench           Moving Immix (ms)  stddev (%)  RSS (MiB)
    activerecord    241.9              0.5         86.6
    chunky-png      447.8              0.8         74.9
    erubi-rails     1183.9             0.8         136.1
    hexapdf         1607.9             2.6         402.3
    liquid-c        45.4               6.7         44.9
    liquid-compile  44.1               9.3         53.0
    liquid-render   105.4              4.5         55.9
    lobsters        650.1              9.7         418.4
    mail            115.4              2.1         64.4
    psych-load      1656.8             0.8         43.6
    railsbench      1653.5             1.3         149.8
    rubocop         127.0              15.6        142.1
    ruby-lsp        130.7              10.5        99.4
    sequel          52.8               7.2         45.6
    shipit          1187.0             3.9         311.0
    --------------  -----------------  ----------  ---------

    --------------  ---------------------  ----------  ---------
    bench           Non-moving Immix (ms)  stddev (%)  RSS (MiB)
    activerecord    218.9                  2.7         86.1
    chunky-png      464.6                  0.8         66.7
    erubi-rails     1119.0                 4.3         132.7
    hexapdf         1539.8                 1.8         425.2
    liquid-c        40.6                   6.9         45.2
    liquid-compile  40.6                   8.1         52.9
    liquid-render   99.3                   2.3         48.3
    mail            107.4                  5.3         65.4
    psych-load      1535.6                 1.0         39.5
    railsbench      1565.6                 1.1         149.6
    rubocop         122.5                  14.3        146.7
    ruby-lsp        128.4                  10.7        106.4
    sequel          44.1                   4.0         45.7
    shipit          1154.5                 2.7         358.5
    --------------  ---------------------  ----------  ---------

Author: peterzhu2118

gc/mmtk/Cargo.toml

@@ -21,7 +21,7 @@ probe = "0.5"
 sysinfo = "0.32.0"
 
 [dependencies.mmtk]
-features = ["is_mmtk_object", "object_pinning", "sticky_immix_non_moving_nursery", "immix_non_moving"]
+features = ["is_mmtk_object", "object_pinning", "sticky_immix_non_moving_nursery"]
 
 # Uncomment the following lines to use mmtk-core from the official repository.
 git = "https://github.com/mmtk/mmtk-core.git"

gc/mmtk/mmtk.c

@@ -183,6 +183,18 @@ rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator)
     RB_GC_VM_UNLOCK(lock_lev);
 }
 
+static void
+rb_mmtk_before_updating_jit_code(void)
+{
+    rb_gc_before_updating_jit_code();
+}
+
+static void
+rb_mmtk_after_updating_jit_code(void)
+{
+    rb_gc_after_updating_jit_code();
+}
+
 static size_t
 rb_mmtk_number_of_mutators(void)
 {
@@ -247,9 +259,19 @@ rb_mmtk_scan_objspace(void)
 }
 
 static void
-rb_mmtk_scan_object_ruby_style(MMTk_ObjectReference object)
+rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference to)
 {
-    rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object);
+    rb_gc_move_obj_during_marking((VALUE)from, (VALUE)to);
+}
+
+static void
+rb_mmtk_update_object_references(MMTk_ObjectReference mmtk_object)
+{
+    VALUE object = (VALUE)mmtk_object;
+
+    if (!RB_FL_TEST(object, RUBY_FL_WEAK_REFERENCE)) {
+        rb_gc_update_object_references(rb_gc_get_objspace(), object);
+    }
 }
 
 static void
@@ -259,9 +281,15 @@ rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object)
 }
 
 static void
-rb_mmtk_handle_weak_references(MMTk_ObjectReference object)
+rb_mmtk_handle_weak_references(MMTk_ObjectReference mmtk_object, bool moving)
 {
-    rb_gc_handle_weak_references((VALUE)object);
+    VALUE object = (VALUE)mmtk_object;
+
+    rb_gc_handle_weak_references(object);
+
+    if (moving) {
+        rb_gc_update_object_references(rb_gc_get_objspace(), object);
+    }
 }
 
 static void
@@ -332,27 +360,50 @@ rb_mmtk_update_finalizer_table(void)
 }
 
 static int
-rb_mmtk_update_table_i(VALUE val, void *data)
+rb_mmtk_global_tables_count(void)
+{
+    return RB_GC_VM_WEAK_TABLE_COUNT;
+}
+
+static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin);
+
+static int
+rb_mmtk_update_global_tables_i(VALUE val, void *data)
 {
     if (!mmtk_is_reachable((MMTk_ObjectReference)val)) {
         return ST_DELETE;
     }
 
+    // TODO: check only if in moving GC
+    if (rb_mmtk_call_object_closure(val, false) != val) {
+        return ST_REPLACE;
+    }
+
     return ST_CONTINUE;
 }
 
 static int
-rb_mmtk_global_tables_count(void)
+rb_mmtk_update_global_tables_replace_i(VALUE *ptr, void *data)
 {
-    return RB_GC_VM_WEAK_TABLE_COUNT;
+    // TODO: cache the new location so we don't call rb_mmtk_call_object_closure twice
+    *ptr = rb_mmtk_call_object_closure(*ptr, false);
+
+    return ST_CONTINUE;
 }
 
 static void
 rb_mmtk_update_global_tables(int table)
 {
     RUBY_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT);
 
-    rb_gc_vm_weak_table_foreach(rb_mmtk_update_table_i, NULL, NULL, true, (enum rb_gc_vm_weak_tables)table);
+    // TODO: set weak_only to true for non-moving GC
+    rb_gc_vm_weak_table_foreach(
+        rb_mmtk_update_global_tables_i,
+        rb_mmtk_update_global_tables_replace_i,
+        NULL,
+        false,
+        (enum rb_gc_vm_weak_tables)table
+    );
 }
 
 static bool
@@ -376,11 +427,14 @@ MMTk_RubyUpcalls ruby_upcalls = {
     rb_mmtk_stop_the_world,
     rb_mmtk_resume_mutators,
     rb_mmtk_block_for_gc,
+    rb_mmtk_before_updating_jit_code,
+    rb_mmtk_after_updating_jit_code,
     rb_mmtk_number_of_mutators,
     rb_mmtk_get_mutators,
     rb_mmtk_scan_gc_roots,
     rb_mmtk_scan_objspace,
-    rb_mmtk_scan_object_ruby_style,
+    rb_mmtk_move_obj_during_marking,
+    rb_mmtk_update_object_references,
     rb_mmtk_call_gc_mark_children,
     rb_mmtk_handle_weak_references,
     rb_mmtk_call_obj_free,
@@ -738,33 +792,42 @@ rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
 void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) { }
 
 // Marking
+static inline VALUE
+rb_mmtk_call_object_closure(VALUE obj, bool pin)
+{
+    return (VALUE)rb_mmtk_gc_thread_tls->object_closure.c_function(
+        rb_mmtk_gc_thread_tls->object_closure.rust_closure,
+        rb_mmtk_gc_thread_tls->gc_context,
+        (MMTk_ObjectReference)obj,
+        pin
+    );
+}
+
 void
 rb_gc_impl_mark(void *objspace_ptr, VALUE obj)
 {
     if (RB_SPECIAL_CONST_P(obj)) return;
 
-    rb_mmtk_gc_thread_tls->object_closure.c_function(rb_mmtk_gc_thread_tls->object_closure.rust_closure,
-                                                     rb_mmtk_gc_thread_tls->gc_context,
-                                                     (MMTk_ObjectReference)obj,
-                                                     false);
+    rb_mmtk_call_object_closure(obj, false);
 }
 
 void
 rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr)
 {
     if (RB_SPECIAL_CONST_P(*ptr)) return;
 
-    // TODO: make it movable
-    rb_gc_impl_mark(objspace_ptr, *ptr);
+    VALUE new_obj = rb_mmtk_call_object_closure(*ptr, false);
+    if (new_obj != *ptr) {
+        *ptr = new_obj;
+    }
 }
 
 void
 rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj)
 {
     if (RB_SPECIAL_CONST_P(obj)) return;
 
-    // TODO: also pin
-    rb_gc_impl_mark(objspace_ptr, obj);
+    rb_mmtk_call_object_closure(obj, true);
 }
 
 void
@@ -775,11 +838,10 @@ rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
     }
 }
 
-// Weak references
-
 void
 rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj)
 {
+    RB_FL_SET(obj, RUBY_FL_WEAK_REFERENCE);
     mmtk_declare_weak_references((MMTk_ObjectReference)obj);
 }
 
@@ -790,16 +852,22 @@ rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj)
 }
 
 // Compaction
+void
+rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj)
+{
+    mmtk_register_pinning_obj((MMTk_ObjectReference)obj);
+}
+
 bool
 rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj)
 {
-    rb_bug("unimplemented");
+    return rb_mmtk_call_object_closure(obj, false) != obj;
 }
 
 VALUE
-rb_gc_impl_location(void *objspace_ptr, VALUE value)
+rb_gc_impl_location(void *objspace_ptr, VALUE obj)
 {
-    rb_bug("unimplemented");
+    return rb_mmtk_call_object_closure(obj, false);
 }
 
 // Write barriers

gc/mmtk/mmtk.h

@@ -62,13 +62,16 @@ typedef struct MMTk_RubyUpcalls {
     void (*stop_the_world)(void);
     void (*resume_mutators)(void);
     void (*block_for_gc)(MMTk_VMMutatorThread tls);
+    void (*before_updating_jit_code)(void);
+    void (*after_updating_jit_code)(void);
     size_t (*number_of_mutators)(void);
     void (*get_mutators)(void (*visit_mutator)(MMTk_Mutator*, void*), void *data);
     void (*scan_gc_roots)(void);
     void (*scan_objspace)(void);
-    void (*scan_object_ruby_style)(MMTk_ObjectReference object);
+    void (*move_obj_during_marking)(MMTk_ObjectReference from, MMTk_ObjectReference to);
+    void (*update_object_references)(MMTk_ObjectReference object);
     void (*call_gc_mark_children)(MMTk_ObjectReference object);
-    void (*handle_weak_references)(MMTk_ObjectReference object);
+    void (*handle_weak_references)(MMTk_ObjectReference object, bool moving);
     void (*call_obj_free)(MMTk_ObjectReference object);
     size_t (*vm_live_bytes)(void);
     void (*update_global_tables)(int tbl_idx);
@@ -125,6 +128,8 @@ void mmtk_declare_weak_references(MMTk_ObjectReference object);
 
 bool mmtk_weak_references_alive_p(MMTk_ObjectReference object);
 
+void mmtk_register_pinning_obj(MMTk_ObjectReference obj);
+
 void mmtk_object_reference_write_post(MMTk_Mutator *mutator, MMTk_ObjectReference object);
 
 void mmtk_register_wb_unprotected_object(MMTk_ObjectReference object);

gc/mmtk/src/abi.rs

@@ -299,16 +299,19 @@ pub struct RubyUpcalls {
     pub stop_the_world: extern "C" fn(),
     pub resume_mutators: extern "C" fn(),
     pub block_for_gc: extern "C" fn(tls: VMMutatorThread),
+    pub before_updating_jit_code: extern "C" fn(),
+    pub after_updating_jit_code: extern "C" fn(),
     pub number_of_mutators: extern "C" fn() -> usize,
     pub get_mutators: extern "C" fn(
         visit_mutator: extern "C" fn(*mut RubyMutator, *mut libc::c_void),
         data: *mut libc::c_void,
     ),
     pub scan_gc_roots: extern "C" fn(),
     pub scan_objspace: extern "C" fn(),
-    pub scan_object_ruby_style: extern "C" fn(object: ObjectReference),
+    pub move_obj_during_marking: extern "C" fn(from: ObjectReference, to: ObjectReference),
+    pub update_object_references: extern "C" fn(object: ObjectReference),
     pub call_gc_mark_children: extern "C" fn(object: ObjectReference),
-    pub handle_weak_references: extern "C" fn(object: ObjectReference),
+    pub handle_weak_references: extern "C" fn(object: ObjectReference, moving: bool),
     pub call_obj_free: extern "C" fn(object: ObjectReference),
     pub vm_live_bytes: extern "C" fn() -> usize,
     pub update_global_tables: extern "C" fn(tbl_idx: c_int),

gc/mmtk/src/api.rs

@@ -312,6 +312,13 @@ pub extern "C" fn mmtk_weak_references_alive_p(object: ObjectReference) -> bool
     object.is_reachable()
 }
 
+// =============== Compaction ===============
+
+#[no_mangle]
+pub extern "C" fn mmtk_register_pinning_obj(obj: ObjectReference) {
+    crate::binding().pinning_registry.register(obj);
+}
+
 // =============== Write barriers ===============
 
 #[no_mangle]

gc/mmtk/src/binding.rs

@@ -9,6 +9,7 @@ use mmtk::MMTK;
 
 use crate::abi;
 use crate::abi::RubyBindingOptions;
+use crate::pinning_registry::PinningRegistry;
 use crate::weak_proc::WeakProcessor;
 use crate::Ruby;
 
@@ -54,6 +55,7 @@ pub struct RubyBinding {
     pub upcalls: *const abi::RubyUpcalls,
     pub plan_name: Mutex<Option<CString>>,
     pub weak_proc: WeakProcessor,
+    pub pinning_registry: PinningRegistry,
     pub gc_thread_join_handles: Mutex<Vec<JoinHandle<()>>>,
     pub wb_unprotected_objects: Mutex<HashSet<ObjectReference>>,
 }
@@ -77,6 +79,7 @@ impl RubyBinding {
             upcalls,
             plan_name: Mutex::new(None),
             weak_proc: WeakProcessor::new(),
+            pinning_registry: PinningRegistry::new(),
             gc_thread_join_handles: Default::default(),
             wb_unprotected_objects: Default::default(),
         }

gc/mmtk/src/collection.rs

@@ -8,28 +8,45 @@ use mmtk::scheduler::*;
 use mmtk::util::heap::GCTriggerPolicy;
 use mmtk::util::{VMMutatorThread, VMThread, VMWorkerThread};
 use mmtk::vm::{Collection, GCThreadContext};
+use std::sync::atomic::AtomicBool;
 use std::sync::atomic::Ordering;
 use std::thread;
 
+static CURRENT_GC_MAY_MOVE: AtomicBool = AtomicBool::new(false);
+
 pub struct VMCollection {}
 
 impl Collection<Ruby> for VMCollection {
     fn is_collection_enabled() -> bool {
         crate::CONFIGURATION.gc_enabled.load(Ordering::Relaxed)
     }
 
-    fn stop_all_mutators<F>(_tls: VMWorkerThread, mut mutator_visitor: F)
+    fn stop_all_mutators<F>(tls: VMWorkerThread, mut mutator_visitor: F)
     where
         F: FnMut(&'static mut mmtk::Mutator<Ruby>),
     {
         (upcalls().stop_the_world)();
+
+        if crate::mmtk().get_plan().current_gc_may_move_object() {
+            CURRENT_GC_MAY_MOVE.store(true, Ordering::Relaxed);
+            (upcalls().before_updating_jit_code)();
+        } else {
+            CURRENT_GC_MAY_MOVE.store(false, Ordering::Relaxed);
+        }
+
+        crate::binding().pinning_registry.pin_children(tls);
+
         (upcalls().get_mutators)(
             Self::notify_mutator_ready::<F>,
             &mut mutator_visitor as *mut F as *mut _,
         );
     }
 
     fn resume_mutators(_tls: VMWorkerThread) {
+        if CURRENT_GC_MAY_MOVE.load(Ordering::Relaxed) {
+            (upcalls().after_updating_jit_code)();
+        }
+
         (upcalls().resume_mutators)();
     }
 

gc/mmtk/src/lib.rs

@@ -27,6 +27,7 @@ pub mod binding;
 pub mod collection;
 pub mod heap;
 pub mod object_model;
+pub mod pinning_registry;
 pub mod reference_glue;
 pub mod scanning;
 pub mod utils;