Add C2 slowpath allocation barrier (#205)

mmtk/mmtk-core#785

Add the missing C2 slow-path allocation barrier.

The C2 slow-path allocation code can do deoptimization after the allocation and before returning to C2 compiled code. The deoptimization itself contains a safepoint.

For generational plans, if a GC happens at this safepoint, the allocated object will be promoted, and all the subsequent field initialization should be recorded. The solution is to unconditionally treat this object as being written if the object is not in nursery, and record this object in the modbuf. This is also the solution for G1: https://github.com/mmtk/openjdk/blob/e1e420b3c4fd627218e45ec977972561c559acec/src/hotspot/share/gc/shared/cardTableBarrierSet.cpp#L136-L160

Author: wenyuzhao

mmtk/Cargo.lock

@@ -30,7 +30,7 @@ once_cell = "1.10.0"
 # - change branch
 # - change repo name
 # But other changes including adding/removing whitespaces in commented lines may break the CI.
-mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "08fd8ab0cfaa240a6e9a401afaec963ec0de8be5" }
+mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "c634762cf1417fe2d8f17b1b7b41e14a951044c3" }
 # Uncomment the following to build locally
 # mmtk = { path = "../repos/mmtk-core" }
 

mmtk/Cargo.toml

@@ -382,6 +382,15 @@ pub extern "C" fn mmtk_array_copy_post(
         .memory_region_copy_post(src..src + bytes, dst..dst + bytes);
 }
 
+/// C2 Slowpath allocation barrier
+#[no_mangle]
+pub extern "C" fn mmtk_object_probable_write(
+    mutator: &'static mut Mutator<OpenJDK>,
+    obj: ObjectReference,
+) {
+    mutator.barrier().object_probable_write(obj);
+}
+
 // finalization
 #[no_mangle]
 pub extern "C" fn add_finalizer(object: ObjectReference) {

mmtk/src/api.rs

@@ -2,6 +2,25 @@
 #include "mmtkObjectBarrier.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 
+void MMTkObjectBarrierSetRuntime::object_probable_write(oop new_obj) const {
+#if MMTK_ENABLE_BARRIER_FASTPATH
+  // Do fast-path check before entering mmtk rust code, to improve mutator performance.
+  // This is identical to calling `mmtk_object_probable_write` directly without a fast-path.
+  intptr_t addr = (intptr_t) (void*) new_obj;
+  uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6));
+  intptr_t shift = (addr >> 3) & 0b111;
+  uint8_t byte_val = *meta_addr;
+  if (((byte_val >> shift) & 1) == 1) {
+    // Only promoted objects will reach here.
+    // The duplicated unlog bit check inside slow-path still remains correct.
+    mmtk_object_probable_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) new_obj);
+  }
+#else
+  // The slow-call will do the unlog bit check again (same as the above fast-path check)
+  mmtk_object_probable_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) new_obj);
+#endif
+}
+
 void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot, oop target) const {
 #if MMTK_ENABLE_BARRIER_FASTPATH
   intptr_t addr = (intptr_t) (void*) src;

openjdk/barriers/mmtkObjectBarrier.cpp

@@ -25,6 +25,7 @@ class MMTkObjectBarrierSetRuntime: public MMTkBarrierSetRuntime {
   virtual void object_reference_array_copy_post(oop* src, oop* dst, size_t count) const override {
     object_reference_array_copy_post_call((void*) src, (void*) dst, count);
   }
+  virtual void object_probable_write(oop new_obj) const override;
 };
 
 class MMTkObjectBarrierSetAssembler: public MMTkBarrierSetAssembler {

openjdk/barriers/mmtkObjectBarrier.hpp

@@ -61,6 +61,9 @@ extern void mmtk_array_copy_pre(MMTk_Mutator mutator, void* src, void* dst, size
 /// Full array-copy post-barrier
 extern void mmtk_array_copy_post(MMTk_Mutator mutator, void* src, void* dst, size_t count);
 
+/// C2 slowpath allocation barrier
+extern void mmtk_object_probable_write(MMTk_Mutator mutator, void* obj);
+
 extern void release_buffer(void** buffer, size_t len, size_t cap);
 
 extern bool is_in_mmtk_spaces(void* ref);

openjdk/mmtk.h

@@ -87,6 +87,10 @@ class MMTkBarrierSetRuntime: public CHeapObj<mtGC> {
   virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const {};
   /// Full arraycopy post-barrier
   virtual void object_reference_array_copy_post(oop* src, oop* dst, size_t count) const {};
+  /// Called at the end of every C2 slowpath allocation.
+  /// Deoptimization can happen after C2 slowpath allocation, and the newly allocated object can be promoted.
+  /// So this callback is requierd for any generational collectors.
+  virtual void object_probable_write(oop new_obj) const {};
 };
 
 class MMTkBarrierC1;
@@ -134,6 +138,10 @@ class MMTkBarrierSet : public BarrierSet {
   virtual void on_thread_attach(JavaThread* thread);
   virtual void on_thread_detach(JavaThread* thread);
 
+  virtual void on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) override {
+    runtime()->object_probable_write(new_obj);
+  }
+
   // Inform the BarrierSet that the the covered heap region that starts
   // with "base" has been changed to have the given size (possibly from 0,
   // for initialization.)