Mark compact (#124)

Author: tianleq

.github/scripts/ci-test-normal.sh

@@ -102,6 +102,23 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea
 #build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar benchmarks/dacapo-2006-10-MR2.jar pmd - OOM
 #build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar benchmarks/dacapo-2006-10-MR2.jar xalan - OOM
 
+# --- MarkCompact ---
+export MMTK_PLAN=MarkCompact
+# Test - the benchmarks that are commented out do not work yet
+# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints.
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar antlr
+#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar bloat - does not work for stock build
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar fop
+#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar jython - does not work for stock build
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar luindex
+#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar lusearch - validation failed
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar pmd
+#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar xalan - mmtk-core gets stuck in slowdebug build
+
+# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar hsqldb
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar eclipse
+
 # --- MarkSweep ---
 export MMTK_PLAN=MarkSweep
 

mmtk/Cargo.toml

@@ -20,7 +20,7 @@ lazy_static = "1.1"
 # - change branch
 # - change repo name
 # But other changes including adding/removing whitespaces in commented lines may break the CI.
-mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "8e50e2c3fa8b45f7e4cd4b9e87e986fb5c17b07e" }
+mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "4bd6ca82ad037dac97920bf3a1408e4912a6bf04" }
 # Uncomment the following to build locally
 # mmtk = { path = "../repos/mmtk-core" }
 

mmtk/src/lib.rs

@@ -81,6 +81,8 @@ pub struct OpenJDK_Upcalls {
     pub scan_vm_thread_roots: extern "C" fn(process_edges: ProcessEdgesFn),
     pub number_of_mutators: extern "C" fn() -> usize,
     pub schedule_finalizer: extern "C" fn(),
+    pub prepare_for_roots_re_scanning: extern "C" fn(),
+    pub object_alignment: extern "C" fn() -> i32,
 }
 
 pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut();
@@ -118,10 +120,16 @@ lazy_static! {
         std::env::set_var("MMTK_PLAN", "GenCopy");
         #[cfg(feature = "marksweep")]
         std::env::set_var("MMTK_PLAN", "MarkSweep");
+        #[cfg(feature = "markcompact")]
+        std::env::set_var("MMTK_PLAN", "MarkCompact");
         #[cfg(feature = "pageprotect")]
         std::env::set_var("MMTK_PLAN", "PageProtect");
         #[cfg(feature = "immix")]
         std::env::set_var("MMTK_PLAN", "Immix");
         MMTK::new()
     };
 }
+
+#[no_mangle]
+pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize =
+    mmtk::util::alloc::MarkCompactAllocator::<OpenJDK>::HEADER_RESERVED_IN_BYTES;

mmtk/src/object_model.rs

@@ -2,6 +2,7 @@ use std::sync::atomic::Ordering;
 
 use super::UPCALLS;
 use crate::{vm_metadata, OpenJDK};
+use mmtk::util::alloc::fill_alignment_gap;
 use mmtk::util::metadata::header_metadata::HeaderMetadataSpec;
 use mmtk::util::{Address, ObjectReference};
 use mmtk::vm::*;
@@ -102,8 +103,23 @@ impl ObjectModel<OpenJDK> for VMObjectModel {
         to_obj
     }
 
-    fn copy_to(_from: ObjectReference, _to: ObjectReference, _region: Address) -> Address {
-        unimplemented!()
+    fn copy_to(from: ObjectReference, to: ObjectReference, region: Address) -> Address {
+        let need_copy = from != to;
+        let bytes = unsafe { ((*UPCALLS).get_object_size)(from) };
+        if need_copy {
+            // copy obj to target
+            let dst = to.to_address();
+            // Copy
+            let src = from.to_address();
+            for i in 0..bytes {
+                unsafe { (dst + i).store((src + i).load::<u8>()) };
+            }
+        }
+        let start = Self::object_start_ref(to);
+        if region != Address::ZERO {
+            fill_alignment_gap::<OpenJDK>(region, start);
+        }
+        start + bytes
     }
 
     fn get_reference_when_copied_to(_from: ObjectReference, _to: Address) -> ObjectReference {
@@ -114,6 +130,19 @@ impl ObjectModel<OpenJDK> for VMObjectModel {
         unsafe { ((*UPCALLS).get_object_size)(object) }
     }
 
+    fn get_size_when_copied(object: ObjectReference) -> usize {
+        Self::get_current_size(object)
+    }
+
+    fn get_align_when_copied(_object: ObjectReference) -> usize {
+        // FIXME figure out the proper alignment
+        ::std::mem::size_of::<usize>()
+    }
+
+    fn get_align_offset_when_copied(_object: ObjectReference) -> isize {
+        0
+    }
+
     fn get_type_descriptor(_reference: ObjectReference) -> &'static [i8] {
         unimplemented!()
     }

mmtk/src/scanning.rs

@@ -101,4 +101,10 @@ impl Scanning<OpenJDK> for VMScanning {
     fn supports_return_barrier() -> bool {
         unimplemented!()
     }
+
+    fn prepare_for_roots_re_scanning() {
+        unsafe {
+            ((*UPCALLS).prepare_for_roots_re_scanning)();
+        }
+    }
 }

openjdk/mmtk.h

@@ -15,6 +15,7 @@ typedef void* MMTk_TraceLocal;
 extern const uintptr_t GLOBAL_SIDE_METADATA_BASE_ADDRESS;
 extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS;
 extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS;
+extern const size_t MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES;
 
 /**
  * Allocation
@@ -50,10 +51,11 @@ struct AllocatorSelector {
     uint8_t index;
 };
 
-#define TAG_BUMP_POINTER    0
-#define TAG_LARGE_OBJECT    1
-#define TAG_MALLOC          2
-#define TAG_IMMIX           3
+#define TAG_BUMP_POINTER              0
+#define TAG_LARGE_OBJECT              1
+#define TAG_MALLOC                    2
+#define TAG_IMMIX                     3
+#define TAG_MARK_COMPACT              4
 
 extern AllocatorSelector get_allocator_mapping(int allocator);
 extern size_t get_max_non_los_default_alloc_bytes();
@@ -133,6 +135,7 @@ typedef struct {
     void (*scan_vm_thread_roots) (ProcessEdgesFn process_edges);
     size_t (*number_of_mutators)();
     void (*schedule_finalizer)();
+    void (*prepare_for_roots_re_scanning)();
 } OpenJDK_Upcalls;
 
 extern void openjdk_gc_init(OpenJDK_Upcalls *calls, size_t heap_size);

openjdk/mmtkBarrierSetAssembler_x86.cpp

@@ -47,30 +47,32 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa
     // default space.
     assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized");
     size_t max_non_los_bytes = MMTkMutatorContext::max_non_los_default_alloc_bytes;
+    size_t extra_header = 0;
+    // fastpath, we only use default allocator
+    Allocator allocator = AllocatorDefault;
+    // We need to figure out which allocator we are using by querying MMTk.
+    AllocatorSelector selector = get_allocator_mapping(allocator);
+    if (selector.tag == TAG_MARK_COMPACT) extra_header = MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES;
+
     if (var_size_in_bytes == noreg) {
       // constant alloc size. If it is larger than max_non_los_bytes, we directly go to slowpath.
-      if ((size_t)con_size_in_bytes > max_non_los_bytes) {
+      if ((size_t)con_size_in_bytes > max_non_los_bytes - extra_header) {
         __ jmp(slow_case);
         return;
       }
     } else {
       // var alloc size. We compare with max_non_los_bytes and conditionally jump to slowpath.
-      __ cmpptr(var_size_in_bytes, max_non_los_bytes);
+      __ cmpptr(var_size_in_bytes, max_non_los_bytes - extra_header);
       __ jcc(Assembler::aboveEqual, slow_case);
     }
 
-    // fastpath, we only use default allocator
-    Allocator allocator = AllocatorDefault;
-    // We need to figure out which allocator we are using by querying MMTk.
-    AllocatorSelector selector = get_allocator_mapping(allocator);
-
     if (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT) {
       __ jmp(slow_case);
       return;
     }
 
     // Only bump pointer allocator is implemented.
-    if (selector.tag != TAG_BUMP_POINTER && selector.tag != TAG_IMMIX) {
+    if (selector.tag != TAG_BUMP_POINTER && selector.tag != TAG_MARK_COMPACT && selector.tag != TAG_IMMIX) {
       fatal("unimplemented allocator fastpath\n");
     }
 
@@ -85,16 +87,26 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa
         + selector.index * sizeof(ImmixAllocator);
       cursor = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, cursor)));
       limit = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, limit)));
-    } else {
+    } else if(selector.tag == TAG_BUMP_POINTER) {
       allocator_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset())
         + in_bytes(byte_offset_of(MMTkMutatorContext, allocators))
         + in_bytes(byte_offset_of(Allocators, bump_pointer))
         + selector.index * sizeof(BumpAllocator);
       cursor = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor)));
       limit = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit)));
+    } else {
+      // markcompact allocator
+      allocator_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset())
+        + in_bytes(byte_offset_of(MMTkMutatorContext, allocators))
+        + in_bytes(byte_offset_of(Allocators, markcompact))
+        + selector.index * sizeof(MarkCompactAllocator)
+        + in_bytes(byte_offset_of(MarkCompactAllocator, bump_allocator));
+      cursor = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor)));
+      limit = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit)));
     }
     // obj = load lab.cursor
     __ movptr(obj, cursor);
+    if (selector.tag == TAG_MARK_COMPACT) __ addptr(obj, extra_header);
     // end = obj + size
     Register end = t1;
     if (var_size_in_bytes == noreg) {
@@ -110,8 +122,12 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa
     __ jcc(Assembler::above, slow_case);
     // lab.cursor = end
     __ movptr(cursor, end);
-
-#ifdef MMTK_ENABLE_GLOBAL_ALLOC_BIT
+  bool enable_global_alloc_bit = false;
+  #ifdef MMTK_ENABLE_GLOBAL_ALLOC_BIT
+  enable_global_alloc_bit = true;
+  #endif
+// #ifdef MMTK_ENABLE_GLOBAL_ALLOC_BIT
+  if(enable_global_alloc_bit || selector.tag == TAG_MARK_COMPACT) {
     Register tmp3 = rdi;
     Register tmp2 = rscratch1;
     assert_different_registers(obj, tmp2, tmp3, rcx);
@@ -137,15 +153,16 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa
     __ shrptr(tmp3, 6);
     __ movptr(rcx, ALLOC_BIT_BASE_ADDRESS);
     __ movb(Address(rcx, tmp3), tmp2);  
-
-#endif
+  }
+// #endif
 
     // BarrierSetAssembler::incr_allocated_bytes
     if (var_size_in_bytes->is_valid()) {
       __ addq(Address(r15_thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
     } else {
       __ addq(Address(r15_thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
     }
+    __ addq(Address(r15_thread, in_bytes(JavaThread::allocated_bytes_offset())), extra_header);
   }
 }
 

openjdk/mmtkBarrierSetC2.cpp

@@ -78,11 +78,17 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x,
   // The max non-los bytes from MMTk
   assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized");
   size_t max_non_los_bytes = MMTkMutatorContext::max_non_los_default_alloc_bytes;
+  size_t extra_header = 0;
+  // We always use the default allocator.
+  // But we need to figure out which allocator we are using by querying MMTk.
+  AllocatorSelector selector = get_allocator_mapping(AllocatorDefault);
+  if (selector.tag == TAG_MARK_COMPACT) extra_header = MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES;
+
   // Check if allocation size is constant
   long const_size = x->_igvn.find_long_con(size_in_bytes, -1);
   if (const_size >= 0) {
     // Constant alloc size. We know it is non-negative, it is safe to cast to unsigned long and compare with size_t
-    if (((unsigned long)const_size) > max_non_los_bytes) {
+    if (((unsigned long)const_size) > max_non_los_bytes - extra_header) {
       // We know at JIT time that we need to go to slowpath
       always_slow = true;
       initial_slow_test = NULL;
@@ -91,7 +97,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x,
     // Variable alloc size
 
     // Create a node for the constant and compare with size_in_bytes
-    Node *max_non_los_bytes_node = ConLNode::make((long)max_non_los_bytes);
+    Node *max_non_los_bytes_node = ConLNode::make((long)max_non_los_bytes - extra_header);
     x->transform_later(max_non_los_bytes_node);
     Node *mmtk_size_cmp = new CmpLNode(size_in_bytes, max_non_los_bytes_node);
     x->transform_later(mmtk_size_cmp);
@@ -121,10 +127,6 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x,
     }
   }
 
-  // We always use the default allocator.
-  // But we need to figure out which allocator we are using by querying MMTk.
-  AllocatorSelector selector = get_allocator_mapping(AllocatorDefault);
-
   if (x->C->env()->dtrace_alloc_probes() || !MMTK_ENABLE_ALLOCATION_FASTPATH
       // Malloc allocator has no fastpath
       || (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT)) {
@@ -173,7 +175,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x,
 
     {
       // Only bump pointer allocator fastpath is implemented.
-      if (selector.tag != TAG_BUMP_POINTER && selector.tag != TAG_IMMIX) {
+      if (selector.tag != TAG_BUMP_POINTER && selector.tag != TAG_MARK_COMPACT && selector.tag != TAG_IMMIX) {
         fatal("unimplemented allocator fastpath\n");
       }
 
@@ -187,12 +189,20 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x,
           + selector.index * sizeof(ImmixAllocator);
         tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, cursor));
         tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, limit));
-      } else {
+      } else if (selector.tag == TAG_BUMP_POINTER) {
         int allocator_base_offset = allocators_base_offset
           + in_bytes(byte_offset_of(Allocators, bump_pointer))
           + selector.index * sizeof(BumpAllocator);
         tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor));
         tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit));
+      } else {
+        // markcompact allocator
+        int allocator_base_offset = allocators_base_offset
+          + in_bytes(byte_offset_of(Allocators, bump_pointer))
+          + selector.index * sizeof(MarkCompactAllocator)
+          + in_bytes(byte_offset_of(MarkCompactAllocator, bump_allocator));
+        tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor));
+        tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit));
       }
       Node* thread = x->transform_later(new ThreadLocalNode());
       eden_top_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_top_offset);
@@ -226,8 +236,17 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x,
 
     // Load(-locked) the heap top.
     // See note above concerning the control input when using a TLAB
-    Node *old_eden_top = new LoadPNode(ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered);
-
+    Node *old_eden_top; 
+
+    if (selector.tag == TAG_MARK_COMPACT) {
+      Node *offset = ConLNode::make(extra_header);
+      x->transform_later(offset);
+      Node *node = new LoadPNode(ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered);
+      x->transform_later(node);
+      old_eden_top = new AddPNode(x->top(), node, offset);
+    } else {
+      old_eden_top = new LoadPNode(ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered);
+    }
     x->transform_later(old_eden_top);
     // Add to heap top to get a new heap top
     Node *new_eden_top = new AddPNode(x->top(), old_eden_top, size_in_bytes);
@@ -277,7 +296,12 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x,
     fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
     fast_oop_rawmem = store_eden_top;
 
-#ifdef MMTK_ENABLE_GLOBAL_ALLOC_BIT
+    bool enable_global_alloc_bit = false;
+    #ifdef MMTK_ENABLE_GLOBAL_ALLOC_BIT
+    enable_global_alloc_bit = true;
+    #endif
+// #ifdef MMTK_ENABLE_GLOBAL_ALLOC_BIT
+  if(enable_global_alloc_bit || selector.tag == TAG_MARK_COMPACT) {
     // set the alloc bit:          
     // intptr_t addr = (intptr_t) (void*) fast_oop;
     // uint8_t* meta_addr = (uint8_t*) (ALLOC_BIT_BASE_ADDRESS + (addr >> 6));
@@ -334,7 +358,8 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x,
     x->transform_later(set_alloc_bit);
 
     fast_oop_rawmem = set_alloc_bit;
-#endif
+  }
+// #endif
 
     InitializeNode* init = alloc->initialization();
     fast_oop_rawmem = x->initialize_object(alloc,

openjdk/mmtkMutator.hpp

@@ -23,6 +23,7 @@ const int MAX_BUMP_ALLOCATORS = 5;
 const int MAX_LARGE_OBJECT_ALLOCATORS = 1;
 const int MAX_MALLOC_ALLOCATORS = 1;
 const int MAX_IMMIX_ALLOCATORS = 1;
+const int MAX_MARK_COMPACT_ALLOCATORS = 1;
 
 // The following types should have the same layout as the types with the same name in MMTk core (Rust)
 
@@ -63,11 +64,16 @@ struct MallocAllocator {
   RustDynPtr plan;
 };
 
+struct MarkCompactAllocator {
+  struct BumpAllocator bump_allocator;
+};
+
 struct Allocators {
   BumpAllocator bump_pointer[MAX_BUMP_ALLOCATORS];
   LargeObjectAllocator large_object[MAX_LARGE_OBJECT_ALLOCATORS];
   MallocAllocator malloc[MAX_MALLOC_ALLOCATORS];
   ImmixAllocator immix[MAX_IMMIX_ALLOCATORS];
+  MarkCompactAllocator markcompact[MAX_MARK_COMPACT_ALLOCATORS];
 };
 
 struct MutatorConfig {