Mimalloc marksweep support (#180)

qinsoon · paigereeves · web-flow · commit 528f8c6f2e38 · 2022-12-05T19:31:42.000+13:00
Co-authored-by: paigereeves &lt;paigereeves006@gmail.com&gt;
diff --git a/.github/scripts/ci-test-assertions.sh b/.github/scripts/ci-test-assertions.sh
@@ -39,6 +39,12 @@ export MMTK_PLAN=NoGC
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
 
+# --- MarkSweep ---
+export MMTK_PLAN=MarkSweep
+
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+
 # -- PageProtect --
 sudo sysctl -w vm.max_map_count=655300
 export MMTK_PLAN=PageProtect
diff --git a/.github/scripts/ci-test-global-alloc-bit.sh b/.github/scripts/ci-test-global-alloc-bit.sh
@@ -40,6 +40,13 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
 
+# --- MarkSweep ---
+export MMTK_PLAN=MarkSweep
+
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+
 # --- NoGC ---
 
 export MMTK_PLAN=NoGC
diff --git a/.github/scripts/ci-test-only-normal.sh b/.github/scripts/ci-test-only-normal.sh
@@ -110,6 +110,23 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb
 build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse
 
+# --- MarkSweep ---
+export MMTK_PLAN=MarkSweep
+# Test - the benchmarks that are commented out do not work yet
+# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints.
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr
+#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar bloat - does not work for stock build
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
+#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar jython - does not work for stock build
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar lusearch - validation failed
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd
+#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar xalan - mmtk-core gets stuck in slowdebug build
+
+# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb
+build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:TieredStopAtLevel=1 -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse
+
 # --- PageProtect ---
 # Make sure this runs last in our tests unless we want to set it back to the default limit.
 sudo sysctl -w vm.max_map_count=655300
diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock
diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml
@@ -30,7 +30,7 @@ once_cell = "1.10.0"
 # - change branch
 # - change repo name
 # But other changes including adding/removing whitespaces in commented lines may break the CI.
-mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "69b4fe46ee4353a2fdeb4911ba914b63a7ffa29a" }
+mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "6e1b4df8e6bf41f86c24824b4c664f05b2b9fbfb" }
 # Uncomment the following to build locally
 # mmtk = { path = "../repos/mmtk-core" }
 
@@ -59,6 +59,6 @@ malloc_mark_sweep = ["mmtk/malloc_mark_sweep"]
 nogc = []
 semispace = []
 gencopy = []
-marksweep = ["mmtk/malloc_mark_sweep"]
+marksweep = []
 pageprotect = []
 immix = []
diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs
@@ -106,7 +106,7 @@ pub extern "C" fn bind_mutator(tls: VMMutatorThread) -> *mut Mutator<OpenJDK> {
 // It is fine we turn the pointer back to box, as we turned a boxed value to the raw pointer in bind_mutator()
 #[allow(clippy::not_unsafe_ptr_arg_deref)]
 pub extern "C" fn destroy_mutator(mutator: *mut Mutator<OpenJDK>) {
-    memory_manager::destroy_mutator(unsafe { Box::from_raw(mutator) })
+    memory_manager::destroy_mutator(unsafe { &mut *mutator })
 }
 
 #[no_mangle]
diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs
@@ -111,6 +111,10 @@ pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t =
 pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t =
     mmtk::util::metadata::side_metadata::ALLOC_SIDE_METADATA_ADDR.as_usize();
 
+#[no_mangle]
+pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t =
+    std::mem::size_of::<mmtk::util::alloc::FreeListAllocator<OpenJDK>>();
+
 #[derive(Default)]
 pub struct OpenJDK;
 
@@ -130,6 +134,10 @@ impl VMBinding for OpenJDK {
 
     type VMEdge = OpenJDKEdge;
     type VMMemorySlice = Range<Address>;
+
+    const MIN_ALIGNMENT: usize = 8;
+    const MAX_ALIGNMENT: usize = 8;
+    const USE_ALLOCATION_OFFSET: bool = false;
 }
 
 use std::sync::atomic::AtomicBool;
diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs
@@ -18,6 +18,9 @@ impl ObjectModel<OpenJDK> for VMObjectModel {
     const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = vm_metadata::MARKING_METADATA_SPEC;
     const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = vm_metadata::LOS_METADATA_SPEC;
 
+    const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = true;
+    const OBJECT_REF_OFFSET_LOWER_BOUND: isize = 0;
+
     #[inline]
     fn copy(
         from: ObjectReference,
diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h
@@ -22,6 +22,7 @@ extern const uintptr_t GLOBAL_SIDE_METADATA_BASE_ADDRESS;
 extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS;
 extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS;
 extern const size_t MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES;
+extern const uintptr_t FREE_LIST_ALLOCATOR_SIZE;
 
 extern const char* get_mmtk_version();
 
@@ -77,6 +78,7 @@ struct AllocatorSelector {
 #define TAG_MALLOC                    2
 #define TAG_IMMIX                     3
 #define TAG_MARK_COMPACT              4
+#define TAG_FREE_LIST                 5
 
 extern AllocatorSelector get_allocator_mapping(int allocator);
 extern size_t get_max_non_los_default_alloc_bytes();
diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp
@@ -105,6 +105,7 @@ void MMTkBarrierSet::resize_covered_region(MemRegion new_region) {
 
 void MMTkBarrierSet::on_thread_destroy(Thread* thread) {
   thread->third_party_heap_mutator.flush();
+  thread->third_party_heap_mutator.destroy();
 }
 
 void MMTkBarrierSet::on_thread_attach(JavaThread* thread) {
diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp
@@ -68,7 +68,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa
       __ jcc(Assembler::aboveEqual, slow_case);
     }
 
-    if (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT) {
+    if (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT || selector.tag == TAG_FREE_LIST) {
       __ jmp(slow_case);
       return;
     }
diff --git a/openjdk/mmtkBarrierSetC2.cpp b/openjdk/mmtkBarrierSetC2.cpp
@@ -132,7 +132,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x,
 
   if (x->C->env()->dtrace_alloc_probes() || !MMTK_ENABLE_ALLOCATION_FASTPATH
       // Malloc allocator has no fastpath
-      || (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT)) {
+      || (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT || selector.tag == TAG_FREE_LIST)) {
     // Force slow-path allocation
     always_slow = true;
     initial_slow_test = NULL;
diff --git a/openjdk/mmtkMutator.cpp b/openjdk/mmtkMutator.cpp
@@ -6,6 +6,10 @@
 size_t MMTkMutatorContext::max_non_los_default_alloc_bytes = 0;
 
 MMTkMutatorContext MMTkMutatorContext::bind(::Thread* current) {
+  if (FREE_LIST_ALLOCATOR_SIZE != sizeof(FreeListAllocator)) {
+    printf("ERROR: Unmatched free list allocator size: rs=%zu cpp=%zu\n", FREE_LIST_ALLOCATOR_SIZE, sizeof(FreeListAllocator));
+    guarantee(false, "ERROR");
+  }
   return *((MMTkMutatorContext*) ::bind_mutator((void*) current));
 }
 
@@ -34,3 +38,7 @@ HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) {
 void MMTkMutatorContext::flush() {
   ::flush_mutator((MMTk_Mutator) this);
 }
+
+void MMTkMutatorContext::destroy() {
+  ::destroy_mutator((MMTk_Mutator) this);
+}
diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp
@@ -23,6 +23,7 @@ const int MAX_BUMP_ALLOCATORS = 6;
 const int MAX_LARGE_OBJECT_ALLOCATORS = 2;
 const int MAX_MALLOC_ALLOCATORS = 1;
 const int MAX_IMMIX_ALLOCATORS = 1;
+const int MAX_FREE_LIST_ALLOCATORS = 2;
 const int MAX_MARK_COMPACT_ALLOCATORS = 1;
 
 // The following types should have the same layout as the types with the same name in MMTk core (Rust)
@@ -57,6 +58,27 @@ struct ImmixAllocator {
   uintptr_t line_opt;
 };
 
+struct FLBlock {
+  void* Address;
+};
+
+struct FLBlockList {
+  FLBlock first;
+  FLBlock last;
+  size_t size;
+  char lock;
+};
+
+struct FreeListAllocator {
+  void* tls;
+  void* space;
+  RustDynPtr plan;
+  FLBlockList* available_blocks;
+  FLBlockList* available_blocks_stress;
+  FLBlockList* unswept_blocks;
+  FLBlockList* consumed_blocks;
+};
+
 struct MallocAllocator {
   void* tls;
   void* space;
@@ -72,6 +94,7 @@ struct Allocators {
   LargeObjectAllocator large_object[MAX_LARGE_OBJECT_ALLOCATORS];
   MallocAllocator malloc[MAX_MALLOC_ALLOCATORS];
   ImmixAllocator immix[MAX_IMMIX_ALLOCATORS];
+  FreeListAllocator free_list[MAX_FREE_LIST_ALLOCATORS];
   MarkCompactAllocator markcompact[MAX_MARK_COMPACT_ALLOCATORS];
 };
 
@@ -92,6 +115,7 @@ struct MMTkMutatorContext {
   HeapWord* alloc(size_t bytes, Allocator allocator = AllocatorDefault);
 
   void flush();
+  void destroy();
 
   static MMTkMutatorContext bind(::Thread* current);
   static bool is_ready_to_bind();

Original file line number	Diff line number	Diff line change
`@@ -106,7 +106,7 @@ pub extern "C" fn bind_mutator(tls: VMMutatorThread) -> *mut Mutator<OpenJDK> {`
`106`	`106`	`// It is fine we turn the pointer back to box, as we turned a boxed value to the raw pointer in bind_mutator()`
`107`	`107`	`#[allow(clippy::not_unsafe_ptr_arg_deref)]`
`108`	`108`	`pub extern "C" fn destroy_mutator(mutator: *mut Mutator<OpenJDK>) {`
`109`		`- memory_manager::destroy_mutator(unsafe { Box::from_raw(mutator) })`
	`109`	`+ memory_manager::destroy_mutator(unsafe { &mut *mutator })`
`110`	`110`	`}`
`111`	`111`
`112`	`112`	`#[no_mangle]`
Original file line number	Diff line number	Diff line change
`@@ -105,6 +105,7 @@ void MMTkBarrierSet::resize_covered_region(MemRegion new_region) {`
`105`	`105`
`106`	`106`	`void MMTkBarrierSet::on_thread_destroy(Thread* thread) {`
`107`	`107`	`thread->third_party_heap_mutator.flush();`
	`108`	`+ thread->third_party_heap_mutator.destroy();`
`108`	`109`	`}`
`109`	`110`
`110`	`111`	`void MMTkBarrierSet::on_thread_attach(JavaThread* thread) {`