Fix object copy performance (#130)

* Fix get-object-size and object-copy performance

* Minor

* Fix CI

Author: wenyuzhao

mmtk/src/abi.rs

@@ -1,6 +1,7 @@
 use super::UPCALLS;
 use mmtk::util::constants::*;
 use mmtk::util::conversions;
+use mmtk::util::ObjectReference;
 use mmtk::util::{Address, OpaquePointer};
 use std::ffi::CStr;
 use std::fmt;
@@ -64,9 +65,31 @@ pub struct Klass {
 }
 
 impl Klass {
+    pub const LH_NEUTRAL_VALUE: i32 = 0;
+    pub const LH_INSTANCE_SLOW_PATH_BIT: i32 = 0x01;
+    #[allow(clippy::erasing_op)]
+    pub const LH_LOG2_ELEMENT_SIZE_SHIFT: i32 = BITS_IN_BYTE as i32 * 0;
+    pub const LH_LOG2_ELEMENT_SIZE_MASK: i32 = BITS_IN_LONG as i32 - 1;
+    pub const LH_HEADER_SIZE_SHIFT: i32 = BITS_IN_BYTE as i32 * 2;
+    pub const LH_HEADER_SIZE_MASK: i32 = (1 << BITS_IN_BYTE) - 1;
     pub unsafe fn cast<'a, T>(&self) -> &'a T {
         &*(self as *const _ as usize as *const T)
     }
+    /// Force slow-path for instance size calculation?
+    #[inline(always)]
+    const fn layout_helper_needs_slow_path(lh: i32) -> bool {
+        (lh & Self::LH_INSTANCE_SLOW_PATH_BIT) != 0
+    }
+    /// Get log2 array element size
+    #[inline(always)]
+    const fn layout_helper_log2_element_size(lh: i32) -> i32 {
+        (lh >> Self::LH_LOG2_ELEMENT_SIZE_SHIFT) & Self::LH_LOG2_ELEMENT_SIZE_MASK
+    }
+    /// Get array header size
+    #[inline(always)]
+    const fn layout_helper_header_size(lh: i32) -> i32 {
+        (lh >> Self::LH_HEADER_SIZE_SHIFT) & Self::LH_HEADER_SIZE_MASK
+    }
 }
 
 #[repr(C)]
@@ -238,6 +261,13 @@ pub struct OopDesc {
     pub klass: &'static Klass,
 }
 
+impl OopDesc {
+    #[inline(always)]
+    pub fn start(&self) -> Address {
+        unsafe { mem::transmute(self) }
+    }
+}
+
 impl fmt::Debug for OopDesc {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let c_string = unsafe { ((*UPCALLS).dump_object_string)(mem::transmute(self)) };
@@ -249,6 +279,22 @@ impl fmt::Debug for OopDesc {
 
 pub type Oop = &'static OopDesc;
 
+/// Convert ObjectReference to Oop
+impl From<ObjectReference> for &OopDesc {
+    #[inline(always)]
+    fn from(o: ObjectReference) -> Self {
+        unsafe { mem::transmute(o) }
+    }
+}
+
+/// Convert Oop to ObjectReference
+impl From<&OopDesc> for ObjectReference {
+    #[inline(always)]
+    fn from(o: &OopDesc) -> Self {
+        unsafe { mem::transmute(o) }
+    }
+}
+
 impl OopDesc {
     pub unsafe fn as_array_oop<T>(&self) -> ArrayOop<T> {
         &*(self as *const OopDesc as *const ArrayOopDesc<T>)
@@ -257,6 +303,40 @@ impl OopDesc {
     pub fn get_field_address(&self, offset: i32) -> Address {
         Address::from_ref(self) + offset as isize
     }
+
+    /// Slow-path for calculating object instance size
+    #[inline(always)]
+    unsafe fn size_slow(&self) -> usize {
+        ((*UPCALLS).get_object_size)(self.into())
+    }
+
+    /// Calculate object instance size
+    #[inline(always)]
+    pub unsafe fn size(&self) -> usize {
+        let klass = self.klass;
+        let lh = klass.layout_helper;
+        // The (scalar) instance size is pre-recorded in the TIB?
+        if lh > Klass::LH_NEUTRAL_VALUE {
+            if !Klass::layout_helper_needs_slow_path(lh) {
+                lh as _
+            } else {
+                self.size_slow()
+            }
+        } else if lh <= Klass::LH_NEUTRAL_VALUE {
+            if lh < Klass::LH_NEUTRAL_VALUE {
+                // Calculate array size
+                let array_length = self.as_array_oop::<()>().length();
+                let mut size_in_bytes: usize =
+                    (array_length as usize) << Klass::layout_helper_log2_element_size(lh);
+                size_in_bytes += Klass::layout_helper_header_size(lh) as usize;
+                (size_in_bytes + 0b111) & !0b111
+            } else {
+                self.size_slow()
+            }
+        } else {
+            unreachable!()
+        }
+    }
 }
 
 #[repr(C)]

mmtk/src/object_model.rs

@@ -1,6 +1,7 @@
 use std::sync::atomic::Ordering;
 
 use super::UPCALLS;
+use crate::abi::Oop;
 use crate::{vm_metadata, OpenJDK};
 use mmtk::util::alloc::fill_alignment_gap;
 use mmtk::util::copy::*;
@@ -89,13 +90,11 @@ impl ObjectModel<OpenJDK> for VMObjectModel {
         copy: CopySemantics,
         copy_context: &mut GCWorkerCopyContext<OpenJDK>,
     ) -> ObjectReference {
-        let bytes = unsafe { ((*UPCALLS).get_object_size)(from) };
+        let bytes = unsafe { Oop::from(from).size() };
         let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::<usize>(), 0, copy);
         // Copy
         let src = from.to_address();
-        for i in 0..bytes {
-            unsafe { (dst + i).store((src + i).load::<u8>()) };
-        }
+        unsafe { std::ptr::copy_nonoverlapping::<u8>(src.to_ptr(), dst.to_mut_ptr(), bytes) }
         let to_obj = unsafe { dst.to_object_reference() };
         copy_context.post_copy(to_obj, bytes, copy);
         to_obj
@@ -125,7 +124,7 @@ impl ObjectModel<OpenJDK> for VMObjectModel {
     }
 
     fn get_current_size(object: ObjectReference) -> usize {
-        unsafe { ((*UPCALLS).get_object_size)(object) }
+        unsafe { Oop::from(object).size() }
     }
 
     fn get_size_when_copied(object: ObjectReference) -> usize {

openjdk/mmtkUpcalls.cpp

@@ -207,7 +207,9 @@ static void mmtk_dump_object(void* object) {
 
 static size_t mmtk_get_object_size(void* object) {
   oop o = (oop) object;
-  return o->size() * HeapWordSize;
+  // Slow-dispatch only. The fast-path code is moved to rust.
+  auto klass = o->klass();
+  return klass->oop_size(o) << LogHeapWordSize;
 }
 
 static int mmtk_enter_vm() {