From 3f242b53f0552637e85a84db510e95f94ca967e2 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 8 Nov 2024 10:33:22 +1300 Subject: [PATCH 01/17] Process pinning roots - port PR #142 (#192) Port https://github.com/mmtk/mmtk-julia/pull/142 to `dev`. Process both transitively pinning roots, and non transitively pinning roots. --- .gitignore | 4 +++- mmtk/src/julia_scanning.rs | 29 +++++++++++++++++++++-------- mmtk/src/scanning.rs | 18 +++++++++++++++--- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 3b9f4dee..02fc026b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .idea/ julia/*.o julia/*.dbj.obj -.vscode \ No newline at end of file +.vscode +mmtk/src/julia_types.rs + diff --git a/mmtk/src/julia_scanning.rs b/mmtk/src/julia_scanning.rs index ca9a6def..c6c12dd9 100644 --- a/mmtk/src/julia_scanning.rs +++ b/mmtk/src/julia_scanning.rs @@ -162,7 +162,7 @@ pub unsafe fn scan_julia_object>(obj: Address, clos let ta = obj.to_ptr::(); - mmtk_scan_gcstack(ta, closure); + mmtk_scan_gcstack(ta, closure, None); let layout = (*jl_task_type).layout; debug_assert!((*layout).fielddesc_type_custom() == 0); @@ -373,9 +373,10 @@ unsafe fn mmtk_jl_genericmemory_data_owner_field_address(m: *const jl_genericmem // mmtk_jl_genericmemory_data_owner_field_address(m).load::<*const mmtk_jl_value_t>() // } -pub unsafe fn mmtk_scan_gcstack>( +pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor>( ta: *const jl_task_t, - closure: &mut EV, + mut closure: &'a mut EV, + mut pclosure: Option<&'a mut EV>, ) { let stkbuf = (*ta).ctx.stkbuf; let copy_stack = (*ta).ctx.copy_stack_custom(); @@ -404,16 +405,28 @@ pub unsafe fn mmtk_scan_gcstack>( let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); let mut nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); debug_assert!(nroots.as_usize() as u32 <= u32::MAX); - let mut nr = nroots >> 2; + let mut nr = nroots >> 3; loop { + // if the 'pin' bit on the root type is not set, must transitively pin + // and therefore use transitive pinning closure + let closure_to_use: &mut &mut EV = if (nroots.as_usize() & 4) == 0 { + &mut closure + } else { + // otherwise, use the pinning closure (if available) + match &mut pclosure { + Some(c) => c, + None => &mut closure, + } + }; + let rts = Address::from_mut_ptr(s).shift::
(2); let mut i = 0; while i < nr { if (nroots.as_usize() & 1) != 0 { let slot = read_stack(rts.shift::
(i as isize), offset, lb, ub); let real_addr = get_stack_addr(slot, offset, lb, ub); - process_slot(closure, real_addr); + process_slot(*closure_to_use, real_addr); } else { let real_addr = get_stack_addr(rts.shift::
(i as isize), offset, lb, ub); @@ -429,12 +442,12 @@ pub unsafe fn mmtk_scan_gcstack>( // pointer is not malloced but function is native, so skip it if gc_ptr_tag(slot, 1) { - process_offset_slot(closure, real_addr, 1); + process_offset_slot(*closure_to_use, real_addr, 1); i += 2; continue; } - process_slot(closure, real_addr); + process_slot(*closure_to_use, real_addr); } i += 1; @@ -450,7 +463,7 @@ pub unsafe fn mmtk_scan_gcstack>( let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); let new_nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); nroots = new_nroots; - nr = nroots >> 2; + nr = nroots >> 3; continue; } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index a82209ed..b276d5c5 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -53,14 +53,19 @@ impl Scanning for VMScanning { use mmtk::util::Address; let ptls: &mut _jl_tls_states_t = unsafe { std::mem::transmute(mutator.mutator_tls) }; - let mut slot_buffer = SlotBuffer { buffer: vec![] }; // need to be tpinned as they're all from the shadow stack + let mut tpinning_slot_buffer = SlotBuffer { buffer: vec![] }; // need to be transitively pinned + let mut pinning_slot_buffer = SlotBuffer { buffer: vec![] }; // roots from the shadow stack that we know that do not need to be transitively pinned let mut node_buffer = vec![]; // Scan thread local from ptls: See gc_queue_thread_local in gc.c let mut root_scan_task = |task: *const _jl_task_t, task_is_root: bool| { if !task.is_null() { unsafe { - crate::julia_scanning::mmtk_scan_gcstack(task, &mut slot_buffer); + crate::julia_scanning::mmtk_scan_gcstack( + task, + &mut tpinning_slot_buffer, + Some(&mut pinning_slot_buffer), + ); } if task_is_root { // captures wrong root nodes before creating the work @@ -136,13 +141,20 @@ impl Scanning for VMScanning { // Push work const CAPACITY_PER_PACKET: usize = 4096; - for tpinning_roots in slot_buffer + for tpinning_roots in tpinning_slot_buffer .buffer .chunks(CAPACITY_PER_PACKET) .map(|c| c.to_vec()) { factory.create_process_tpinning_roots_work(tpinning_roots); } + for pinning_roots in pinning_slot_buffer + .buffer + .chunks(CAPACITY_PER_PACKET) + .map(|c| c.to_vec()) + { + factory.create_process_pinning_roots_work(pinning_roots); + } for nodes in node_buffer.chunks(CAPACITY_PER_PACKET).map(|c| c.to_vec()) { factory.create_process_pinning_roots_work(nodes); } From 62fff574e95fb719a12cb4f6539f6861cf8a6939 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 29 Nov 2024 15:31:10 +1300 Subject: [PATCH 02/17] Deal with GC preserve stacks (#191) This PR ports https://github.com/mmtk/mmtk-julia/pull/156 to `dev`. --- .gitignore | 1 - mmtk/src/julia_scanning.rs | 57 ++++++++++++++++++++++++++++++++++++++ mmtk/src/scanning.rs | 4 +++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 02fc026b..6707474f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,3 @@ julia/*.o julia/*.dbj.obj .vscode mmtk/src/julia_types.rs - diff --git a/mmtk/src/julia_scanning.rs b/mmtk/src/julia_scanning.rs index c6c12dd9..eea2c507 100644 --- a/mmtk/src/julia_scanning.rs +++ b/mmtk/src/julia_scanning.rs @@ -373,11 +373,68 @@ unsafe fn mmtk_jl_genericmemory_data_owner_field_address(m: *const jl_genericmem // mmtk_jl_genericmemory_data_owner_field_address(m).load::<*const mmtk_jl_value_t>() // } +pub unsafe fn mmtk_scan_gcpreserve_stack<'a, EV: SlotVisitor>( + ta: *const jl_task_t, + closure: &'a mut EV, +) { + // process transitively pinning stack + let mut s = (*ta).gcpreserve_stack; + let (offset, lb, ub) = (0 as isize, 0 as u64, u64::MAX); + + if s != std::ptr::null_mut() { + let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); + let mut nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); + debug_assert!(nroots.as_usize() <= u32::MAX as usize); + let mut nr = nroots >> 3; + + loop { + let rts = Address::from_mut_ptr(s).shift::
(2); + let mut i = 0; + + while i < nr { + let real_addr = get_stack_addr(rts.shift::
(i as isize), offset, lb, ub); + + let slot = read_stack(rts.shift::
(i as isize), offset, lb, ub); + use crate::julia_finalizer::gc_ptr_tag; + // malloced pointer tagged in jl_gc_add_quiescent + // skip both the next element (native function), and the object + if slot & 3usize == 3 { + i += 2; + continue; + } + + // pointer is not malloced but function is native, so skip it + if gc_ptr_tag(slot, 1) { + i += 2; + continue; + } + + process_slot(closure, real_addr); + i += 1; + } + + let s_prev_address = ::std::ptr::addr_of!((*s).prev); + let sprev = read_stack(Address::from_ptr(s_prev_address), offset, lb, ub); + if sprev.is_zero() { + break; + } + + s = sprev.to_mut_ptr::(); + let s_nroots_addr = ::std::ptr::addr_of!((*s).nroots); + let new_nroots = read_stack(Address::from_ptr(s_nroots_addr), offset, lb, ub); + nroots = new_nroots; + nr = nroots >> 3; + continue; + } + } +} + pub unsafe fn mmtk_scan_gcstack<'a, EV: SlotVisitor>( ta: *const jl_task_t, mut closure: &'a mut EV, mut pclosure: Option<&'a mut EV>, ) { + // process Julia's standard shadow (GC) stack let stkbuf = (*ta).ctx.stkbuf; let copy_stack = (*ta).ctx.copy_stack_custom(); diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index b276d5c5..c430d5f1 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -61,6 +61,10 @@ impl Scanning for VMScanning { let mut root_scan_task = |task: *const _jl_task_t, task_is_root: bool| { if !task.is_null() { unsafe { + crate::julia_scanning::mmtk_scan_gcpreserve_stack( + task, + &mut tpinning_slot_buffer, + ); crate::julia_scanning::mmtk_scan_gcstack( task, &mut tpinning_slot_buffer, From 457c5728488f75508ccf1dfd63b74c8ebee8b02a Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 5 Feb 2025 23:19:07 +0000 Subject: [PATCH 03/17] Update FFI --- mmtk/src/julia_types.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mmtk/src/julia_types.rs b/mmtk/src/julia_types.rs index 530c2d96..5ee703a3 100644 --- a/mmtk/src/julia_types.rs +++ b/mmtk/src/julia_types.rs @@ -643,6 +643,7 @@ pub struct _jl_task_t { pub threadpoolid: i8, pub reentrant_timing: u8, pub gcstack: *mut jl_gcframe_t, + pub gcpreserve_stack: *mut jl_gcframe_t, pub world_age: usize, pub ptls: jl_ptls_t, pub excstack: *mut jl_excstack_t, @@ -651,7 +652,7 @@ pub struct _jl_task_t { } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of _jl_task_t"][::std::mem::size_of::<_jl_task_t>() - 224usize]; + ["Size of _jl_task_t"][::std::mem::size_of::<_jl_task_t>() - 232usize]; ["Alignment of _jl_task_t"][::std::mem::align_of::<_jl_task_t>() - 8usize]; ["Offset of field: _jl_task_t::next"][::std::mem::offset_of!(_jl_task_t, next) - 0usize]; ["Offset of field: _jl_task_t::queue"][::std::mem::offset_of!(_jl_task_t, queue) - 8usize]; @@ -688,13 +689,15 @@ const _: () = { [::std::mem::offset_of!(_jl_task_t, reentrant_timing) - 147usize]; ["Offset of field: _jl_task_t::gcstack"] [::std::mem::offset_of!(_jl_task_t, gcstack) - 152usize]; + ["Offset of field: _jl_task_t::gcpreserve_stack"] + [::std::mem::offset_of!(_jl_task_t, gcpreserve_stack) - 160usize]; ["Offset of field: _jl_task_t::world_age"] - [::std::mem::offset_of!(_jl_task_t, world_age) - 160usize]; - ["Offset of field: _jl_task_t::ptls"][::std::mem::offset_of!(_jl_task_t, ptls) - 168usize]; + [::std::mem::offset_of!(_jl_task_t, world_age) - 168usize]; + ["Offset of field: _jl_task_t::ptls"][::std::mem::offset_of!(_jl_task_t, ptls) - 176usize]; ["Offset of field: _jl_task_t::excstack"] - [::std::mem::offset_of!(_jl_task_t, excstack) - 176usize]; - ["Offset of field: _jl_task_t::eh"][::std::mem::offset_of!(_jl_task_t, eh) - 184usize]; - ["Offset of field: _jl_task_t::ctx"][::std::mem::offset_of!(_jl_task_t, ctx) - 192usize]; + [::std::mem::offset_of!(_jl_task_t, excstack) - 184usize]; + ["Offset of field: _jl_task_t::eh"][::std::mem::offset_of!(_jl_task_t, eh) - 192usize]; + ["Offset of field: _jl_task_t::ctx"][::std::mem::offset_of!(_jl_task_t, ctx) - 200usize]; }; pub type jl_task_t = _jl_task_t; #[repr(C)] From 80a7a6551e5067c2334369249c3cb927a8be372b Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 5 Feb 2025 23:56:53 +0000 Subject: [PATCH 04/17] Use saturating_sub in gc trigger to avoid overflow --- mmtk/src/gc_trigger.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mmtk/src/gc_trigger.rs b/mmtk/src/gc_trigger.rs index 056892cf..a03fefc6 100644 --- a/mmtk/src/gc_trigger.rs +++ b/mmtk/src/gc_trigger.rs @@ -128,9 +128,8 @@ impl GCTriggerPolicy for JuliaGCTrigger { let heap_size = conversions::pages_to_bytes(reserved_pages_now); let user_max = self.max_total_memory.load(Ordering::Relaxed) * 80 / 100; - let alloc_diff = self.before_free_heap_size.load(Ordering::Relaxed) - - self.old_heap_size.load(Ordering::Relaxed); - let freed_diff = self.before_free_heap_size.load(Ordering::Relaxed) - heap_size; + let alloc_diff = self.before_free_heap_size.load(Ordering::Relaxed).saturating_sub(self.old_heap_size.load(Ordering::Relaxed)); + let freed_diff = self.before_free_heap_size.load(Ordering::Relaxed).saturating_sub(heap_size); self.old_heap_size.store(heap_size, Ordering::Relaxed); // update the heap target only if the user did not force a GC From 84d048af48d9d587c454a78b87a0a8f002d79868 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 4 Dec 2024 18:16:36 +1300 Subject: [PATCH 05/17] Support VO bit (#197) This PR ports https://github.com/mmtk/mmtk-julia/pull/158 to `dev`, and updates Julia to https://github.com/mmtk/julia/pull/76. --- mmtk/Cargo.toml | 3 +- mmtk/src/api.rs | 21 ++++++ mmtk/src/lib.rs | 1 + mmtk/src/vo_bit.rs | 161 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 mmtk/src/vo_bit.rs diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index a1c42705..2091f0e9 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ memoffset = "*" # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" } [features] -default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning"] +default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning", "is_mmtk_object"] # Plans nogc = [] @@ -48,6 +48,7 @@ immix = [] stickyimmix = ["mmtk/sticky_immix_non_moving_nursery", "mmtk/immix_smaller_block"] marksweep = [] object_pinning = ["mmtk/object_pinning"] +is_mmtk_object = ["mmtk/is_mmtk_object"] # This feature disables moving non_moving = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index f58bb500..cfedb174 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -350,6 +350,8 @@ pub extern "C" fn mmtk_set_vm_space(start: Address, size: usize) { #[cfg(feature = "stickyimmix")] set_side_log_bit_for_region(start, size); + #[cfg(feature = "is_mmtk_object")] + set_side_vo_bit_for_region(start, size); } #[no_mangle] @@ -381,6 +383,8 @@ pub extern "C" fn mmtk_memory_region_copy( pub extern "C" fn mmtk_immortal_region_post_alloc(start: Address, size: usize) { #[cfg(feature = "stickyimmix")] set_side_log_bit_for_region(start, size); + #[cfg(feature = "is_mmtk_object")] + set_side_vo_bit_for_region(start, size); } #[cfg(feature = "stickyimmix")] @@ -393,6 +397,18 @@ fn set_side_log_bit_for_region(start: Address, size: usize) { } } +#[cfg(feature = "is_mmtk_object")] +fn set_side_vo_bit_for_region(start: Address, size: usize) { + debug!( + "Bulk set VO bit {} to {} ({} bytes)", + start, + start + size, + size + ); + + crate::vo_bit::bulk_update_vo_bit(start, size) +} + #[no_mangle] pub extern "C" fn mmtk_object_reference_write_post( mutator: *mut Mutator, @@ -427,6 +443,11 @@ pub extern "C" fn mmtk_object_reference_write_slow( pub static MMTK_SIDE_LOG_BIT_BASE_ADDRESS: Address = mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; +/// VO bit base address +#[no_mangle] +pub static MMTK_SIDE_VO_BIT_BASE_ADDRESS: Address = + mmtk::util::metadata::side_metadata::VO_BIT_SIDE_METADATA_ADDR; + #[no_mangle] pub extern "C" fn mmtk_object_is_managed_by_mmtk(addr: usize) -> bool { crate::api::mmtk_is_mapped_address(unsafe { Address::from_usize(addr) }) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 2d1ad00b..3d0d35cf 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -27,6 +27,7 @@ pub mod reference_glue; pub mod scanning; pub mod slots; pub mod util; +mod vo_bit; pub mod julia_finalizer; pub mod julia_scanning; diff --git a/mmtk/src/vo_bit.rs b/mmtk/src/vo_bit.rs new file mode 100644 index 00000000..4f5d3ba2 --- /dev/null +++ b/mmtk/src/vo_bit.rs @@ -0,0 +1,161 @@ +use crate::api::MMTK_SIDE_VO_BIT_BASE_ADDRESS; +use core::sync::atomic::Ordering; +use mmtk::util::Address; +use std::sync::atomic::AtomicU8; + +// This module is a duplicate of MMTK core's side metadata to allow bulk setting for VO bit. +// The problem is that VO bit is internal to MMTk core, and we cannot access VO bit. +// FIXME: We should consider refactoring MMTk core to either expose `SideMetadataSpec` for VO bit, +// or allow the binding to construct `SideMetadataSpec` for VO bit. For either case, we can +// remove this module and remove this code duplication. + +// Functions to set the side metadata for the VO bit (copied from mmtk-core) +pub const VO_BIT_LOG_NUM_OF_BITS: i32 = 0; +pub const VO_BIT_LOG_BYTES_PER_REGION: usize = mmtk::util::constants::LOG_MIN_OBJECT_SIZE as usize; + +pub fn bulk_update_vo_bit(start: Address, size: usize) { + // Update bits for a contiguous side metadata spec. We can simply calculate the data end address, and + // calculate the metadata address for the data end. + let update_contiguous = |data_start: Address, data_bytes: usize| { + if data_bytes == 0 { + return; + } + let meta_start = address_to_meta_address(data_start); + let meta_start_shift = meta_byte_lshift(data_start); + let meta_end = address_to_meta_address(data_start + data_bytes); + let meta_end_shift = meta_byte_lshift(data_start + data_bytes); + set_meta_bits(meta_start, meta_start_shift, meta_end, meta_end_shift); + }; + + // VO bit is global + update_contiguous(start, size); +} + +/// Performs the translation of data address (`data_addr`) to metadata address for the specified metadata (`metadata_spec`). +pub fn address_to_meta_address(data_addr: Address) -> Address { + #[cfg(target_pointer_width = "32")] + let res = { + if metadata_spec.is_global { + address_to_contiguous_meta_address(metadata_spec, data_addr) + } else { + address_to_chunked_meta_address(metadata_spec, data_addr) + } + }; + #[cfg(target_pointer_width = "64")] + let res = { address_to_contiguous_meta_address(data_addr) }; + + res +} + +/// Performs address translation in contiguous metadata spaces (e.g. global and policy-specific in 64-bits, and global in 32-bits) +pub fn address_to_contiguous_meta_address(data_addr: Address) -> Address { + let rshift = (mmtk::util::constants::LOG_BITS_IN_BYTE as i32) - VO_BIT_LOG_NUM_OF_BITS; + + if rshift >= 0 { + MMTK_SIDE_VO_BIT_BASE_ADDRESS + ((data_addr >> VO_BIT_LOG_BYTES_PER_REGION) >> rshift) + } else { + MMTK_SIDE_VO_BIT_BASE_ADDRESS + ((data_addr >> VO_BIT_LOG_BYTES_PER_REGION) << (-rshift)) + } +} + +pub fn meta_byte_lshift(data_addr: Address) -> u8 { + if VO_BIT_LOG_NUM_OF_BITS >= 3 { + return 0; + } + let rem_shift = mmtk::util::constants::BITS_IN_WORD as i32 + - ((mmtk::util::constants::LOG_BITS_IN_BYTE as i32) - VO_BIT_LOG_NUM_OF_BITS); + ((((data_addr >> VO_BIT_LOG_BYTES_PER_REGION) << rem_shift) >> rem_shift) + << VO_BIT_LOG_NUM_OF_BITS) as u8 +} + +/// This method is used for bulk updating side metadata for a data address range. As we cannot guarantee +/// that the data address range can be mapped to whole metadata bytes, we have to deal with cases that +/// we need to mask and zero certain bits in a metadata byte. The end address and the end bit are exclusive. +/// The end bit for update_bits could be 8, so overflowing needs to be taken care of. +pub fn update_meta_bits( + meta_start_addr: Address, + meta_start_bit: u8, + meta_end_addr: Address, + meta_end_bit: u8, + update_bytes: &impl Fn(Address, Address), + update_bits: &impl Fn(Address, u8, u8), +) { + // Start/end is the same, we don't need to do anything. + if meta_start_addr == meta_end_addr && meta_start_bit == meta_end_bit { + return; + } + + // zeroing bytes + if meta_start_bit == 0 && meta_end_bit == 0 { + update_bytes(meta_start_addr, meta_end_addr); + return; + } + + if meta_start_addr == meta_end_addr { + // Update bits in the same byte between start and end bit + update_bits(meta_start_addr, meta_start_bit, meta_end_bit); + } else if meta_start_addr + 1usize == meta_end_addr && meta_end_bit == 0 { + // Update bits in the same byte after the start bit (between start bit and 8) + update_bits(meta_start_addr, meta_start_bit, 8); + } else { + // update bits in the first byte + update_meta_bits( + meta_start_addr, + meta_start_bit, + meta_start_addr + 1usize, + 0, + update_bytes, + update_bits, + ); + // update bytes in the middle + update_meta_bits( + meta_start_addr + 1usize, + 0, + meta_end_addr, + 0, + update_bytes, + update_bits, + ); + // update bits in the last byte + update_meta_bits( + meta_end_addr, + 0, + meta_end_addr, + meta_end_bit, + update_bytes, + update_bits, + ); + } +} + +/// This method is used for bulk setting side metadata for a data address range. +pub fn set_meta_bits( + meta_start_addr: Address, + meta_start_bit: u8, + meta_end_addr: Address, + meta_end_bit: u8, +) { + let set_bytes = |start: Address, end: Address| { + set(start, 0xff, end - start); + }; + let set_bits = |addr: Address, start_bit: u8, end_bit: u8| { + // we are setting selected bits in one byte + let mask: u8 = !(u8::MAX.checked_shl(end_bit.into()).unwrap_or(0)) & (u8::MAX << start_bit); // Get a mask that the bits we need to set are 1, and the other bits are 0. + unsafe { addr.as_ref::() }.fetch_or(mask, Ordering::SeqCst); + }; + update_meta_bits( + meta_start_addr, + meta_start_bit, + meta_end_addr, + meta_end_bit, + &set_bytes, + &set_bits, + ); +} + +/// Set a range of memory to the given value. Similar to memset. +pub fn set(start: Address, val: u8, len: usize) { + unsafe { + std::ptr::write_bytes::(start.to_mut_ptr(), val, len); + } +} From 64e402ed06f76aabc10397d1e7a9690f346c19f4 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 6 Feb 2025 02:46:04 +0000 Subject: [PATCH 06/17] Add some missing variables/functions in mmtk.h --- mmtk/api/mmtk.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mmtk/api/mmtk.h b/mmtk/api/mmtk.h index 8e52239a..4c3085f9 100644 --- a/mmtk/api/mmtk.h +++ b/mmtk/api/mmtk.h @@ -85,7 +85,7 @@ extern void mmtk_block_thread_for_gc(void); extern void* mmtk_new_mutator_iterator(void); extern void* mmtk_get_next_mutator_tls(void*); extern void* mmtk_close_mutator_iterator(void*); - +extern void mmtk_store_obj_size_c(void* obj, size_t size); /** * VM Accounting @@ -106,6 +106,11 @@ extern void mmtk_add_phantom_candidate(void* ref); extern void mmtk_harness_begin(void *tls); extern void mmtk_harness_end(void); +/** + * Copying + */ +extern const void* MMTK_SIDE_VO_BIT_BASE_ADDRESS; + #ifdef __cplusplus } #endif From 20e6a004774f65632ff32b0c9124684fbddfe5aa Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 6 Feb 2025 03:54:11 +0000 Subject: [PATCH 07/17] Update julia_types for gcpreserve_stack in jl_task_t --- mmtk/src/julia_types.rs | 186 ++++++++++++++++++++++++++++++++++------ 1 file changed, 160 insertions(+), 26 deletions(-) diff --git a/mmtk/src/julia_types.rs b/mmtk/src/julia_types.rs index 5ee703a3..b6183e28 100644 --- a/mmtk/src/julia_types.rs +++ b/mmtk/src/julia_types.rs @@ -174,6 +174,9 @@ pub struct std_atomic<_Tp> { pub _M_i: _Tp, } pub type std_atomic_value_type<_Tp> = _Tp; +pub type __uint16_t = ::std::os::raw::c_ushort; +pub type __uint32_t = ::std::os::raw::c_uint; +pub type __uint64_t = ::std::os::raw::c_ulong; pub type __sig_atomic_t = ::std::os::raw::c_int; pub type jl_gcframe_t = _jl_gcframe_t; #[repr(C)] @@ -187,6 +190,7 @@ const _: () = { ["Alignment of __sigset_t"][::std::mem::align_of::<__sigset_t>() - 8usize]; ["Offset of field: __sigset_t::__val"][::std::mem::offset_of!(__sigset_t, __val) - 0usize]; }; +pub type sigset_t = __sigset_t; pub type pthread_t = ::std::os::raw::c_ulong; #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -254,6 +258,133 @@ pub struct _jl_value_t { _unused: [u8; 0], } pub type sig_atomic_t = __sig_atomic_t; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct stack_t { + pub ss_sp: *mut ::std::os::raw::c_void, + pub ss_flags: ::std::os::raw::c_int, + pub ss_size: usize, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of stack_t"][::std::mem::size_of::() - 24usize]; + ["Alignment of stack_t"][::std::mem::align_of::() - 8usize]; + ["Offset of field: stack_t::ss_sp"][::std::mem::offset_of!(stack_t, ss_sp) - 0usize]; + ["Offset of field: stack_t::ss_flags"][::std::mem::offset_of!(stack_t, ss_flags) - 8usize]; + ["Offset of field: stack_t::ss_size"][::std::mem::offset_of!(stack_t, ss_size) - 16usize]; +}; +pub type greg_t = ::std::os::raw::c_longlong; +pub type gregset_t = [greg_t; 23usize]; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _libc_fpxreg { + pub significand: [::std::os::raw::c_ushort; 4usize], + pub exponent: ::std::os::raw::c_ushort, + pub __glibc_reserved1: [::std::os::raw::c_ushort; 3usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _libc_fpxreg"][::std::mem::size_of::<_libc_fpxreg>() - 16usize]; + ["Alignment of _libc_fpxreg"][::std::mem::align_of::<_libc_fpxreg>() - 2usize]; + ["Offset of field: _libc_fpxreg::significand"] + [::std::mem::offset_of!(_libc_fpxreg, significand) - 0usize]; + ["Offset of field: _libc_fpxreg::exponent"] + [::std::mem::offset_of!(_libc_fpxreg, exponent) - 8usize]; + ["Offset of field: _libc_fpxreg::__glibc_reserved1"] + [::std::mem::offset_of!(_libc_fpxreg, __glibc_reserved1) - 10usize]; +}; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _libc_xmmreg { + pub element: [__uint32_t; 4usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _libc_xmmreg"][::std::mem::size_of::<_libc_xmmreg>() - 16usize]; + ["Alignment of _libc_xmmreg"][::std::mem::align_of::<_libc_xmmreg>() - 4usize]; + ["Offset of field: _libc_xmmreg::element"] + [::std::mem::offset_of!(_libc_xmmreg, element) - 0usize]; +}; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct _libc_fpstate { + pub cwd: __uint16_t, + pub swd: __uint16_t, + pub ftw: __uint16_t, + pub fop: __uint16_t, + pub rip: __uint64_t, + pub rdp: __uint64_t, + pub mxcsr: __uint32_t, + pub mxcr_mask: __uint32_t, + pub _st: [_libc_fpxreg; 8usize], + pub _xmm: [_libc_xmmreg; 16usize], + pub __glibc_reserved1: [__uint32_t; 24usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _libc_fpstate"][::std::mem::size_of::<_libc_fpstate>() - 512usize]; + ["Alignment of _libc_fpstate"][::std::mem::align_of::<_libc_fpstate>() - 8usize]; + ["Offset of field: _libc_fpstate::cwd"][::std::mem::offset_of!(_libc_fpstate, cwd) - 0usize]; + ["Offset of field: _libc_fpstate::swd"][::std::mem::offset_of!(_libc_fpstate, swd) - 2usize]; + ["Offset of field: _libc_fpstate::ftw"][::std::mem::offset_of!(_libc_fpstate, ftw) - 4usize]; + ["Offset of field: _libc_fpstate::fop"][::std::mem::offset_of!(_libc_fpstate, fop) - 6usize]; + ["Offset of field: _libc_fpstate::rip"][::std::mem::offset_of!(_libc_fpstate, rip) - 8usize]; + ["Offset of field: _libc_fpstate::rdp"][::std::mem::offset_of!(_libc_fpstate, rdp) - 16usize]; + ["Offset of field: _libc_fpstate::mxcsr"] + [::std::mem::offset_of!(_libc_fpstate, mxcsr) - 24usize]; + ["Offset of field: _libc_fpstate::mxcr_mask"] + [::std::mem::offset_of!(_libc_fpstate, mxcr_mask) - 28usize]; + ["Offset of field: _libc_fpstate::_st"][::std::mem::offset_of!(_libc_fpstate, _st) - 32usize]; + ["Offset of field: _libc_fpstate::_xmm"] + [::std::mem::offset_of!(_libc_fpstate, _xmm) - 160usize]; + ["Offset of field: _libc_fpstate::__glibc_reserved1"] + [::std::mem::offset_of!(_libc_fpstate, __glibc_reserved1) - 416usize]; +}; +pub type fpregset_t = *mut _libc_fpstate; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct mcontext_t { + pub gregs: gregset_t, + pub fpregs: fpregset_t, + pub __reserved1: [::std::os::raw::c_ulonglong; 8usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of mcontext_t"][::std::mem::size_of::() - 256usize]; + ["Alignment of mcontext_t"][::std::mem::align_of::() - 8usize]; + ["Offset of field: mcontext_t::gregs"][::std::mem::offset_of!(mcontext_t, gregs) - 0usize]; + ["Offset of field: mcontext_t::fpregs"][::std::mem::offset_of!(mcontext_t, fpregs) - 184usize]; + ["Offset of field: mcontext_t::__reserved1"] + [::std::mem::offset_of!(mcontext_t, __reserved1) - 192usize]; +}; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ucontext_t { + pub uc_flags: ::std::os::raw::c_ulong, + pub uc_link: *mut ucontext_t, + pub uc_stack: stack_t, + pub uc_mcontext: mcontext_t, + pub uc_sigmask: sigset_t, + pub __fpregs_mem: _libc_fpstate, + pub __ssp: [::std::os::raw::c_ulonglong; 4usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of ucontext_t"][::std::mem::size_of::() - 968usize]; + ["Alignment of ucontext_t"][::std::mem::align_of::() - 8usize]; + ["Offset of field: ucontext_t::uc_flags"] + [::std::mem::offset_of!(ucontext_t, uc_flags) - 0usize]; + ["Offset of field: ucontext_t::uc_link"][::std::mem::offset_of!(ucontext_t, uc_link) - 8usize]; + ["Offset of field: ucontext_t::uc_stack"] + [::std::mem::offset_of!(ucontext_t, uc_stack) - 16usize]; + ["Offset of field: ucontext_t::uc_mcontext"] + [::std::mem::offset_of!(ucontext_t, uc_mcontext) - 40usize]; + ["Offset of field: ucontext_t::uc_sigmask"] + [::std::mem::offset_of!(ucontext_t, uc_sigmask) - 296usize]; + ["Offset of field: ucontext_t::__fpregs_mem"] + [::std::mem::offset_of!(ucontext_t, __fpregs_mem) - 424usize]; + ["Offset of field: ucontext_t::__ssp"][::std::mem::offset_of!(ucontext_t, __ssp) - 936usize]; +}; pub type jl_value_t = _jl_value_t; #[repr(C)] #[repr(align(8))] @@ -269,15 +400,18 @@ const _: () = { pub struct jl_gc_tls_states_t { pub mmtk_mutator: MMTkMutatorContext, pub malloc_sz_since_last_poll: std_atomic, + pub ctx_at_the_time_gc_started: ucontext_t, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of jl_gc_tls_states_t"][::std::mem::size_of::() - 704usize]; + ["Size of jl_gc_tls_states_t"][::std::mem::size_of::() - 1672usize]; ["Alignment of jl_gc_tls_states_t"][::std::mem::align_of::() - 8usize]; ["Offset of field: jl_gc_tls_states_t::mmtk_mutator"] [::std::mem::offset_of!(jl_gc_tls_states_t, mmtk_mutator) - 0usize]; ["Offset of field: jl_gc_tls_states_t::malloc_sz_since_last_poll"] [::std::mem::offset_of!(jl_gc_tls_states_t, malloc_sz_since_last_poll) - 696usize]; + ["Offset of field: jl_gc_tls_states_t::ctx_at_the_time_gc_started"] + [::std::mem::offset_of!(jl_gc_tls_states_t, ctx_at_the_time_gc_started) - 704usize]; }; #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -541,7 +675,7 @@ pub struct _jl_tls_states_t { } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of _jl_tls_states_t"][::std::mem::size_of::<_jl_tls_states_t>() - 2560usize]; + ["Size of _jl_tls_states_t"][::std::mem::size_of::<_jl_tls_states_t>() - 3528usize]; ["Alignment of _jl_tls_states_t"][::std::mem::align_of::<_jl_tls_states_t>() - 8usize]; ["Offset of field: _jl_tls_states_t::tid"] [::std::mem::offset_of!(_jl_tls_states_t, tid) - 0usize]; @@ -566,53 +700,53 @@ const _: () = { ["Offset of field: _jl_tls_states_t::gc_tls"] [::std::mem::offset_of!(_jl_tls_states_t, gc_tls) - 40usize]; ["Offset of field: _jl_tls_states_t::gc_tls_common"] - [::std::mem::offset_of!(_jl_tls_states_t, gc_tls_common) - 744usize]; + [::std::mem::offset_of!(_jl_tls_states_t, gc_tls_common) - 1712usize]; ["Offset of field: _jl_tls_states_t::lazily_freed_mtarraylist_buffers"] - [::std::mem::offset_of!(_jl_tls_states_t, lazily_freed_mtarraylist_buffers) - 2024usize]; + [::std::mem::offset_of!(_jl_tls_states_t, lazily_freed_mtarraylist_buffers) - 2992usize]; ["Offset of field: _jl_tls_states_t::defer_signal"] - [::std::mem::offset_of!(_jl_tls_states_t, defer_signal) - 2088usize]; + [::std::mem::offset_of!(_jl_tls_states_t, defer_signal) - 3056usize]; ["Offset of field: _jl_tls_states_t::current_task"] - [::std::mem::offset_of!(_jl_tls_states_t, current_task) - 2096usize]; + [::std::mem::offset_of!(_jl_tls_states_t, current_task) - 3064usize]; ["Offset of field: _jl_tls_states_t::next_task"] - [::std::mem::offset_of!(_jl_tls_states_t, next_task) - 2104usize]; + [::std::mem::offset_of!(_jl_tls_states_t, next_task) - 3072usize]; ["Offset of field: _jl_tls_states_t::previous_task"] - [::std::mem::offset_of!(_jl_tls_states_t, previous_task) - 2112usize]; + [::std::mem::offset_of!(_jl_tls_states_t, previous_task) - 3080usize]; ["Offset of field: _jl_tls_states_t::root_task"] - [::std::mem::offset_of!(_jl_tls_states_t, root_task) - 2120usize]; + [::std::mem::offset_of!(_jl_tls_states_t, root_task) - 3088usize]; ["Offset of field: _jl_tls_states_t::timing_stack"] - [::std::mem::offset_of!(_jl_tls_states_t, timing_stack) - 2128usize]; + [::std::mem::offset_of!(_jl_tls_states_t, timing_stack) - 3096usize]; ["Offset of field: _jl_tls_states_t::stackbase"] - [::std::mem::offset_of!(_jl_tls_states_t, stackbase) - 2136usize]; + [::std::mem::offset_of!(_jl_tls_states_t, stackbase) - 3104usize]; ["Offset of field: _jl_tls_states_t::stacksize"] - [::std::mem::offset_of!(_jl_tls_states_t, stacksize) - 2144usize]; + [::std::mem::offset_of!(_jl_tls_states_t, stacksize) - 3112usize]; ["Offset of field: _jl_tls_states_t::sig_exception"] - [::std::mem::offset_of!(_jl_tls_states_t, sig_exception) - 2152usize]; + [::std::mem::offset_of!(_jl_tls_states_t, sig_exception) - 3120usize]; ["Offset of field: _jl_tls_states_t::bt_data"] - [::std::mem::offset_of!(_jl_tls_states_t, bt_data) - 2160usize]; + [::std::mem::offset_of!(_jl_tls_states_t, bt_data) - 3128usize]; ["Offset of field: _jl_tls_states_t::bt_size"] - [::std::mem::offset_of!(_jl_tls_states_t, bt_size) - 2168usize]; + [::std::mem::offset_of!(_jl_tls_states_t, bt_size) - 3136usize]; ["Offset of field: _jl_tls_states_t::profiling_bt_buffer"] - [::std::mem::offset_of!(_jl_tls_states_t, profiling_bt_buffer) - 2176usize]; + [::std::mem::offset_of!(_jl_tls_states_t, profiling_bt_buffer) - 3144usize]; ["Offset of field: _jl_tls_states_t::signal_request"] - [::std::mem::offset_of!(_jl_tls_states_t, signal_request) - 2184usize]; + [::std::mem::offset_of!(_jl_tls_states_t, signal_request) - 3152usize]; ["Offset of field: _jl_tls_states_t::io_wait"] - [::std::mem::offset_of!(_jl_tls_states_t, io_wait) - 2188usize]; + [::std::mem::offset_of!(_jl_tls_states_t, io_wait) - 3156usize]; ["Offset of field: _jl_tls_states_t::signal_stack"] - [::std::mem::offset_of!(_jl_tls_states_t, signal_stack) - 2192usize]; + [::std::mem::offset_of!(_jl_tls_states_t, signal_stack) - 3160usize]; ["Offset of field: _jl_tls_states_t::signal_stack_size"] - [::std::mem::offset_of!(_jl_tls_states_t, signal_stack_size) - 2200usize]; + [::std::mem::offset_of!(_jl_tls_states_t, signal_stack_size) - 3168usize]; ["Offset of field: _jl_tls_states_t::system_id"] - [::std::mem::offset_of!(_jl_tls_states_t, system_id) - 2208usize]; + [::std::mem::offset_of!(_jl_tls_states_t, system_id) - 3176usize]; ["Offset of field: _jl_tls_states_t::suspend_count"] - [::std::mem::offset_of!(_jl_tls_states_t, suspend_count) - 2216usize]; + [::std::mem::offset_of!(_jl_tls_states_t, suspend_count) - 3184usize]; ["Offset of field: _jl_tls_states_t::finalizers"] - [::std::mem::offset_of!(_jl_tls_states_t, finalizers) - 2224usize]; + [::std::mem::offset_of!(_jl_tls_states_t, finalizers) - 3192usize]; ["Offset of field: _jl_tls_states_t::previous_exception"] - [::std::mem::offset_of!(_jl_tls_states_t, previous_exception) - 2480usize]; + [::std::mem::offset_of!(_jl_tls_states_t, previous_exception) - 3448usize]; ["Offset of field: _jl_tls_states_t::locks"] - [::std::mem::offset_of!(_jl_tls_states_t, locks) - 2488usize]; + [::std::mem::offset_of!(_jl_tls_states_t, locks) - 3456usize]; ["Offset of field: _jl_tls_states_t::engine_nqueued"] - [::std::mem::offset_of!(_jl_tls_states_t, engine_nqueued) - 2552usize]; + [::std::mem::offset_of!(_jl_tls_states_t, engine_nqueued) - 3520usize]; }; pub type jl_function_t = jl_value_t; pub type jl_timing_block_t = _jl_timing_block_t; From d3d799797ddf063a34868afd97b50f64e7fa507b Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 10 Dec 2024 01:28:05 +1300 Subject: [PATCH 08/17] Use MMTk's VO bit spec (#200) This PR updates to mmtk-core: https://github.com/mmtk/mmtk-core/pull/1248. This allows us to remove the duplicate code for vo bit. --------- Co-authored-by: mmtkgc-bot --- mmtk/Cargo.lock | 8 +-- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 2 +- mmtk/src/lib.rs | 1 - mmtk/src/vo_bit.rs | 161 --------------------------------------------- 5 files changed, 6 insertions(+), 168 deletions(-) delete mode 100644 mmtk/src/vo_bit.rs diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index a4d83a52..65fc59ea 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -995,18 +995,18 @@ checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" [[package]] name = "serde" -version = "1.0.217" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 2091f0e9..ab8be320 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ memoffset = "*" # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" } [features] -default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning", "is_mmtk_object"] +default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning", "is_mmtk_object", "mmtk/vo_bit_access"] # Plans nogc = [] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index cfedb174..2269117c 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -406,7 +406,7 @@ fn set_side_vo_bit_for_region(start: Address, size: usize) { size ); - crate::vo_bit::bulk_update_vo_bit(start, size) + mmtk::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_SPEC.bset_metadata(start, size); } #[no_mangle] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 3d0d35cf..2d1ad00b 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -27,7 +27,6 @@ pub mod reference_glue; pub mod scanning; pub mod slots; pub mod util; -mod vo_bit; pub mod julia_finalizer; pub mod julia_scanning; diff --git a/mmtk/src/vo_bit.rs b/mmtk/src/vo_bit.rs deleted file mode 100644 index 4f5d3ba2..00000000 --- a/mmtk/src/vo_bit.rs +++ /dev/null @@ -1,161 +0,0 @@ -use crate::api::MMTK_SIDE_VO_BIT_BASE_ADDRESS; -use core::sync::atomic::Ordering; -use mmtk::util::Address; -use std::sync::atomic::AtomicU8; - -// This module is a duplicate of MMTK core's side metadata to allow bulk setting for VO bit. -// The problem is that VO bit is internal to MMTk core, and we cannot access VO bit. -// FIXME: We should consider refactoring MMTk core to either expose `SideMetadataSpec` for VO bit, -// or allow the binding to construct `SideMetadataSpec` for VO bit. For either case, we can -// remove this module and remove this code duplication. - -// Functions to set the side metadata for the VO bit (copied from mmtk-core) -pub const VO_BIT_LOG_NUM_OF_BITS: i32 = 0; -pub const VO_BIT_LOG_BYTES_PER_REGION: usize = mmtk::util::constants::LOG_MIN_OBJECT_SIZE as usize; - -pub fn bulk_update_vo_bit(start: Address, size: usize) { - // Update bits for a contiguous side metadata spec. We can simply calculate the data end address, and - // calculate the metadata address for the data end. - let update_contiguous = |data_start: Address, data_bytes: usize| { - if data_bytes == 0 { - return; - } - let meta_start = address_to_meta_address(data_start); - let meta_start_shift = meta_byte_lshift(data_start); - let meta_end = address_to_meta_address(data_start + data_bytes); - let meta_end_shift = meta_byte_lshift(data_start + data_bytes); - set_meta_bits(meta_start, meta_start_shift, meta_end, meta_end_shift); - }; - - // VO bit is global - update_contiguous(start, size); -} - -/// Performs the translation of data address (`data_addr`) to metadata address for the specified metadata (`metadata_spec`). -pub fn address_to_meta_address(data_addr: Address) -> Address { - #[cfg(target_pointer_width = "32")] - let res = { - if metadata_spec.is_global { - address_to_contiguous_meta_address(metadata_spec, data_addr) - } else { - address_to_chunked_meta_address(metadata_spec, data_addr) - } - }; - #[cfg(target_pointer_width = "64")] - let res = { address_to_contiguous_meta_address(data_addr) }; - - res -} - -/// Performs address translation in contiguous metadata spaces (e.g. global and policy-specific in 64-bits, and global in 32-bits) -pub fn address_to_contiguous_meta_address(data_addr: Address) -> Address { - let rshift = (mmtk::util::constants::LOG_BITS_IN_BYTE as i32) - VO_BIT_LOG_NUM_OF_BITS; - - if rshift >= 0 { - MMTK_SIDE_VO_BIT_BASE_ADDRESS + ((data_addr >> VO_BIT_LOG_BYTES_PER_REGION) >> rshift) - } else { - MMTK_SIDE_VO_BIT_BASE_ADDRESS + ((data_addr >> VO_BIT_LOG_BYTES_PER_REGION) << (-rshift)) - } -} - -pub fn meta_byte_lshift(data_addr: Address) -> u8 { - if VO_BIT_LOG_NUM_OF_BITS >= 3 { - return 0; - } - let rem_shift = mmtk::util::constants::BITS_IN_WORD as i32 - - ((mmtk::util::constants::LOG_BITS_IN_BYTE as i32) - VO_BIT_LOG_NUM_OF_BITS); - ((((data_addr >> VO_BIT_LOG_BYTES_PER_REGION) << rem_shift) >> rem_shift) - << VO_BIT_LOG_NUM_OF_BITS) as u8 -} - -/// This method is used for bulk updating side metadata for a data address range. As we cannot guarantee -/// that the data address range can be mapped to whole metadata bytes, we have to deal with cases that -/// we need to mask and zero certain bits in a metadata byte. The end address and the end bit are exclusive. -/// The end bit for update_bits could be 8, so overflowing needs to be taken care of. -pub fn update_meta_bits( - meta_start_addr: Address, - meta_start_bit: u8, - meta_end_addr: Address, - meta_end_bit: u8, - update_bytes: &impl Fn(Address, Address), - update_bits: &impl Fn(Address, u8, u8), -) { - // Start/end is the same, we don't need to do anything. - if meta_start_addr == meta_end_addr && meta_start_bit == meta_end_bit { - return; - } - - // zeroing bytes - if meta_start_bit == 0 && meta_end_bit == 0 { - update_bytes(meta_start_addr, meta_end_addr); - return; - } - - if meta_start_addr == meta_end_addr { - // Update bits in the same byte between start and end bit - update_bits(meta_start_addr, meta_start_bit, meta_end_bit); - } else if meta_start_addr + 1usize == meta_end_addr && meta_end_bit == 0 { - // Update bits in the same byte after the start bit (between start bit and 8) - update_bits(meta_start_addr, meta_start_bit, 8); - } else { - // update bits in the first byte - update_meta_bits( - meta_start_addr, - meta_start_bit, - meta_start_addr + 1usize, - 0, - update_bytes, - update_bits, - ); - // update bytes in the middle - update_meta_bits( - meta_start_addr + 1usize, - 0, - meta_end_addr, - 0, - update_bytes, - update_bits, - ); - // update bits in the last byte - update_meta_bits( - meta_end_addr, - 0, - meta_end_addr, - meta_end_bit, - update_bytes, - update_bits, - ); - } -} - -/// This method is used for bulk setting side metadata for a data address range. -pub fn set_meta_bits( - meta_start_addr: Address, - meta_start_bit: u8, - meta_end_addr: Address, - meta_end_bit: u8, -) { - let set_bytes = |start: Address, end: Address| { - set(start, 0xff, end - start); - }; - let set_bits = |addr: Address, start_bit: u8, end_bit: u8| { - // we are setting selected bits in one byte - let mask: u8 = !(u8::MAX.checked_shl(end_bit.into()).unwrap_or(0)) & (u8::MAX << start_bit); // Get a mask that the bits we need to set are 1, and the other bits are 0. - unsafe { addr.as_ref::() }.fetch_or(mask, Ordering::SeqCst); - }; - update_meta_bits( - meta_start_addr, - meta_start_bit, - meta_end_addr, - meta_end_bit, - &set_bytes, - &set_bits, - ); -} - -/// Set a range of memory to the given value. Similar to memset. -pub fn set(start: Address, val: u8, len: usize) { - unsafe { - std::ptr::write_bytes::(start.to_mut_ptr(), val, len); - } -} From e40e4ca9e00a1c82a5518456627621a3c2508bae Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Wed, 18 Dec 2024 23:45:56 +1100 Subject: [PATCH 09/17] Conservative stack scanning (#203) This PR ports https://github.com/mmtk/mmtk-julia/pull/157 and https://github.com/mmtk/mmtk-julia/pull/184 to dev. It also adds an optimization that if a task is not started, the conservative stack scanning can be skipped for the task. Merge with https://github.com/mmtk/julia/pull/80. --------- Co-authored-by: Yi Lin --- mmtk/Cargo.toml | 11 ++-- mmtk/build.rs | 2 + mmtk/src/api.rs | 34 ++++-------- mmtk/src/collection.rs | 27 +++++++--- mmtk/src/conservative.rs | 112 +++++++++++++++++++++++++++++++++++++++ mmtk/src/julia_types.rs | 93 ++++++++++++++++++++++++++++++++ mmtk/src/lib.rs | 26 +++++++++ mmtk/src/object_model.rs | 55 ++++++++++++------- mmtk/src/scanning.rs | 21 ++++++++ 9 files changed, 327 insertions(+), 54 deletions(-) create mode 100644 mmtk/src/conservative.rs diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index ab8be320..6a04bbf8 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,16 +40,17 @@ memoffset = "*" # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" } [features] -default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning", "is_mmtk_object", "mmtk/vo_bit_access"] +# We must build with default features +default = ["mmtk/vm_space", "julia_copy_stack", "mmtk/object_pinning", "mmtk/is_mmtk_object", "mmtk/vo_bit_access"] -# Plans +# Default features +julia_copy_stack = [] + +# Plans: choose one nogc = [] immix = [] stickyimmix = ["mmtk/sticky_immix_non_moving_nursery", "mmtk/immix_smaller_block"] marksweep = [] -object_pinning = ["mmtk/object_pinning"] -is_mmtk_object = ["mmtk/is_mmtk_object"] # This feature disables moving non_moving = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"] -julia_copy_stack = [] diff --git a/mmtk/build.rs b/mmtk/build.rs index 3871e374..0d386983 100644 --- a/mmtk/build.rs +++ b/mmtk/build.rs @@ -54,6 +54,7 @@ fn main() { let bindings = bindgen::Builder::default() .header(format!("{}/src/julia.h", julia_dir)) .header(format!("{}/src/julia_internal.h", julia_dir)) + .header(format!("{}/src/gc-common.h", julia_dir)) // Including the paths to depending .h files .clang_arg("-I") .clang_arg(format!("{}/mmtk/api", mmtk_dir)) @@ -77,6 +78,7 @@ fn main() { .allowlist_item("jl_bt_element_t") .allowlist_item("jl_taggedvalue_t") .allowlist_item("MMTkMutatorContext") + .allowlist_item("_bigval_t") // --opaque-type MMTkMutatorContext .opaque_type("MMTkMutatorContext") // compile using c++ diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 2269117c..819a1b71 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -350,7 +350,6 @@ pub extern "C" fn mmtk_set_vm_space(start: Address, size: usize) { #[cfg(feature = "stickyimmix")] set_side_log_bit_for_region(start, size); - #[cfg(feature = "is_mmtk_object")] set_side_vo_bit_for_region(start, size); } @@ -383,7 +382,7 @@ pub extern "C" fn mmtk_memory_region_copy( pub extern "C" fn mmtk_immortal_region_post_alloc(start: Address, size: usize) { #[cfg(feature = "stickyimmix")] set_side_log_bit_for_region(start, size); - #[cfg(feature = "is_mmtk_object")] + set_side_vo_bit_for_region(start, size); } @@ -397,7 +396,8 @@ fn set_side_log_bit_for_region(start: Address, size: usize) { } } -#[cfg(feature = "is_mmtk_object")] +// We have to set VO bit even if this is a non_moving build. Otherwise, assertions in mmtk-core +// will complain about seeing objects without VO bit. fn set_side_vo_bit_for_region(start: Address, size: usize) { debug!( "Bulk set VO bit {} to {} ({} bytes)", @@ -485,9 +485,10 @@ pub extern "C" fn mmtk_get_obj_size(obj: ObjectReference) -> usize { } } -#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))] #[no_mangle] pub extern "C" fn mmtk_pin_object(object: ObjectReference) -> bool { + crate::early_return_for_non_moving_build!(false); + // We may in the future replace this with a check for the immix space (bound check), which should be much cheaper. if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { memory_manager::pin_object(object) @@ -497,9 +498,10 @@ pub extern "C" fn mmtk_pin_object(object: ObjectReference) -> bool { } } -#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))] #[no_mangle] pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool { + crate::early_return_for_non_moving_build!(false); + if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { memory_manager::unpin_object(object) } else { @@ -508,9 +510,10 @@ pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool { } } -#[cfg(all(feature = "object_pinning", not(feature = "non_moving")))] #[no_mangle] pub extern "C" fn mmtk_is_pinned(object: ObjectReference) -> bool { + crate::early_return_for_non_moving_build!(false); + if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { memory_manager::is_pinned(object) } else { @@ -519,25 +522,6 @@ pub extern "C" fn mmtk_is_pinned(object: ObjectReference) -> bool { } } -// If the `non-moving` feature is selected, pinning/unpinning is a noop and simply returns false -#[cfg(all(feature = "object_pinning", feature = "non_moving"))] -#[no_mangle] -pub extern "C" fn mmtk_pin_object(_object: ObjectReference) -> bool { - false -} - -#[cfg(all(feature = "object_pinning", feature = "non_moving"))] -#[no_mangle] -pub extern "C" fn mmtk_unpin_object(_object: ObjectReference) -> bool { - false -} - -#[cfg(all(feature = "object_pinning", feature = "non_moving"))] -#[no_mangle] -pub extern "C" fn mmtk_is_pinned(_object: ObjectReference) -> bool { - false -} - #[no_mangle] pub extern "C" fn get_mmtk_version() -> *const c_char { crate::build_info::MMTK_JULIA_FULL_VERSION_STRING diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 2ba88641..486c7886 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -4,7 +4,7 @@ use crate::{ jl_hrtime, jl_throw_out_of_memory_error, }; use crate::{JuliaVM, USER_TRIGGERED_GC}; -use log::{info, trace}; +use log::{debug, trace}; use mmtk::util::alloc::AllocationError; use mmtk::util::heap::GCTriggerPolicy; use mmtk::util::opaque_pointer::*; @@ -15,6 +15,7 @@ use std::sync::atomic::{AtomicBool, AtomicIsize, AtomicU64, Ordering}; use crate::{BLOCK_FOR_GC, STW_COND, WORLD_HAS_STOPPED}; pub static GC_START: AtomicU64 = AtomicU64::new(0); +static CURRENT_GC_MAY_MOVE: AtomicBool = AtomicBool::new(true); use std::collections::HashSet; use std::sync::RwLock; @@ -52,11 +53,18 @@ impl Collection for VMCollection { trace!("Stopped the world!"); + // Store if the current GC may move objects -- we will use it when the current GC finishes. + // We cache the value here just in case MMTk may clear it before we use the value. + CURRENT_GC_MAY_MOVE.store( + crate::SINGLETON.get_plan().current_gc_may_move_object(), + Ordering::SeqCst, + ); + // Tell MMTk the stacks are ready. { use mmtk::vm::ActivePlan; for mutator in crate::active_plan::VMActivePlan::mutators() { - info!("stop_all_mutators: visiting {:?}", mutator.mutator_tls); + debug!("stop_all_mutators: visiting {:?}", mutator.mutator_tls); mutator_visitor(mutator); } } @@ -68,6 +76,9 @@ impl Collection for VMCollection { } fn resume_mutators(_tls: VMWorkerThread) { + // unpin conservative roots + crate::conservative::unpin_conservative_roots(); + // Get the end time of the GC let end = unsafe { jl_hrtime() }; trace!("gc_end = {}", end); @@ -86,7 +97,7 @@ impl Collection for VMCollection { let (_, cvar) = &*STW_COND.clone(); cvar.notify_all(); - info!( + debug!( "Live bytes = {}, total bytes = {}", crate::api::mmtk_used_bytes(), crate::api::mmtk_total_bytes() @@ -96,11 +107,11 @@ impl Collection for VMCollection { } fn block_for_gc(_tls: VMMutatorThread) { - info!("Triggered GC!"); + debug!("Triggered GC!"); unsafe { jl_gc_prepare_to_collect() }; - info!("Finished blocking mutator for GC!"); + debug!("Finished blocking mutator for GC!"); } fn spawn_gc_thread(_tls: VMThread, ctx: GCThreadContext) { @@ -158,6 +169,10 @@ pub fn is_current_gc_nursery() -> bool { } } +pub fn is_current_gc_moving() -> bool { + CURRENT_GC_MAY_MOVE.load(Ordering::SeqCst) +} + #[no_mangle] pub extern "C" fn mmtk_block_thread_for_gc() { AtomicBool::store(&BLOCK_FOR_GC, true, Ordering::SeqCst); @@ -165,7 +180,7 @@ pub extern "C" fn mmtk_block_thread_for_gc() { let (lock, cvar) = &*STW_COND.clone(); let mut count = lock.lock().unwrap(); - info!("Blocking for GC!"); + debug!("Blocking for GC!"); AtomicBool::store(&WORLD_HAS_STOPPED, true, Ordering::SeqCst); diff --git a/mmtk/src/conservative.rs b/mmtk/src/conservative.rs new file mode 100644 index 00000000..901a3fe7 --- /dev/null +++ b/mmtk/src/conservative.rs @@ -0,0 +1,112 @@ +use crate::jl_task_stack_buffer; +use crate::julia_types::*; +use mmtk::memory_manager; +use mmtk::util::constants::BYTES_IN_ADDRESS; +use mmtk::util::{Address, ObjectReference}; +use std::collections::HashSet; +use std::sync::Mutex; +lazy_static! { + pub static ref CONSERVATIVE_ROOTS: Mutex> = Mutex::new(HashSet::new()); +} +pub fn pin_conservative_roots() { + crate::early_return_for_non_moving_build!(()); + crate::early_return_for_current_gc!(); + + let mut roots = CONSERVATIVE_ROOTS.lock().unwrap(); + let n_roots = roots.len(); + roots.retain(|obj| mmtk::memory_manager::pin_object(*obj)); + let n_pinned = roots.len(); + log::debug!("Conservative roots: {}, pinned: {}", n_roots, n_pinned); +} +pub fn unpin_conservative_roots() { + crate::early_return_for_non_moving_build!(()); + crate::early_return_for_current_gc!(); + + let mut roots = CONSERVATIVE_ROOTS.lock().unwrap(); + let n_pinned = roots.len(); + let mut n_live = 0; + roots.drain().for_each(|obj| { + if mmtk::memory_manager::is_live_object(obj) { + n_live += 1; + mmtk::memory_manager::unpin_object(obj); + } + }); + log::debug!( + "Conservative roots: pinned: {}, unpinned/live {}", + n_pinned, + n_live + ); +} +pub fn mmtk_conservative_scan_task_stack(ta: *const jl_task_t) { + crate::early_return_for_non_moving_build!(()); + crate::early_return_for_current_gc!(); + + let mut size: u64 = 0; + let mut ptid: i32 = 0; + log::debug!("mmtk_conservative_scan_native_stack begin ta = {:?}", ta); + let stk = unsafe { jl_task_stack_buffer(ta, &mut size as *mut _, &mut ptid as *mut _) }; + log::debug!( + "mmtk_conservative_scan_native_stack continue stk = {}, size = {}, ptid = {:x}", + stk, + size, + ptid + ); + if !stk.is_zero() { + log::debug!("Conservatively scan the stack"); + // See jl_guard_size + // TODO: Are we sure there are always guard pages we need to skip? + const JL_GUARD_PAGE: usize = 4096 * 8; + let guard_page_start = stk + JL_GUARD_PAGE; + log::debug!("Skip guard page: {}, {}", stk, guard_page_start); + conservative_scan_range(guard_page_start, stk + size as usize); + } else { + log::warn!("Skip stack for {:?}", ta); + } +} +pub fn mmtk_conservative_scan_task_registers(ta: *const jl_task_t) { + crate::early_return_for_non_moving_build!(()); + crate::early_return_for_current_gc!(); + + let (lo, hi) = get_range(&unsafe { &*ta }.ctx); + conservative_scan_range(lo, hi); +} +pub fn mmtk_conservative_scan_ptls_registers(ptls: &mut _jl_tls_states_t) { + crate::early_return_for_non_moving_build!(()); + crate::early_return_for_current_gc!(); + + let (lo, hi) = get_range(&((*ptls).gc_tls.ctx_at_the_time_gc_started)); + conservative_scan_range(lo, hi); +} +// TODO: This scans the entire context type, which is slower. +// We actually only need to scan registers. +fn get_range(ctx: &T) -> (Address, Address) { + let start = Address::from_ptr(ctx); + let ty_size = std::mem::size_of::(); + (start, start + ty_size) +} +fn conservative_scan_range(lo: Address, hi: Address) { + // The high address is exclusive + let hi = if hi.is_aligned_to(BYTES_IN_ADDRESS) { + hi - BYTES_IN_ADDRESS + } else { + hi.align_down(BYTES_IN_ADDRESS) + }; + let lo = lo.align_up(BYTES_IN_ADDRESS); + log::trace!("Scan {} (lo) {} (hi)", lo, hi); + let mut cursor = hi; + while cursor >= lo { + let addr = unsafe { cursor.load::
() }; + if let Some(obj) = is_potential_mmtk_object(addr) { + CONSERVATIVE_ROOTS.lock().unwrap().insert(obj); + } + cursor -= BYTES_IN_ADDRESS; + } +} +fn is_potential_mmtk_object(addr: Address) -> Option { + if crate::object_model::is_addr_in_immixspace(addr) { + // We only care about immix space. If the object is in other spaces, we won't move them, and we don't need to pin them. + memory_manager::find_object_from_internal_pointer(addr, usize::MAX) + } else { + None + } +} diff --git a/mmtk/src/julia_types.rs b/mmtk/src/julia_types.rs index b6183e28..f230c411 100644 --- a/mmtk/src/julia_types.rs +++ b/mmtk/src/julia_types.rs @@ -2328,6 +2328,99 @@ const _: () = { ["Offset of field: _jl_excstack_t::reserved_size"] [::std::mem::offset_of!(_jl_excstack_t, reserved_size) - 8usize]; }; +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _bigval_t { + pub next: *mut _bigval_t, + pub prev: *mut _bigval_t, + pub sz: usize, + pub _padding: [*mut ::std::os::raw::c_void; 4usize], + pub __bindgen_anon_1: _bigval_t__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union _bigval_t__bindgen_ty_1 { + pub header: usize, + pub bits: _bigval_t__bindgen_ty_1__bindgen_ty_1, +} +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Copy, Clone)] +pub struct _bigval_t__bindgen_ty_1__bindgen_ty_1 { + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>, + pub __bindgen_padding_0: [u8; 7usize], +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _bigval_t__bindgen_ty_1__bindgen_ty_1"] + [::std::mem::size_of::<_bigval_t__bindgen_ty_1__bindgen_ty_1>() - 8usize]; + ["Alignment of _bigval_t__bindgen_ty_1__bindgen_ty_1"] + [::std::mem::align_of::<_bigval_t__bindgen_ty_1__bindgen_ty_1>() - 8usize]; +}; +impl _bigval_t__bindgen_ty_1__bindgen_ty_1 { + #[inline] + pub fn gc(&self) -> usize { + unsafe { ::std::mem::transmute(self._bitfield_1.get(0usize, 2u8) as u64) } + } + #[inline] + pub fn set_gc(&mut self, val: usize) { + unsafe { + let val: u64 = ::std::mem::transmute(val); + self._bitfield_1.set(0usize, 2u8, val as u64) + } + } + #[inline] + pub unsafe fn gc_raw(this: *const Self) -> usize { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 0usize, + 2u8, + ) as u64) + } + } + #[inline] + pub unsafe fn set_gc_raw(this: *mut Self, val: usize) { + unsafe { + let val: u64 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 0usize, + 2u8, + val as u64, + ) + } + } + #[inline] + pub fn new_bitfield_1(gc: usize) -> __BindgenBitfieldUnit<[u8; 1usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 1usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 2u8, { + let gc: u64 = unsafe { ::std::mem::transmute(gc) }; + gc as u64 + }); + __bindgen_bitfield_unit + } +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _bigval_t__bindgen_ty_1"][::std::mem::size_of::<_bigval_t__bindgen_ty_1>() - 8usize]; + ["Alignment of _bigval_t__bindgen_ty_1"] + [::std::mem::align_of::<_bigval_t__bindgen_ty_1>() - 8usize]; + ["Offset of field: _bigval_t__bindgen_ty_1::header"] + [::std::mem::offset_of!(_bigval_t__bindgen_ty_1, header) - 0usize]; + ["Offset of field: _bigval_t__bindgen_ty_1::bits"] + [::std::mem::offset_of!(_bigval_t__bindgen_ty_1, bits) - 0usize]; +}; +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of _bigval_t"][::std::mem::size_of::<_bigval_t>() - 64usize]; + ["Alignment of _bigval_t"][::std::mem::align_of::<_bigval_t>() - 8usize]; + ["Offset of field: _bigval_t::next"][::std::mem::offset_of!(_bigval_t, next) - 0usize]; + ["Offset of field: _bigval_t::prev"][::std::mem::offset_of!(_bigval_t, prev) - 8usize]; + ["Offset of field: _bigval_t::sz"][::std::mem::offset_of!(_bigval_t, sz) - 16usize]; + ["Offset of field: _bigval_t::_padding"][::std::mem::offset_of!(_bigval_t, _padding) - 24usize]; +}; #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { ["Size of template specialization: std_atomic_open0_size_t_close0"] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 2d1ad00b..c55236a1 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -22,6 +22,7 @@ pub mod api; mod build_info; pub mod collection; pub mod gc_trigger; +pub mod conservative; pub mod object_model; pub mod reference_glue; pub mod scanning; @@ -120,6 +121,12 @@ extern "C" { pub fn jl_gc_get_owner_address_to_mmtk(m: Address) -> Address; pub fn jl_gc_genericmemory_how(m: Address) -> usize; pub fn jl_gc_get_max_memory() -> usize; + pub fn jl_task_stack_buffer( + task: *const crate::julia_types::jl_task_t, + size: *mut u64, + ptid: *mut i32, + ) -> Address; + pub static jl_true: *mut crate::julia_types::jl_value_t; } pub(crate) fn set_panic_hook() { @@ -155,3 +162,22 @@ pub(crate) fn set_panic_hook() { } })); } + +#[macro_export] +macro_rules! early_return_for_non_moving_build { + ($ret_val:expr) => { + if cfg!(feature = "non_moving") { + return $ret_val; + } + }; +} + +/// Skip some methods if the current GC does not move objects +#[macro_export] +macro_rules! early_return_for_current_gc { + () => { + if !crate::collection::is_current_gc_moving() { + return; + } + }; +} diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 86a7b332..84bea7c8 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -21,15 +21,11 @@ pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBit pub(crate) const MARKING_METADATA_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::side_after(LOS_METADATA_SPEC.as_spec()); -#[cfg(feature = "object_pinning")] pub(crate) const LOCAL_PINNING_METADATA_BITS_SPEC: VMLocalPinningBitSpec = VMLocalPinningBitSpec::side_after(MARKING_METADATA_SPEC.as_spec()); -// pub(crate) const LOCAL_FORWARDING_POINTER_METADATA_SPEC: VMLocalForwardingPointerSpec = -// VMLocalForwardingPointerSpec::side_after(MARKING_METADATA_SPEC.as_spec()); - -// pub(crate) const LOCAL_FORWARDING_METADATA_BITS_SPEC: VMLocalForwardingBitsSpec = -// VMLocalForwardingBitsSpec::side_after(LOCAL_FORWARDING_POINTER_METADATA_SPEC.as_spec()); +pub(crate) const LOCAL_FORWARDING_METADATA_BITS_SPEC: VMLocalForwardingBitsSpec = + VMLocalForwardingBitsSpec::side_after(LOCAL_PINNING_METADATA_BITS_SPEC.as_spec()); /// PolicySpecific mark-and-nursery bits metadata spec /// 2-bits per object @@ -41,21 +37,15 @@ impl ObjectModel for VMObjectModel { const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = VMLocalForwardingPointerSpec::in_header(-64); - #[cfg(feature = "object_pinning")] - const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = - VMLocalForwardingBitsSpec::side_after(LOCAL_PINNING_METADATA_BITS_SPEC.as_spec()); - #[cfg(not(feature = "object_pinning"))] + const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = LOCAL_PINNING_METADATA_BITS_SPEC; const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = - VMLocalForwardingBitsSpec::side_after(MARKING_METADATA_SPEC.as_spec()); + LOCAL_FORWARDING_METADATA_BITS_SPEC; const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = MARKING_METADATA_SPEC; const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = LOS_METADATA_SPEC; const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = false; const OBJECT_REF_OFFSET_LOWER_BOUND: isize = 0; - #[cfg(feature = "object_pinning")] - const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = LOCAL_PINNING_METADATA_BITS_SPEC; - fn copy( from: ObjectReference, semantics: CopySemantics, @@ -123,10 +113,20 @@ impl ObjectModel for VMObjectModel { } fn get_current_size(object: ObjectReference) -> usize { - // not being called by objects in LOS - debug_assert!(!is_object_in_los(&object)); - - unsafe { get_so_object_size(object) } + if is_object_in_los(&object) { + unsafe { get_lo_object_size(object) } + } else if is_object_in_immixspace(&object) { + unsafe { get_so_object_size(object) } + } else { + // This is hacky but it should work. + // This covers the cases for immortal space and VM space. + // For those spaces, we only query object size when we try to find the base reference for an internal pointer. + // For those two spaces, we bulk set VO bits so we cannot find the base reference at all. + // We return 0 as the object size, so MMTk core won't find the base reference. + // As we only use the base reference to pin the objects, we cannot pin the objects. But it is fine, + // as objects in those spaces won't be moved. + 0 + } } fn get_size_when_copied(_object: ObjectReference) -> usize { @@ -176,7 +176,19 @@ pub fn is_object_in_los(object: &ObjectReference) -> bool { } #[inline(always)] +pub fn is_object_in_immixspace(object: &ObjectReference) -> bool { + is_addr_in_immixspace((*object).to_raw_address()) +} + +#[inline(always)] +pub fn is_addr_in_immixspace(addr: Address) -> bool { + // FIXME: get the range from MMTk. Or at least assert at boot time to make sure those constants are correct. + addr.as_usize() >= 0x200_0000_0000 && addr.as_usize() < 0x400_0000_0000 +} + /// This function uses mutable static variables and requires unsafe annotation + +#[inline(always)] pub unsafe fn get_so_object_size(object: ObjectReference) -> usize { let obj_address = object.to_raw_address(); let mut vtag = mmtk_jl_typetagof(obj_address); @@ -303,6 +315,13 @@ pub unsafe fn get_so_object_size(object: ObjectReference) -> usize { llt_align(dtsz + JULIA_HEADER_SIZE, 16) } +#[inline(always)] +pub unsafe fn get_lo_object_size(object: ObjectReference) -> usize { + let obj_address = object.to_raw_address(); + let julia_big_object = obj_address.to_ptr::<_bigval_t>(); + return (*julia_big_object).sz; +} + #[inline(always)] pub unsafe fn get_object_start_ref(object: ObjectReference) -> Address { let obj_address = object.to_raw_address(); diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index c430d5f1..25ebc5ee 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -53,13 +53,18 @@ impl Scanning for VMScanning { use mmtk::util::Address; let ptls: &mut _jl_tls_states_t = unsafe { std::mem::transmute(mutator.mutator_tls) }; + let pthread = ptls.system_id; let mut tpinning_slot_buffer = SlotBuffer { buffer: vec![] }; // need to be transitively pinned let mut pinning_slot_buffer = SlotBuffer { buffer: vec![] }; // roots from the shadow stack that we know that do not need to be transitively pinned let mut node_buffer = vec![]; + // Conservatively scan registers saved with the thread + crate::conservative::mmtk_conservative_scan_ptls_registers(ptls); + // Scan thread local from ptls: See gc_queue_thread_local in gc.c let mut root_scan_task = |task: *const _jl_task_t, task_is_root: bool| { if !task.is_null() { + // Scan gc preserve and shadow stacks unsafe { crate::julia_scanning::mmtk_scan_gcpreserve_stack( task, @@ -71,6 +76,19 @@ impl Scanning for VMScanning { Some(&mut pinning_slot_buffer), ); } + + // Conservatively scan native stacks to make sure we won't move objects that the runtime is using. + log::debug!( + "Scanning ptls {:?}, pthread {:x}", + mutator.mutator_tls, + pthread + ); + // Conservative scan stack and registers. If the task hasn't been started, we do not need to scan its stack and registers. + if unsafe { (*task).start == crate::jl_true } { + crate::conservative::mmtk_conservative_scan_task_stack(task); + crate::conservative::mmtk_conservative_scan_task_registers(task); + } + if task_is_root { // captures wrong root nodes before creating the work debug_assert!( @@ -183,6 +201,9 @@ impl Scanning for VMScanning { process_object(object, slot_visitor); } fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { + // pin conservative roots from stack scanning + crate::conservative::pin_conservative_roots(); + let sweep_vm_specific_work = SweepVMSpecific::new(); memory_manager::add_work_packet( &SINGLETON, From 5c8f1b8a737c895180a3c1235cb6aed6004c932a Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 23 Jan 2025 14:50:36 +1300 Subject: [PATCH 10/17] Abort for GC thread panic (#221) The code is mainly from `mmrk-ruby`. We hijack the Rust panic hook to make sure that if the GC threads panic, the process will be aborted. The current behavior is that when a GC thread panics, the process hangs and wait for that GC thread to finish its work. This should be ported to `master`. --- mmtk/src/lib.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index c55236a1..520ff6f2 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -181,3 +181,37 @@ macro_rules! early_return_for_current_gc { } }; } + +pub(crate) fn set_panic_hook() { + let old_hook = std::panic::take_hook(); + + std::panic::set_hook(Box::new(move |panic_info| { + if crate::collection::is_gc_thread() { + eprintln!("ERROR: An MMTk GC thread panicked. This is a bug."); + eprintln!("{panic_info}"); + + let bt = std::backtrace::Backtrace::capture(); + match bt.status() { + std::backtrace::BacktraceStatus::Unsupported => { + eprintln!("Backtrace is unsupported.") + } + std::backtrace::BacktraceStatus::Disabled => { + eprintln!("Backtrace is disabled."); + eprintln!( + "run with `RUST_BACKTRACE=1` environment variable to display a backtrace" + ); + } + std::backtrace::BacktraceStatus::Captured => { + eprintln!("{bt}"); + } + s => { + eprintln!("Unknown backtrace status: {s:?}"); + } + } + + std::process::abort(); + } else { + old_hook(panic_info); + } + })); +} From 9775aaea4ba8675fbbfaf8e5ff8b832d7c14fdea Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 5 Feb 2025 19:04:47 +1300 Subject: [PATCH 11/17] Add pointer pinning to handle internal pointers. Use `jl_active_task_stack`. (#217) This PR adds pinning that handles internal pointers. It also uses `jl_active_task_stack` for stack scanning instead of `jl_task_stack_buffer`. --- mmtk/api/mmtk.h | 4 +- mmtk/src/api.rs | 116 +++++++++++++++++++++++++++++++++++---- mmtk/src/conservative.rs | 37 +++++++++---- mmtk/src/lib.rs | 10 ++-- 4 files changed, 138 insertions(+), 29 deletions(-) diff --git a/mmtk/api/mmtk.h b/mmtk/api/mmtk.h index 4c3085f9..06561871 100644 --- a/mmtk/api/mmtk.h +++ b/mmtk/api/mmtk.h @@ -49,7 +49,9 @@ extern int mmtk_object_is_managed_by_mmtk(void* addr); extern void mmtk_runtime_panic(void); extern void mmtk_unreachable(void); extern unsigned char mmtk_pin_object(void* obj); -extern bool mmtk_is_pinned(void* obj); +extern bool mmtk_is_object_pinned(void* obj); +extern unsigned char mmtk_pin_pointer(void* ptr); +extern bool mmtk_is_pointer_pinned(void* ptr); extern const char* get_mmtk_version(void); extern void mmtk_set_vm_space(void* addr, size_t size); diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 819a1b71..08f0dafc 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -485,39 +485,131 @@ pub extern "C" fn mmtk_get_obj_size(obj: ObjectReference) -> usize { } } +#[allow(unused_variables)] +fn assert_is_object(object: ObjectReference) { + // The checks are quite expensive. Dont run it in normal builds. + const ASSERT_OBJECT: bool = false; + if ASSERT_OBJECT { + #[cfg(debug_assertions)] + { + use crate::object_model::{is_object_in_immixspace, is_object_in_los}; + if !mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { + panic!("{} is not managed by MMTk", object); + } + if !is_object_in_immixspace(&object) && !is_object_in_los(&object) { + // We will use VO bit in the following check. But if the object is not in immix space or LOS, we cannot do the check. + return; + } + if !object + .to_raw_address() + .is_aligned_to(ObjectReference::ALIGNMENT) + { + panic!( + "{} is not aligned, it cannot be an object reference", + object + ) + } + if memory_manager::is_mmtk_object(object.to_raw_address()).is_none() { + error!("{} is not an object", object); + if let Some(base_ref) = memory_manager::find_object_from_internal_pointer( + object.to_raw_address(), + usize::MAX, + ) { + panic!("{} is an internal pointer of {}", object, base_ref); + } else { + panic!( + "{} is not recognised as an object reference, or an internal reference", + object + ); + } + } + } + } +} #[no_mangle] pub extern "C" fn mmtk_pin_object(object: ObjectReference) -> bool { + assert_is_object(object); + crate::early_return_for_non_moving_build!(false); + memory_manager::pin_object(object) +} + +#[no_mangle] +pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool { + assert_is_object(object); + crate::early_return_for_non_moving_build!(false); + memory_manager::unpin_object(object) +} + +#[no_mangle] +pub extern "C" fn mmtk_is_object_pinned(object: ObjectReference) -> bool { + assert_is_object(object); crate::early_return_for_non_moving_build!(false); - // We may in the future replace this with a check for the immix space (bound check), which should be much cheaper. - if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { - memory_manager::pin_object(object) + memory_manager::is_pinned(object) +} + +macro_rules! handle_potential_internal_pointer { + ($func: path, $addr: expr) => {{ + if $addr.is_aligned_to(ObjectReference::ALIGNMENT) { + if let Some(obj) = memory_manager::is_mmtk_object($addr) { + return $func(obj); + } + } + let maybe_objref = memory_manager::find_object_from_internal_pointer($addr, usize::MAX); + if let Some(obj) = maybe_objref { + trace!( + "Attempt to pin {:?}, but it is an internal reference of {:?}", + $addr, + obj + ); + $func(obj) + } else { + warn!( + "Attempt to pin {:?}, but it is not recognised as a object", + $addr + ); + false + } + }}; +} + +#[no_mangle] +pub extern "C" fn mmtk_pin_pointer(addr: Address) -> bool { + crate::early_return_for_non_moving_build!(false); + + if crate::object_model::is_addr_in_immixspace(addr) { + handle_potential_internal_pointer!(memory_manager::pin_object, addr) } else { - debug!("Object is not managed by mmtk - (un)pinning it via this function isn't supported."); + debug!("Object is not in Immix space. MMTk will not move the object. No need to pin the object."); false } } #[no_mangle] -pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool { +pub extern "C" fn mmtk_unpin_pointer(addr: Address) -> bool { crate::early_return_for_non_moving_build!(false); - if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { - memory_manager::unpin_object(object) + if crate::object_model::is_addr_in_immixspace(addr) { + handle_potential_internal_pointer!(memory_manager::unpin_object, addr) } else { - debug!("Object is not managed by mmtk - (un)pinning it via this function isn't supported."); + debug!("Object is not in Immix space. MMTk will not move the object. No need to unpin the object."); false } } #[no_mangle] -pub extern "C" fn mmtk_is_pinned(object: ObjectReference) -> bool { +pub extern "C" fn mmtk_is_pointer_pinned(addr: Address) -> bool { crate::early_return_for_non_moving_build!(false); - if mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()) { - memory_manager::is_pinned(object) + if crate::object_model::is_addr_in_immixspace(addr) { + handle_potential_internal_pointer!(memory_manager::is_pinned, addr) + } else if !mmtk_object_is_managed_by_mmtk(addr.as_usize()) { + debug!( + "Object is not in Immix space. MMTk will not move the object. We assume it is pinned." + ); + true } else { - debug!("Object is not managed by mmtk - checking via this function isn't supported."); + debug!("Object is not managed by mmtk - checking pinning state via this function isn't supported."); false } } diff --git a/mmtk/src/conservative.rs b/mmtk/src/conservative.rs index 901a3fe7..0ce6c80e 100644 --- a/mmtk/src/conservative.rs +++ b/mmtk/src/conservative.rs @@ -1,4 +1,3 @@ -use crate::jl_task_stack_buffer; use crate::julia_types::*; use mmtk::memory_manager; use mmtk::util::constants::BYTES_IN_ADDRESS; @@ -41,24 +40,38 @@ pub fn mmtk_conservative_scan_task_stack(ta: *const jl_task_t) { crate::early_return_for_non_moving_build!(()); crate::early_return_for_current_gc!(); - let mut size: u64 = 0; - let mut ptid: i32 = 0; log::debug!("mmtk_conservative_scan_native_stack begin ta = {:?}", ta); - let stk = unsafe { jl_task_stack_buffer(ta, &mut size as *mut _, &mut ptid as *mut _) }; + let mut active_start = Address::ZERO; + let mut active_end = Address::ZERO; + let mut total_start = Address::ZERO; + let mut total_end = Address::ZERO; + unsafe { + crate::jl_active_task_stack( + ta, + &mut active_start as _, + &mut active_end as _, + &mut total_start as _, + &mut total_end as _, + ) + }; log::debug!( - "mmtk_conservative_scan_native_stack continue stk = {}, size = {}, ptid = {:x}", - stk, - size, - ptid + "mmtk_conservative_scan_native_stack continue, active = {},{}, total = {},{}", + active_start, + active_end, + total_start, + total_end, ); - if !stk.is_zero() { + + let size = active_end - active_start; + + if !active_start.is_zero() { log::debug!("Conservatively scan the stack"); // See jl_guard_size // TODO: Are we sure there are always guard pages we need to skip? const JL_GUARD_PAGE: usize = 4096 * 8; - let guard_page_start = stk + JL_GUARD_PAGE; - log::debug!("Skip guard page: {}, {}", stk, guard_page_start); - conservative_scan_range(guard_page_start, stk + size as usize); + let guard_page_start = active_start + JL_GUARD_PAGE; + log::debug!("Skip guard page: {}, {}", active_start, guard_page_start); + conservative_scan_range(guard_page_start, active_start + size); } else { log::warn!("Skip stack for {:?}", ta); } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 520ff6f2..ae03dd81 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -121,11 +121,13 @@ extern "C" { pub fn jl_gc_get_owner_address_to_mmtk(m: Address) -> Address; pub fn jl_gc_genericmemory_how(m: Address) -> usize; pub fn jl_gc_get_max_memory() -> usize; - pub fn jl_task_stack_buffer( + pub fn jl_active_task_stack( task: *const crate::julia_types::jl_task_t, - size: *mut u64, - ptid: *mut i32, - ) -> Address; + active_start: *mut Address, + active_end: *mut Address, + total_start: *mut Address, + total_end: *mut Address, + ); pub static jl_true: *mut crate::julia_types::jl_value_t; } From 478ba4e9606f9d1ec52b85db0c907fd186f805fa Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 6 Feb 2025 05:03:12 +0000 Subject: [PATCH 12/17] Remove redundant set_panic_hook (merge issue) --- mmtk/src/lib.rs | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index ae03dd81..8414b780 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -131,40 +131,6 @@ extern "C" { pub static jl_true: *mut crate::julia_types::jl_value_t; } -pub(crate) fn set_panic_hook() { - let old_hook = std::panic::take_hook(); - - std::panic::set_hook(Box::new(move |panic_info| { - if crate::collection::is_gc_thread() { - eprintln!("ERROR: An MMTk GC thread panicked. This is a bug."); - eprintln!("{panic_info}"); - - let bt = std::backtrace::Backtrace::capture(); - match bt.status() { - std::backtrace::BacktraceStatus::Unsupported => { - eprintln!("Backtrace is unsupported.") - } - std::backtrace::BacktraceStatus::Disabled => { - eprintln!("Backtrace is disabled."); - eprintln!( - "run with `RUST_BACKTRACE=1` environment variable to display a backtrace" - ); - } - std::backtrace::BacktraceStatus::Captured => { - eprintln!("{bt}"); - } - s => { - eprintln!("Unknown backtrace status: {s:?}"); - } - } - - std::process::abort(); - } else { - old_hook(panic_info); - } - })); -} - #[macro_export] macro_rules! early_return_for_non_moving_build { ($ret_val:expr) => { From 5d65c90e890460ba41ae500542ead1364e677176 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 6 Feb 2025 23:07:42 +0000 Subject: [PATCH 13/17] Adding usings_backedges root --- mmtk/src/julia_scanning.rs | 6 ++++++ mmtk/src/julia_types.rs | 29 ++++++++++++++++------------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/mmtk/src/julia_scanning.rs b/mmtk/src/julia_scanning.rs index eea2c507..2a321b53 100644 --- a/mmtk/src/julia_scanning.rs +++ b/mmtk/src/julia_scanning.rs @@ -132,6 +132,12 @@ pub unsafe fn scan_julia_object>(obj: Address, clos } process_slot(closure, Address::from_ptr(parent_slot)); + let usings_backeges_slot = ::std::ptr::addr_of!((*m).usings_backedges); + if PRINT_OBJ_TYPE { + println!(" - scan parent: {:?}\n", usings_backeges_slot); + } + process_slot(closure, Address::from_ptr(usings_backeges_slot)); + // m.usings.items may be inlined in the module when the array list size <= AL_N_INLINE (cf. arraylist_new) // In that case it may be an mmtk object and not a malloced address. // If it is an mmtk object, (*m).usings.items will then be an internal pointer to the module diff --git a/mmtk/src/julia_types.rs b/mmtk/src/julia_types.rs index f230c411..f4f15f4e 100644 --- a/mmtk/src/julia_types.rs +++ b/mmtk/src/julia_types.rs @@ -2122,6 +2122,7 @@ pub struct _jl_module_t { pub bindingkeyset: u64, pub file: *mut jl_sym_t, pub line: i32, + pub usings_backedges: *mut jl_value_t, pub usings: arraylist_t, pub build_id: jl_uuid_t, pub uuid: jl_uuid_t, @@ -2137,7 +2138,7 @@ pub struct _jl_module_t { } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of _jl_module_t"][::std::mem::size_of::<_jl_module_t>() - 376usize]; + ["Size of _jl_module_t"][::std::mem::size_of::<_jl_module_t>() - 384usize]; ["Alignment of _jl_module_t"][::std::mem::align_of::<_jl_module_t>() - 8usize]; ["Offset of field: _jl_module_t::name"][::std::mem::offset_of!(_jl_module_t, name) - 0usize]; ["Offset of field: _jl_module_t::parent"] @@ -2148,27 +2149,29 @@ const _: () = { [::std::mem::offset_of!(_jl_module_t, bindingkeyset) - 24usize]; ["Offset of field: _jl_module_t::file"][::std::mem::offset_of!(_jl_module_t, file) - 32usize]; ["Offset of field: _jl_module_t::line"][::std::mem::offset_of!(_jl_module_t, line) - 40usize]; + ["Offset of field: _jl_module_t::usings_backedges"] + [::std::mem::offset_of!(_jl_module_t, usings_backedges) - 48usize]; ["Offset of field: _jl_module_t::usings"] - [::std::mem::offset_of!(_jl_module_t, usings) - 48usize]; + [::std::mem::offset_of!(_jl_module_t, usings) - 56usize]; ["Offset of field: _jl_module_t::build_id"] - [::std::mem::offset_of!(_jl_module_t, build_id) - 304usize]; - ["Offset of field: _jl_module_t::uuid"][::std::mem::offset_of!(_jl_module_t, uuid) - 320usize]; + [::std::mem::offset_of!(_jl_module_t, build_id) - 312usize]; + ["Offset of field: _jl_module_t::uuid"][::std::mem::offset_of!(_jl_module_t, uuid) - 328usize]; ["Offset of field: _jl_module_t::counter"] - [::std::mem::offset_of!(_jl_module_t, counter) - 336usize]; + [::std::mem::offset_of!(_jl_module_t, counter) - 344usize]; ["Offset of field: _jl_module_t::nospecialize"] - [::std::mem::offset_of!(_jl_module_t, nospecialize) - 340usize]; + [::std::mem::offset_of!(_jl_module_t, nospecialize) - 348usize]; ["Offset of field: _jl_module_t::optlevel"] - [::std::mem::offset_of!(_jl_module_t, optlevel) - 344usize]; + [::std::mem::offset_of!(_jl_module_t, optlevel) - 352usize]; ["Offset of field: _jl_module_t::compile"] - [::std::mem::offset_of!(_jl_module_t, compile) - 345usize]; + [::std::mem::offset_of!(_jl_module_t, compile) - 353usize]; ["Offset of field: _jl_module_t::infer"] - [::std::mem::offset_of!(_jl_module_t, infer) - 346usize]; + [::std::mem::offset_of!(_jl_module_t, infer) - 354usize]; ["Offset of field: _jl_module_t::istopmod"] - [::std::mem::offset_of!(_jl_module_t, istopmod) - 347usize]; + [::std::mem::offset_of!(_jl_module_t, istopmod) - 355usize]; ["Offset of field: _jl_module_t::max_methods"] - [::std::mem::offset_of!(_jl_module_t, max_methods) - 348usize]; - ["Offset of field: _jl_module_t::lock"][::std::mem::offset_of!(_jl_module_t, lock) - 352usize]; - ["Offset of field: _jl_module_t::hash"][::std::mem::offset_of!(_jl_module_t, hash) - 368usize]; + [::std::mem::offset_of!(_jl_module_t, max_methods) - 356usize]; + ["Offset of field: _jl_module_t::lock"][::std::mem::offset_of!(_jl_module_t, lock) - 360usize]; + ["Offset of field: _jl_module_t::hash"][::std::mem::offset_of!(_jl_module_t, hash) - 376usize]; }; pub type jl_module_t = _jl_module_t; #[repr(C)] From 7cee345f3c0cda7df220d7af8e884f3b1d40fbd8 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Fri, 7 Feb 2025 00:24:42 +0000 Subject: [PATCH 14/17] Skipping Distributed tests --- .github/scripts/ci-test-stdlib.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/ci-test-stdlib.sh b/.github/scripts/ci-test-stdlib.sh index f2858768..f50d9b73 100755 --- a/.github/scripts/ci-test-stdlib.sh +++ b/.github/scripts/ci-test-stdlib.sh @@ -22,6 +22,8 @@ declare -a tests_to_skip=( "SparseArrays" # Running LinearAlgebra in a separate job "LinearAlgebra" + # Skipping Distributed tests + "Distributed" ) # These tests need multiple workers. declare -a tests_with_multi_workers=( From aa84850dc69f0c05f5e91f7e1eecf60342c5bf09 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 7 Feb 2025 03:47:04 +0000 Subject: [PATCH 15/17] Run moving in CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72d7a7ac..1e45266e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: fail-fast: false matrix: gc_plan: [Immix] - moving: [Non_Moving] + moving: [Non_Moving, Moving] uses: ./.github/workflows/binding-tests.yml with: gc_plan: ${{ matrix.gc_plan }} From 231660ddcc0f1203f607bf9e9d762d12f224cd59 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 11 Mar 2025 03:18:26 +0000 Subject: [PATCH 16/17] Always conservatively scan tasks without considering task.start --- mmtk/src/scanning.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 25ebc5ee..34addb08 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -84,10 +84,12 @@ impl Scanning for VMScanning { pthread ); // Conservative scan stack and registers. If the task hasn't been started, we do not need to scan its stack and registers. - if unsafe { (*task).start == crate::jl_true } { + // We cannot use `task->start` to skip conservative scanning, as before a task is started, the runtime may evaluate the code and we need to make sure the runtime objects are properly scanned. + // However, without this check, we may timeout in a test that spawns a lot of tasks. + // if unsafe { (*task).start == crate::jl_true } { crate::conservative::mmtk_conservative_scan_task_stack(task); crate::conservative::mmtk_conservative_scan_task_registers(task); - } + // } if task_is_root { // captures wrong root nodes before creating the work From 11729f1e907fb22ad0a67a46e1b851b6d86e31c0 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Sun, 23 Mar 2025 23:08:41 +0000 Subject: [PATCH 17/17] Fix get_lo_object_size --- mmtk/src/object_model.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 84bea7c8..feefc983 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -318,7 +318,7 @@ pub unsafe fn get_so_object_size(object: ObjectReference) -> usize { #[inline(always)] pub unsafe fn get_lo_object_size(object: ObjectReference) -> usize { let obj_address = object.to_raw_address(); - let julia_big_object = obj_address.to_ptr::<_bigval_t>(); + let julia_big_object = (obj_address - std::mem::size_of::<_bigval_t>()).to_ptr::<_bigval_t>(); return (*julia_big_object).sz; }