diff --git a/src/mmtk.rs b/src/mmtk.rs index 2916f98f82..34969c0027 100644 --- a/src/mmtk.rs +++ b/src/mmtk.rs @@ -48,7 +48,7 @@ lazy_static! { pub static ref VM_MAP: Box = layout::create_vm_map(); /// A global Mmapper for mmaping and protection of virtual memory. - pub static ref MMAPPER: Box = layout::create_mmapper(); + pub static ref MMAPPER: Box = layout::create_mmapper(); } use crate::util::rust_util::InitializeOnce; diff --git a/src/plan/global.rs b/src/plan/global.rs index 5a5bb38ab5..4b7a3286e9 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -377,7 +377,7 @@ pub struct BasePlan { /// is passed to each plan's constructor. pub struct CreateGeneralPlanArgs<'a, VM: VMBinding> { pub vm_map: &'static dyn VMMap, - pub mmapper: &'static dyn Mmapper, + pub mmapper: &'static Mmapper, pub options: Arc, pub state: Arc, pub gc_trigger: Arc>, diff --git a/src/policy/space.rs b/src/policy/space.rs index e44874fe5b..0946bceaf5 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -517,7 +517,7 @@ pub struct CommonSpace { pub extent: usize, pub vm_map: &'static dyn VMMap, - pub mmapper: &'static dyn Mmapper, + pub mmapper: &'static Mmapper, pub(crate) metadata: SideMetadataContext, @@ -551,7 +551,7 @@ pub struct PlanCreateSpaceArgs<'a, VM: VMBinding> { pub vmrequest: VMRequest, pub global_side_metadata_specs: Vec, pub vm_map: &'static dyn VMMap, - pub mmapper: &'static dyn Mmapper, + pub mmapper: &'static Mmapper, pub heap: &'a mut HeapMeta, pub constraints: &'a PlanConstraints, pub gc_trigger: Arc>, diff --git a/src/util/heap/layout/byte_map_mmapper.rs b/src/util/heap/layout/byte_map_mmapper.rs deleted file mode 100644 index 412e53ddcb..0000000000 --- a/src/util/heap/layout/byte_map_mmapper.rs +++ /dev/null @@ -1,428 +0,0 @@ -use super::mmapper::MapState; -use super::Mmapper; -use crate::util::memory::MmapAnnotation; -use crate::util::Address; - -use crate::util::constants::*; -use crate::util::conversions::pages_to_bytes; -use crate::util::heap::layout::vm_layout::*; -use crate::util::memory::MmapStrategy; -use std::fmt; -use std::sync::atomic::Ordering; -use std::sync::Mutex; - -use atomic::Atomic; -use std::io::Result; - -const MMAP_NUM_CHUNKS: usize = if LOG_BYTES_IN_ADDRESS_SPACE == 32 { - 1 << (LOG_BYTES_IN_ADDRESS_SPACE as usize - LOG_MMAP_CHUNK_BYTES) -} else { - 1 << (33 - LOG_MMAP_CHUNK_BYTES) -}; -pub const VERBOSE: bool = true; - -pub struct ByteMapMmapper { - lock: Mutex<()>, - mapped: [Atomic; MMAP_NUM_CHUNKS], -} - -impl fmt::Debug for ByteMapMmapper { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "ByteMapMmapper({})", MMAP_NUM_CHUNKS) - } -} - -impl Mmapper for ByteMapMmapper { - fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]) { - unimplemented!() - } - - fn mark_as_mapped(&self, start: Address, bytes: usize) { - let start_chunk = Self::address_to_mmap_chunks_down(start); - let end_chunk = Self::address_to_mmap_chunks_up(start + bytes) - 1; - for i in start_chunk..=end_chunk { - self.mapped[i].store(MapState::Mapped, Ordering::Relaxed); - } - } - - fn ensure_mapped( - &self, - start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - let start_chunk = Self::address_to_mmap_chunks_down(start); - let end_chunk = Self::address_to_mmap_chunks_up(start + pages_to_bytes(pages)); - trace!( - "Calling ensure_mapped with start={:?} and {} pages, {}-{}", - start, - pages, - Self::mmap_chunks_to_address(start_chunk), - Self::mmap_chunks_to_address(end_chunk) - ); - - for chunk in start_chunk..end_chunk { - if self.mapped[chunk].load(Ordering::Relaxed) == MapState::Mapped { - continue; - } - - let mmap_start = Self::mmap_chunks_to_address(chunk); - let _guard = self.lock.lock().unwrap(); - MapState::transition_to_mapped(&self.mapped[chunk], mmap_start, strategy, anno) - .unwrap(); - } - - Ok(()) - } - - fn quarantine_address_range( - &self, - start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - let start_chunk = Self::address_to_mmap_chunks_down(start); - let end_chunk = Self::address_to_mmap_chunks_up(start + pages_to_bytes(pages)); - trace!( - "Calling quarantine_address_range with start={:?} and {} pages, {}-{}", - start, - pages, - Self::mmap_chunks_to_address(start_chunk), - Self::mmap_chunks_to_address(end_chunk) - ); - - for chunk in start_chunk..end_chunk { - if self.mapped[chunk].load(Ordering::Relaxed) == MapState::Mapped { - continue; - } - - let mmap_start = Self::mmap_chunks_to_address(chunk); - let _guard = self.lock.lock().unwrap(); - MapState::transition_to_quarantined(&self.mapped[chunk], mmap_start, strategy, anno) - .unwrap(); - } - - Ok(()) - } - - /** - * Return {@code true} if the given address has been mmapped - * - * @param addr The address in question. - * @return {@code true} if the given address has been mmapped - */ - fn is_mapped_address(&self, addr: Address) -> bool { - let chunk = Self::address_to_mmap_chunks_down(addr); - self.mapped[chunk].load(Ordering::Relaxed) == MapState::Mapped - } - - fn protect(&self, start: Address, pages: usize) { - let start_chunk = Self::address_to_mmap_chunks_down(start); - let chunks = Self::pages_to_mmap_chunks_up(pages); - let end_chunk = start_chunk + chunks; - let _guard = self.lock.lock().unwrap(); - - for chunk in start_chunk..end_chunk { - let mmap_start = Self::mmap_chunks_to_address(chunk); - MapState::transition_to_protected(&self.mapped[chunk], mmap_start).unwrap(); - } - } -} - -impl ByteMapMmapper { - pub fn new() -> Self { - // Because AtomicU8 does not implement Copy, it is a compilation error to usen the - // expression `[Atomic::new(MapState::Unmapped); MMAP_NUM_CHUNKS]` because that involves - // copying. We must define a constant for it. - // - // TODO: Use the inline const expression `const { Atomic::new(MapState::Unmapped) }` after - // we bump MSRV to 1.79. - - // If we declare a const Atomic, Clippy will warn about const items being interior mutable. - // Using inline const expression will eliminate this warning, but that is experimental until - // 1.79. Fix it after we bump MSRV. - #[allow(clippy::declare_interior_mutable_const)] - const INITIAL_ENTRY: Atomic = Atomic::new(MapState::Unmapped); - - ByteMapMmapper { - lock: Mutex::new(()), - mapped: [INITIAL_ENTRY; MMAP_NUM_CHUNKS], - } - } - - fn bytes_to_mmap_chunks_up(bytes: usize) -> usize { - (bytes + MMAP_CHUNK_BYTES - 1) >> LOG_MMAP_CHUNK_BYTES - } - - fn pages_to_mmap_chunks_up(pages: usize) -> usize { - Self::bytes_to_mmap_chunks_up(pages_to_bytes(pages)) - } - - fn address_to_mmap_chunks_down(addr: Address) -> usize { - addr >> LOG_MMAP_CHUNK_BYTES - } - - fn mmap_chunks_to_address(chunk: usize) -> Address { - unsafe { Address::from_usize(chunk << LOG_MMAP_CHUNK_BYTES) } - } - - fn address_to_mmap_chunks_up(addr: Address) -> usize { - (addr + MMAP_CHUNK_BYTES - 1) >> LOG_MMAP_CHUNK_BYTES - } -} - -impl Default for ByteMapMmapper { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::ByteMapMmapper; - use crate::mmap_anno_test; - use crate::util::heap::layout::Mmapper; - use crate::util::Address; - - use crate::util::constants::LOG_BYTES_IN_PAGE; - use crate::util::conversions::pages_to_bytes; - use crate::util::heap::layout::mmapper::MapState; - use crate::util::heap::layout::vm_layout::MMAP_CHUNK_BYTES; - use crate::util::memory::{self, MmapStrategy}; - use crate::util::test_util::BYTE_MAP_MMAPPER_TEST_REGION; - use crate::util::test_util::{serial_test, with_cleanup}; - use std::sync::atomic::Ordering; - - const CHUNK_SIZE: usize = 1 << 22; - const FIXED_ADDRESS: Address = BYTE_MAP_MMAPPER_TEST_REGION.start; - const MAX_SIZE: usize = BYTE_MAP_MMAPPER_TEST_REGION.size; - - #[test] - fn address_to_mmap_chunks() { - for i in 0..10 { - unsafe { - let start = CHUNK_SIZE * i; - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_up(Address::from_usize(start)), - i - ); - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_down(Address::from_usize(start)), - i - ); - - let middle = start + 8; - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_up(Address::from_usize(middle)), - i + 1 - ); - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_down(Address::from_usize(middle)), - i - ); - - let end = start + CHUNK_SIZE; - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_up(Address::from_usize(end)), - i + 1 - ); - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_down(Address::from_usize(end)), - i + 1 - ); - } - } - } - - #[test] - fn ensure_mapped_1page() { - serial_test(|| { - let pages = 1; - let start_chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - let end_chunk = - ByteMapMmapper::address_to_mmap_chunks_up(FIXED_ADDRESS + pages_to_bytes(pages)); - let test_memory_bytes = (end_chunk - start_chunk) * MMAP_CHUNK_BYTES; - with_cleanup( - || { - let mmapper = ByteMapMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - for chunk in start_chunk..end_chunk { - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Mapped - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, test_memory_bytes).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_1chunk() { - serial_test(|| { - let pages = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - let start_chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - let end_chunk = - ByteMapMmapper::address_to_mmap_chunks_up(FIXED_ADDRESS + pages_to_bytes(pages)); - let test_memory_bytes = (end_chunk - start_chunk) * MMAP_CHUNK_BYTES; - with_cleanup( - || { - let mmapper = ByteMapMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - for chunk in start_chunk..end_chunk { - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Mapped - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, test_memory_bytes).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_more_than_1chunk() { - serial_test(|| { - let pages = (MMAP_CHUNK_BYTES + MMAP_CHUNK_BYTES / 2) >> LOG_BYTES_IN_PAGE as usize; - let start_chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - let end_chunk = - ByteMapMmapper::address_to_mmap_chunks_up(FIXED_ADDRESS + pages_to_bytes(pages)); - let test_memory_bytes = (end_chunk - start_chunk) * MMAP_CHUNK_BYTES; - with_cleanup( - || { - let mmapper = ByteMapMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let start_chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - let end_chunk = ByteMapMmapper::address_to_mmap_chunks_up( - FIXED_ADDRESS + pages_to_bytes(pages), - ); - assert_eq!(end_chunk - start_chunk, 2); - for chunk in start_chunk..end_chunk { - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Mapped - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, test_memory_bytes).unwrap(); - }, - ) - }) - } - - #[test] - fn protect() { - serial_test(|| { - let test_memory_bytes = MMAP_CHUNK_BYTES * 2; - let test_memory_pages = test_memory_bytes >> LOG_BYTES_IN_PAGE; - let protect_memory_bytes = MMAP_CHUNK_BYTES; - let protect_memory_pages = protect_memory_bytes >> LOG_BYTES_IN_PAGE; - with_cleanup( - || { - // map 2 chunks - let mmapper = ByteMapMmapper::new(); - mmapper - .ensure_mapped( - FIXED_ADDRESS, - test_memory_pages, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - - // protect 1 chunk - mmapper.protect(FIXED_ADDRESS, protect_memory_pages); - - let chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Protected - ); - assert_eq!( - mmapper.mapped[chunk + 1].load(Ordering::Relaxed), - MapState::Mapped - ); - }, - || { - memory::munmap(FIXED_ADDRESS, test_memory_bytes).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_on_protected_chunks() { - serial_test(|| { - let test_memory_bytes = MMAP_CHUNK_BYTES * 2; - let test_memory_pages = test_memory_bytes >> LOG_BYTES_IN_PAGE; - let protect_memory_pages_1 = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE; // protect one chunk in the first protect - let protect_memory_pages_2 = test_memory_pages; // protect both chunks in the second protect - with_cleanup( - || { - // map 2 chunks - let mmapper = ByteMapMmapper::new(); - mmapper - .ensure_mapped( - FIXED_ADDRESS, - test_memory_pages, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - - // protect 1 chunk - mmapper.protect(FIXED_ADDRESS, protect_memory_pages_1); - - let chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Protected - ); - assert_eq!( - mmapper.mapped[chunk + 1].load(Ordering::Relaxed), - MapState::Mapped - ); - - // ensure mapped - this will unprotect the previously protected chunk - mmapper - .ensure_mapped( - FIXED_ADDRESS, - protect_memory_pages_2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Mapped - ); - assert_eq!( - mmapper.mapped[chunk + 1].load(Ordering::Relaxed), - MapState::Mapped - ); - }, - || { - memory::munmap(FIXED_ADDRESS, test_memory_bytes).unwrap(); - }, - ) - }) - } -} diff --git a/src/util/heap/layout/mmapper.rs b/src/util/heap/layout/mmapper.rs deleted file mode 100644 index 364cc0bc0d..0000000000 --- a/src/util/heap/layout/mmapper.rs +++ /dev/null @@ -1,240 +0,0 @@ -use crate::util::heap::layout::vm_layout::*; -use crate::util::memory::*; -use crate::util::rust_util::rev_group::RevisitableGroupByForIterator; -use crate::util::Address; -use atomic::{Atomic, Ordering}; -use bytemuck::NoUninit; -use std::io::Result; - -/// Generic mmap and protection functionality -pub trait Mmapper: Sync { - /// Given an address array describing the regions of virtual memory to be used - /// by MMTk, demand zero map all of them if they are not already mapped. - /// - /// Arguments: - /// * `spaceMap`: An address array containing a pairs of start and end - /// addresses for each of the regions to be mapped - fn eagerly_mmap_all_spaces(&self, space_map: &[Address]); - - /// Mark a number of pages as mapped, without making any - /// request to the operating system. Used to mark pages - /// that the VM has already mapped. - /// - /// Arguments: - /// * `start`: Address of the first page to be mapped - /// * `bytes`: Number of bytes to ensure mapped - fn mark_as_mapped(&self, start: Address, bytes: usize); - - /// Quarantine/reserve address range. We mmap from the OS with no reserve and with PROT_NONE, - /// which should be little overhead. This ensures that we can reserve certain address range that - /// we can use if needed. Quarantined memory needs to be mapped before it can be used. - /// - /// Arguments: - /// * `start`: Address of the first page to be quarantined - /// * `bytes`: Number of bytes to quarantine from the start - /// * `strategy`: The mmap strategy. The `prot` field is ignored because we always use - /// `PROT_NONE`. - /// * `anno`: Human-readable annotation to apply to newly mapped memory ranges. - fn quarantine_address_range( - &self, - start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()>; - - /// Ensure that a range of pages is mmapped (or equivalent). If the - /// pages are not yet mapped, demand-zero map them. Note that mapping - /// occurs at chunk granularity, not page granularity. - /// - /// Arguments: - /// * `start`: The start of the range to be mapped. - /// * `pages`: The size of the range to be mapped, in pages - /// * `strategy`: The mmap strategy. - /// * `anno`: Human-readable annotation to apply to newly mapped memory ranges. - // NOTE: There is a monotonicity assumption so that only updates require lock - // acquisition. - // TODO: Fix the above to support unmapping. - fn ensure_mapped( - &self, - start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()>; - - /// Is the page pointed to by this address mapped? Returns true if - /// the page at the given address is mapped. - /// - /// Arguments: - /// * `addr`: Address in question - fn is_mapped_address(&self, addr: Address) -> bool; - - /// Mark a number of pages as inaccessible. - /// - /// Arguments: - /// * `start`: Address of the first page to be protected - /// * `pages`: Number of pages to be protected - fn protect(&self, start: Address, pages: usize); -} - -/// The mmap state of a mmap chunk. -#[repr(u8)] -#[derive(Copy, Clone, PartialEq, Eq, Debug, NoUninit)] -pub(super) enum MapState { - /// The chunk is unmapped and not managed by MMTk. - Unmapped, - /// The chunk is reserved for future use. MMTk reserved the address range but hasn't used it yet. - /// We have reserved the addresss range with mmap_noreserve with PROT_NONE. - Quarantined, - /// The chunk is mapped by MMTk and is in use. - Mapped, - /// The chunk is mapped and is also protected by MMTk. - Protected, -} - -impl MapState { - /// Check the current MapState of the chunk, and transition the chunk to MapState::Mapped. - /// The caller should hold a lock before invoking this method. - pub(super) fn transition_to_mapped( - state: &Atomic, - mmap_start: Address, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - trace!( - "Trying to map {} - {}", - mmap_start, - mmap_start + MMAP_CHUNK_BYTES - ); - let res = match state.load(Ordering::Relaxed) { - MapState::Unmapped => dzmmap_noreplace(mmap_start, MMAP_CHUNK_BYTES, strategy, anno), - MapState::Protected => munprotect(mmap_start, MMAP_CHUNK_BYTES, strategy.prot), - MapState::Quarantined => unsafe { - dzmmap(mmap_start, MMAP_CHUNK_BYTES, strategy, anno) - }, - // might have become MapState::Mapped here - MapState::Mapped => Ok(()), - }; - if res.is_ok() { - state.store(MapState::Mapped, Ordering::Relaxed); - } - res - } - - /// Check the current MapState of the chunk, and transition the chunk to MapState::Quarantined. - /// The caller should hold a lock before invoking this method. - pub(super) fn transition_to_quarantined( - state: &Atomic, - mmap_start: Address, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - trace!( - "Trying to quarantine {} - {}", - mmap_start, - mmap_start + MMAP_CHUNK_BYTES - ); - let res = match state.load(Ordering::Relaxed) { - MapState::Unmapped => mmap_noreserve(mmap_start, MMAP_CHUNK_BYTES, strategy, anno), - MapState::Quarantined => Ok(()), - MapState::Mapped => { - // If a chunk is mapped by us and we try to quarantine it, we simply don't do anything. - // We allow this as it is possible to have a situation like this: - // we have global side metadata S, and space A and B. We quarantine memory X for S for A, then map - // X for A, and then we quarantine memory Y for S for B. It is possible that X and Y is the same chunk, - // so the chunk is already mapped for A, and we try quarantine it for B. We simply allow this transition. - return Ok(()); - } - MapState::Protected => panic!("Cannot quarantine protected memory"), - }; - if res.is_ok() { - state.store(MapState::Quarantined, Ordering::Relaxed); - } - res - } - - /// Equivalent to calling `transition_to_quarantined` on each element of `states`, but faster. - /// The caller should hold a lock before invoking this method. - /// - /// The memory region to transition starts from `mmap_start`. The size is the chunk size - /// multiplied by the total number of `Atomic` in the nested slice of slices. - /// - /// This function is introduced to speed up initialization of MMTk. MMTk tries to quarantine - /// very large amount of memory during start-up. If we quarantine one chunk at a time, it will - /// require thousands of `mmap` calls to cover gigabytes of quarantined memory, introducing a - /// noticeable delay. - /// - /// Arguments: - /// - /// * `state_slices`: A slice of slices. Each inner slice is a part of a `Slab`. - /// * `mmap_start`: The start of the region to transition. - /// * `strategy`: The mmap strategy. - /// * `anno`: Human-readable annotation to apply to newly mapped memory ranges. - pub(super) fn bulk_transition_to_quarantined( - state_slices: &[&[Atomic]], - mmap_start: Address, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - trace!( - "Trying to bulk-quarantine {} - {}", - mmap_start, - mmap_start + MMAP_CHUNK_BYTES * state_slices.iter().map(|s| s.len()).sum::(), - ); - - let mut start_index = 0; - - for group in state_slices - .iter() - .copied() - .flatten() - .revisitable_group_by(|s| s.load(Ordering::Relaxed)) - { - let end_index = start_index + group.len; - let start_addr = mmap_start + MMAP_CHUNK_BYTES * start_index; - let end_addr = mmap_start + MMAP_CHUNK_BYTES * end_index; - - match group.key { - MapState::Unmapped => { - trace!("Trying to quarantine {} - {}", start_addr, end_addr); - mmap_noreserve(start_addr, end_addr - start_addr, strategy, anno)?; - - for state in group { - state.store(MapState::Quarantined, Ordering::Relaxed); - } - } - MapState::Quarantined => { - trace!("Already quarantine {} - {}", start_addr, end_addr); - } - MapState::Mapped => { - trace!("Already mapped {} - {}", start_addr, end_addr); - } - MapState::Protected => { - panic!("Cannot quarantine protected memory") - } - } - - start_index = end_index; - } - - Ok(()) - } - - /// Check the current MapState of the chunk, and transition the chunk to MapState::Protected. - /// The caller should hold a lock before invoking this method. - pub(super) fn transition_to_protected( - state: &Atomic, - mmap_start: Address, - ) -> Result<()> { - match state.load(Ordering::Relaxed) { - MapState::Mapped => { - crate::util::memory::mprotect(mmap_start, MMAP_CHUNK_BYTES).unwrap(); - state.store(MapState::Protected, Ordering::Relaxed); - } - MapState::Protected => {} - _ => panic!("Cannot transition {:?} to protected", mmap_start), - } - Ok(()) - } -} diff --git a/src/util/heap/layout/mmapper/byte_map_storage.rs b/src/util/heap/layout/mmapper/byte_map_storage.rs new file mode 100644 index 0000000000..b912796019 --- /dev/null +++ b/src/util/heap/layout/mmapper/byte_map_storage.rs @@ -0,0 +1,140 @@ +use super::MapState; +use crate::util::conversions::raw_is_aligned; +use crate::util::heap::layout::mmapper::MapStateStorage; +use crate::util::rust_util::rev_group::RevisitableGroupByForIterator; +use crate::util::Address; + +use crate::util::constants::*; +use crate::util::heap::layout::vm_layout::*; +use std::fmt; +use std::sync::atomic::Ordering; +use std::sync::Mutex; + +use atomic::Atomic; +use std::io::Result; + +const MMAP_NUM_CHUNKS: usize = if LOG_BYTES_IN_ADDRESS_SPACE == 32 { + 1 << (LOG_BYTES_IN_ADDRESS_SPACE as usize - LOG_MMAP_CHUNK_BYTES) +} else { + 1 << (33 - LOG_MMAP_CHUNK_BYTES) +}; +pub const VERBOSE: bool = true; + +pub struct ByteMapStateStorage { + lock: Mutex<()>, + mapped: [Atomic; MMAP_NUM_CHUNKS], +} + +impl fmt::Debug for ByteMapStateStorage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ByteMapStateStorage({})", MMAP_NUM_CHUNKS) + } +} + +impl MapStateStorage for ByteMapStateStorage { + fn get_state(&self, chunk: Address) -> Option { + let index = chunk >> LOG_BYTES_IN_CHUNK; + let slot = self.mapped.get(index)?; + Some(slot.load(Ordering::SeqCst)) + } + + // fn set_state(&self, chunk: Address, state: MapState) { + // let index = chunk >> LOG_BYTES_IN_CHUNK; + // let Some(slot) = self.mapped.get(index) else { + // panic!("Chunk {chunk} out of range."); + // }; + // slot.store(state, Ordering::SeqCst); + // } + + fn bulk_set_state(&self, start: Address, bytes: usize, state: MapState) { + debug_assert!( + start.is_aligned_to(BYTES_IN_CHUNK), + "start {start} is not aligned" + ); + debug_assert!( + raw_is_aligned(bytes, BYTES_IN_CHUNK), + "bytes {bytes} is not aligned" + ); + let index_start = start >> LOG_BYTES_IN_CHUNK; + let index_limit = (start + bytes) >> LOG_BYTES_IN_CHUNK; + if index_start >= self.mapped.len() { + panic!("chunk {start} out of range"); + } + if index_limit >= self.mapped.len() { + panic!("bytes {bytes} out of range"); + } + for index in index_start..index_limit { + self.mapped[index].store(state, Ordering::Relaxed); + } + } + + fn bulk_transition_state(&self, start: Address, bytes: usize, mut transformer: F) -> Result<()> + where + F: FnMut(Address, usize, MapState) -> Result>, + { + debug_assert!( + start.is_aligned_to(BYTES_IN_CHUNK), + "start {start} is not aligned" + ); + debug_assert!( + raw_is_aligned(bytes, BYTES_IN_CHUNK), + "bytes {bytes} is not aligned" + ); + let index_start = start >> LOG_BYTES_IN_CHUNK; + let index_limit = (start + bytes) >> LOG_BYTES_IN_CHUNK; + if index_start >= self.mapped.len() { + panic!("start {start} out of range"); + } + if index_limit >= self.mapped.len() { + panic!("bytes {bytes} out of range"); + } + + let mut group_start = index_start; + for group in self.mapped.as_slice()[index_start..index_limit] + .iter() + .revisitable_group_by(|s| s.load(Ordering::SeqCst)) + { + let state = group.key; + let group_end = group_start + group.len; + let group_start_addr = + unsafe { Address::from_usize(group_start << LOG_BYTES_IN_CHUNK) }; + let group_bytes = group.len << LOG_BYTES_IN_CHUNK; + if let Some(new_state) = transformer(group_start_addr, group_bytes, state)? { + for index in group_start..group_end { + self.mapped[index].store(new_state, Ordering::Relaxed); + } + } + group_start = group_end; + } + + Ok(()) + } +} + +impl ByteMapStateStorage { + pub fn new() -> Self { + // Because AtomicU8 does not implement Copy, it is a compilation error to usen the + // expression `[Atomic::new(MapState::Unmapped); MMAP_NUM_CHUNKS]` because that involves + // copying. We must define a constant for it. + // + // TODO: Use the inline const expression `const { Atomic::new(MapState::Unmapped) }` after + // we bump MSRV to 1.79. + + // If we declare a const Atomic, Clippy will warn about const items being interior mutable. + // Using inline const expression will eliminate this warning, but that is experimental until + // 1.79. Fix it after we bump MSRV. + #[allow(clippy::declare_interior_mutable_const)] + const INITIAL_ENTRY: Atomic = Atomic::new(MapState::Unmapped); + + ByteMapStateStorage { + lock: Mutex::new(()), + mapped: [INITIAL_ENTRY; MMAP_NUM_CHUNKS], + } + } +} + +impl Default for ByteMapStateStorage { + fn default() -> Self { + Self::new() + } +} diff --git a/src/util/heap/layout/mmapper/mod.rs b/src/util/heap/layout/mmapper/mod.rs new file mode 100644 index 0000000000..12b6186b69 --- /dev/null +++ b/src/util/heap/layout/mmapper/mod.rs @@ -0,0 +1,435 @@ +use crate::util::memory::*; +use crate::util::Address; +use crate::util::{constants::BYTES_IN_PAGE, heap::layout::vm_layout::*}; +use bytemuck::NoUninit; +use std::{io::Result, sync::Mutex}; + +mod byte_map_storage; +#[cfg(target_pointer_width = "64")] +mod two_level_storage; + +#[cfg(target_pointer_width = "32")] +type ChosenMapStateStorage = byte_map_storage::ByteMapStateStorage; +#[cfg(target_pointer_width = "64")] +type ChosenMapStateStorage = two_level_storage::TwoLevelStateStorage; + +trait MapStateStorage { + fn get_state(&self, chunk: Address) -> Option; + // fn set_state(&self, chunk: Address, state: MapState); + fn bulk_set_state(&self, start: Address, bytes: usize, state: MapState); + fn bulk_transition_state(&self, start: Address, bytes: usize, transformer: F) -> Result<()> + where + F: FnMut(Address, usize, MapState) -> Result>; +} + +pub struct Mmapper { + /// Lock for transitioning map states. + transition_lock: Mutex<()>, + /// This holds the [`MapState`] for each chunk. + storage: ChosenMapStateStorage, +} + +/// Generic mmap and protection functionality +impl Mmapper { + pub fn new() -> Self { + Self { + transition_lock: Default::default(), + storage: ChosenMapStateStorage::new(), + } + } + + /// Given an address array describing the regions of virtual memory to be used + /// by MMTk, demand zero map all of them if they are not already mapped. + /// + /// Arguments: + /// * `spaceMap`: An address array containing a pairs of start and end + /// addresses for each of the regions to be mapped + pub fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]) { + unimplemented!() + } + + /// Mark a number of pages as mapped, without making any + /// request to the operating system. Used to mark pages + /// that the VM has already mapped. + /// + /// Arguments: + /// * `start`: Address of the first page to be mapped + /// * `bytes`: Number of bytes to ensure mapped + pub fn mark_as_mapped(&self, start: Address, bytes: usize) { + let _guard = self.transition_lock.lock().unwrap(); + + let chunk_start: Address = start.align_down(BYTES_IN_CHUNK); + let chunk_end = (start + bytes).align_up(BYTES_IN_CHUNK); + let aligned_bytes = chunk_end - chunk_start; + self.storage + .bulk_set_state(chunk_start, aligned_bytes, MapState::Mapped); + } + + /// Quarantine/reserve address range. We mmap from the OS with no reserve and with PROT_NONE, + /// which should be little overhead. This ensures that we can reserve certain address range that + /// we can use if needed. Quarantined memory needs to be mapped before it can be used. + /// + /// Arguments: + /// * `start`: Address of the first page to be quarantined + /// * `bytes`: Number of bytes to quarantine from the start + /// * `strategy`: The mmap strategy. The `prot` field is ignored because we always use + /// `PROT_NONE`. + /// * `anno`: Human-readable annotation to apply to newly mapped memory ranges. + pub fn quarantine_address_range( + &self, + start: Address, + pages: usize, + strategy: MmapStrategy, + anno: &MmapAnnotation, + ) -> Result<()> { + let _guard = self.transition_lock.lock().unwrap(); + + let chunk_start: Address = start.align_down(BYTES_IN_CHUNK); + let chunk_end = (start + pages * BYTES_IN_PAGE).align_up(BYTES_IN_CHUNK); + let aligned_bytes = chunk_end - chunk_start; + self.storage.bulk_transition_state( + chunk_start, + aligned_bytes, + |group_start, group_bytes, state| { + let group_end = group_start + group_bytes; + + match state { + MapState::Unmapped => { + trace!("Trying to quarantine {} - {}", group_start, group_end); + mmap_noreserve(group_start, group_bytes, strategy, anno)?; + Ok(Some(MapState::Quarantined)) + } + MapState::Quarantined => { + trace!("Already quarantine {} - {}", group_start, group_end); + Ok(None) + } + MapState::Mapped => { + trace!("Already mapped {} - {}", group_start, group_end); + Ok(None) + } + MapState::Protected => { + panic!("Cannot quarantine protected memory") + } + } + }, + ) + } + + /// Ensure that a range of pages is mmapped (or equivalent). If the + /// pages are not yet mapped, demand-zero map them. Note that mapping + /// occurs at chunk granularity, not page granularity. + /// + /// Arguments: + /// * `start`: The start of the range to be mapped. + /// * `pages`: The size of the range to be mapped, in pages + /// * `strategy`: The mmap strategy. + /// * `anno`: Human-readable annotation to apply to newly mapped memory ranges. + // NOTE: There is a monotonicity assumption so that only updates require lock + // acquisition. + // TODO: Fix the above to support unmapping. + pub fn ensure_mapped( + &self, + start: Address, + pages: usize, + strategy: MmapStrategy, + anno: &MmapAnnotation, + ) -> Result<()> { + let _guard = self.transition_lock.lock().unwrap(); + + let chunk_start: Address = start.align_down(BYTES_IN_CHUNK); + let chunk_end = (start + pages * BYTES_IN_PAGE).align_up(BYTES_IN_CHUNK); + let aligned_bytes = chunk_end - chunk_start; + self.storage.bulk_transition_state( + chunk_start, + aligned_bytes, + |group_start, group_bytes, state| match state { + MapState::Unmapped => { + dzmmap_noreplace(group_start, group_bytes, strategy, anno)?; + Ok(Some(MapState::Mapped)) + } + MapState::Protected => { + munprotect(group_start, group_bytes, strategy.prot)?; + Ok(Some(MapState::Mapped)) + } + MapState::Quarantined => { + unsafe { dzmmap(group_start, group_bytes, strategy, anno) }?; + Ok(Some(MapState::Mapped)) + } + MapState::Mapped => Ok(None), + }, + ) + } + + /// Is the page pointed to by this address mapped? Returns true if + /// the page at the given address is mapped. + /// + /// Arguments: + /// * `addr`: Address in question + pub fn is_mapped_address(&self, addr: Address) -> bool { + self.storage.get_state(addr) == Some(MapState::Mapped) + } + + /// Mark a number of pages as inaccessible. + /// + /// Arguments: + /// * `start`: Address of the first page to be protected + /// * `pages`: Number of pages to be protected + pub fn protect(&self, start: Address, pages: usize) { + let _guard = self.transition_lock.lock().unwrap(); + + let chunk_start: Address = start.align_down(BYTES_IN_CHUNK); + let chunk_end = (start + pages * BYTES_IN_PAGE).align_up(BYTES_IN_CHUNK); + let aligned_bytes = chunk_end - chunk_start; + self.storage + .bulk_transition_state( + chunk_start, + aligned_bytes, + |group_start, group_bytes, state| { + let group_end = group_start + group_bytes; + + match state { + MapState::Mapped => { + crate::util::memory::mprotect(group_start, group_bytes).unwrap(); + Ok(Some(MapState::Protected)) + } + MapState::Protected => Ok(None), + _ => panic!( + "Cannot transition {}-{} to protected", + group_start, group_end + ), + } + }, + ) + .unwrap(); + } + + #[cfg(test)] + fn get_state(&self, chunk: Address) -> Option { + self.storage.get_state(chunk) + } +} + +/// The mmap state of a mmap chunk. +#[repr(u8)] +#[derive(Copy, Clone, PartialEq, Eq, Debug, NoUninit)] +enum MapState { + /// The chunk is unmapped and not managed by MMTk. + Unmapped, + /// The chunk is reserved for future use. MMTk reserved the address range but hasn't used it yet. + /// We have reserved the addresss range with mmap_noreserve with PROT_NONE. + Quarantined, + /// The chunk is mapped by MMTk and is in use. + Mapped, + /// The chunk is mapped and is also protected by MMTk. + Protected, +} + + +#[cfg(test)] +mod tests { + use super::*; + use crate::mmap_anno_test; + use crate::util::constants::LOG_BYTES_IN_PAGE; + use crate::util::heap::layout::vm_layout::MMAP_CHUNK_BYTES; + use crate::util::memory; + use crate::util::test_util::FRAGMENTED_MMAPPER_TEST_REGION; + use crate::util::test_util::{serial_test, with_cleanup}; + use crate::util::{conversions, Address}; + + const FIXED_ADDRESS: Address = FRAGMENTED_MMAPPER_TEST_REGION.start; + const MAX_BYTES: usize = FRAGMENTED_MMAPPER_TEST_REGION.size; + + fn pages_to_chunks_up(pages: usize) -> usize { + conversions::raw_align_up(pages, MMAP_CHUNK_BYTES) / MMAP_CHUNK_BYTES + } + + fn get_chunk_map_state(mmapper: &Mmapper, chunk: Address) -> Option { + assert_eq!(conversions::mmap_chunk_align_up(chunk), chunk); + mmapper.get_state(chunk) + } + + #[test] + fn ensure_mapped_1page() { + serial_test(|| { + let pages = 1; + with_cleanup( + || { + let mmapper = Mmapper::new(); + mmapper + .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .unwrap(); + + let chunks = pages_to_chunks_up(pages); + for i in 0..chunks { + assert_eq!( + get_chunk_map_state( + &mmapper, + FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) + ), + Some(MapState::Mapped) + ); + } + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + #[test] + fn ensure_mapped_1chunk() { + serial_test(|| { + let pages = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; + with_cleanup( + || { + let mmapper = Mmapper::new(); + mmapper + .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .unwrap(); + + let chunks = pages_to_chunks_up(pages); + for i in 0..chunks { + assert_eq!( + get_chunk_map_state( + &mmapper, + FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) + ), + Some(MapState::Mapped) + ); + } + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + + #[test] + fn ensure_mapped_more_than_1chunk() { + serial_test(|| { + let pages = (MMAP_CHUNK_BYTES + MMAP_CHUNK_BYTES / 2) >> LOG_BYTES_IN_PAGE as usize; + with_cleanup( + || { + let mmapper = Mmapper::new(); + mmapper + .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .unwrap(); + + let chunks = pages_to_chunks_up(pages); + for i in 0..chunks { + assert_eq!( + get_chunk_map_state( + &mmapper, + FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) + ), + Some(MapState::Mapped) + ); + } + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + + #[test] + fn protect() { + serial_test(|| { + with_cleanup( + || { + // map 2 chunks + let mmapper = Mmapper::new(); + let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; + mmapper + .ensure_mapped( + FIXED_ADDRESS, + pages_per_chunk * 2, + MmapStrategy::TEST, + mmap_anno_test!(), + ) + .unwrap(); + + // protect 1 chunk + mmapper.protect(FIXED_ADDRESS, pages_per_chunk); + + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + Some(MapState::Protected) + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), + Some(MapState::Mapped) + ); + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + + #[test] + fn ensure_mapped_on_protected_chunks() { + serial_test(|| { + with_cleanup( + || { + // map 2 chunks + let mmapper = Mmapper::new(); + let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; + mmapper + .ensure_mapped( + FIXED_ADDRESS, + pages_per_chunk * 2, + MmapStrategy::TEST, + mmap_anno_test!(), + ) + .unwrap(); + + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + Some(MapState::Mapped) + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), + Some(MapState::Mapped) + ); + + // protect 1 chunk + mmapper.protect(FIXED_ADDRESS, pages_per_chunk); + + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + Some(MapState::Protected) + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), + Some(MapState::Mapped) + ); + + // ensure mapped - this will unprotect the previously protected chunk + mmapper + .ensure_mapped( + FIXED_ADDRESS, + pages_per_chunk * 2, + MmapStrategy::TEST, + mmap_anno_test!(), + ) + .unwrap(); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + Some(MapState::Mapped) + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), + Some(MapState::Mapped) + ); + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } +} diff --git a/src/util/heap/layout/mmapper/two_level_storage.rs b/src/util/heap/layout/mmapper/two_level_storage.rs new file mode 100644 index 0000000000..b200e9a6f8 --- /dev/null +++ b/src/util/heap/layout/mmapper/two_level_storage.rs @@ -0,0 +1,233 @@ +//! This module contains [`TwoLevelStateStorage`], an implementation of [`MapStateStorage`] that is +//! designed to work well on 64-bit machines. Currently it supports 48-bit address spaces, and many +//! constants and data structures (such as [`Slab`]) are larger than `i32::MAX`. For this reason, +//! this module is only available on 64-bit machines. + +use super::MapState; +use crate::util::conversions::raw_is_aligned; +use crate::util::heap::layout::mmapper::MapStateStorage; +use crate::util::heap::layout::vm_layout::*; +use crate::util::rust_util::atomic_box::OnceOptionBox; +use crate::util::rust_util::rev_group::RevisitableGroupByForIterator; +use crate::util::Address; +use atomic::{Atomic, Ordering}; +use std::fmt; +use std::io::Result; + +/// Logarithm of the address space size a user-space program is allowed to use. +/// This is enough for ARM64, x86_64 and some other architectures. +/// Feel free to increase it if we plan to support larger address spaces. +const LOG_MAPPABLE_BYTES: usize = 48; +/// Address space size a user-space program is allowed to use. +const MAPPABLE_BYTES: usize = 1 << LOG_MAPPABLE_BYTES; + +/// Log number of bytes per slab. +/// For a two-level array, it is advisable to choose the arithmetic mean of [`LOG_MAPPABLE_BYTES`] +/// and [`LOG_MMAP_CHUNK_BYTES`] in order to make [`MMAP_SLAB_BYTES`] the geometric mean of +/// [`MAPPABLE_BYTES`] and [`MMAP_CHUNK_BYTES`]. This will balance the array size of +/// [`TwoLevelStateStorage::slabs`] and [`Slab`]. +/// +/// TODO: Use `usize::midpoint` after bumping MSRV to 1.85 +const LOG_MMAP_SLAB_BYTES: usize = + LOG_MMAP_CHUNK_BYTES + (LOG_MAPPABLE_BYTES - LOG_MMAP_CHUNK_BYTES) / 2; +/// Number of bytes per slab. +const MMAP_SLAB_BYTES: usize = 1 << LOG_MMAP_SLAB_BYTES; + +/// Log number of chunks per slab. +const LOG_MMAP_CHUNKS_PER_SLAB: usize = LOG_MMAP_SLAB_BYTES - LOG_MMAP_CHUNK_BYTES; +/// Number of chunks per slab. +const MMAP_CHUNKS_PER_SLAB: usize = 1 << LOG_MMAP_CHUNKS_PER_SLAB; + +/// Mask for getting in-slab bits from an address. +/// Invert this to get out-of-slab bits. +const MMAP_SLAB_MASK: usize = (1 << LOG_MMAP_SLAB_BYTES) - 1; + +/// Logarithm of maximum number of slabs. +const LOG_MAX_SLABS: usize = LOG_MAPPABLE_BYTES - LOG_MMAP_SLAB_BYTES; +/// maximum number of slabs. +const MAX_SLABS: usize = 1 << LOG_MAX_SLABS; + +/// The slab type. Each slab holds the `MapState` of multiple chunks. +type Slab = [Atomic; MMAP_CHUNKS_PER_SLAB]; + +/// A two-level implementation of `MapStateStorage`. +/// +/// It is essentially a lazily initialized array of [`Atomic`]. Because it is designed to +/// govern a large address range, and the array is sparse, we use a two-level design. The higher +/// level holds a vector of slabs, and each slab holds an array of [`Atomic`]. Each slab +/// governs an aligned region of [`MMAP_CHUNKS_PER_SLAB`] chunks. Slabs are lazily created when the +/// user intends to write into one of its `MapState`. +pub struct TwoLevelStateStorage { + /// Slabs + slabs: Vec>, +} + +unsafe impl Send for TwoLevelStateStorage {} +unsafe impl Sync for TwoLevelStateStorage {} + +impl fmt::Debug for TwoLevelStateStorage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "TwoLevelMapper({})", 1 << LOG_MAX_SLABS) + } +} + +impl MapStateStorage for TwoLevelStateStorage { + fn get_state(&self, chunk: Address) -> Option { + self.slab_table(chunk) + .map(|slab| slab[Self::in_slab_index(chunk)].load(Ordering::Relaxed)) + } + + fn bulk_set_state(&self, start: Address, bytes: usize, state: MapState) { + self.foreach_slab_slice_for_write(start, bytes, |_low, _high, slice| { + for slot in slice { + slot.store(state, Ordering::Relaxed); + } + }); + } + + fn bulk_transition_state( + &self, + start: Address, + bytes: usize, + mut transformer: F, + ) -> Result<()> + where + F: FnMut(Address, usize, MapState) -> Result>, + { + let mut slice_indices = Vec::new(); + + self.foreach_slab_slice_for_write(start, bytes, |_low, _high, slice| { + slice_indices.push(slice); + }); + + // Chunk index from `start`. + let mut start_index: usize = 0usize; + + for group in slice_indices + .iter() + .copied() + .flatten() + .revisitable_group_by(|s| s.load(Ordering::Relaxed)) + { + let state = group.key; + let end_index = start_index + group.len; + let start_addr = start + MMAP_CHUNK_BYTES * start_index; + let group_bytes = group.len << LOG_MMAP_CHUNK_BYTES; + + if let Some(new_state) = transformer(start_addr, group_bytes, state)? { + for slot in group { + slot.store(new_state, Ordering::Relaxed); + } + } + + start_index = end_index; + } + + Ok(()) + } +} + +impl TwoLevelStateStorage { + pub fn new() -> Self { + Self { + slabs: unsafe { crate::util::rust_util::zeroed_alloc::new_zeroed_vec(MAX_SLABS) }, + } + } + + fn new_slab() -> Slab { + std::array::from_fn(|_| Atomic::new(MapState::Unmapped)) + } + + fn slab_table(&self, addr: Address) -> Option<&Slab> { + let index: usize = Self::slab_index(addr); + let slot = self.slabs.get(index)?; + // Note: We don't need acquire here. See `get_or_allocate_slab_table`. + slot.get(Ordering::Relaxed) + } + + fn get_or_allocate_slab_table(&self, addr: Address) -> &Slab { + let index: usize = Self::slab_index(addr); + let Some(slot) = self.slabs.get(index) else { + panic!("Cannot allocate slab for address: {addr}"); + }; + // Note: We set both order_load and order_store to `Relaxed` because we never populate the + // content of the slab before making the `OnceOptionBox` point to the new slab. For this + // reason, the release-acquire relation is not needed here. + slot.get_or_init(Ordering::Relaxed, Ordering::Relaxed, Self::new_slab) + } + + fn slab_index(addr: Address) -> usize { + addr >> LOG_MMAP_SLAB_BYTES + } + + fn in_slab_index(addr: Address) -> usize { + (addr & MMAP_SLAB_MASK) >> LOG_BYTES_IN_CHUNK + } + + fn foreach_slab_slice_for_write<'s, F>(&'s self, start: Address, bytes: usize, mut f: F) + where + F: FnMut(Address, Address, &'s [Atomic]), + { + debug_assert!( + start.is_aligned_to(BYTES_IN_CHUNK), + "start {start} is not aligned" + ); + debug_assert!( + raw_is_aligned(bytes, BYTES_IN_CHUNK), + "bytes {bytes} is not aligned" + ); + if start.as_usize() >= MAPPABLE_BYTES { + panic!("start {start} out of range"); + } + let limit = start + bytes; + if limit.as_usize() >= MAPPABLE_BYTES { + panic!("bytes {bytes} out of range"); + } + + let mut low = start; + while low < limit { + let high = (low + MMAP_SLAB_BYTES) + .align_down(MMAP_SLAB_BYTES) + .min(limit); + + let slab = self.get_or_allocate_slab_table(low); + let low_index = Self::in_slab_index(low); + let high_index = Self::in_slab_index(high); + let ub_index = if high_index == 0 { + MMAP_CHUNKS_PER_SLAB + } else { + high_index + }; + f(low, high, &slab[low_index..ub_index]); + + low = high; + } + } + + fn chunk_index_to_address(base: Address, chunk: usize) -> Address { + base + (chunk << LOG_MMAP_CHUNK_BYTES) + } + + /// Align `addr` down to slab size. + fn slab_align_down(addr: Address) -> Address { + addr.align_down(MMAP_SLAB_BYTES) + } + + /// Get the base address of the next slab after the slab that contains `addr`. + fn slab_limit(addr: Address) -> Address { + Self::slab_align_down(addr) + MMAP_SLAB_BYTES + } + + /// Return the index of the chunk that contains `addr` within the slab starting at `slab`. + /// If `addr` is beyond the end of the slab, the result could be beyond the end of the slab. + fn chunk_index(slab: Address, addr: Address) -> usize { + let delta = addr - slab; + delta >> LOG_MMAP_CHUNK_BYTES + } +} + +impl Default for TwoLevelStateStorage { + fn default() -> Self { + Self::new() + } +} diff --git a/src/util/heap/layout/mod.rs b/src/util/heap/layout/mod.rs index 5412c280ea..9615d34a23 100644 --- a/src/util/heap/layout/mod.rs +++ b/src/util/heap/layout/mod.rs @@ -3,9 +3,6 @@ pub mod vm_layout; mod mmapper; pub use self::mmapper::Mmapper; -mod byte_map_mmapper; -#[cfg(target_pointer_width = "64")] -mod two_level_mmapper; mod map; pub(crate) use self::map::CreateFreeListResult; @@ -35,9 +32,9 @@ pub fn create_mmapper() -> Box { } #[cfg(target_pointer_width = "64")] -pub fn create_mmapper() -> Box { +pub fn create_mmapper() -> Box { // TODO: ByteMapMmapper for 39-bit or less virtual space - Box::new(two_level_mmapper::TwoLevelMmapper::new()) + Box::new(Mmapper::new()) } use crate::util::Address; diff --git a/src/util/heap/layout/two_level_mmapper.rs b/src/util/heap/layout/two_level_mmapper.rs deleted file mode 100644 index a6e609570c..0000000000 --- a/src/util/heap/layout/two_level_mmapper.rs +++ /dev/null @@ -1,507 +0,0 @@ -//! This module contains [`TwoLevelMmapper`], an implementation of [`Mmapper`] that is designed to -//! work well on 64-bit machines. Currently it supports 48-bit address spaces, and many constants -//! and data structures (such as [`Slab`]) are larger than `i32::MAX`. For this reason, this module -//! is only available on 64-bit machines. - -use super::mmapper::MapState; -use super::Mmapper; -use crate::util::constants::BYTES_IN_PAGE; -use crate::util::conversions; -use crate::util::heap::layout::vm_layout::*; -use crate::util::memory::{MmapAnnotation, MmapStrategy}; -use crate::util::rust_util::atomic_box::OnceOptionBox; -use crate::util::Address; -use atomic::{Atomic, Ordering}; -use std::fmt; -use std::io::Result; -use std::sync::Mutex; - -/// Logarithm of the address space size a user-space program is allowed to use. -/// This is enough for ARM64, x86_64 and some other architectures. -/// Feel free to increase it if we plan to support larger address spaces. -const LOG_MAPPABLE_BYTES: usize = 48; -/// Address space size a user-space program is allowed to use. -const MAPPABLE_BYTES: usize = 1 << LOG_MAPPABLE_BYTES; - -/// Log number of bytes per slab. -/// For a two-level array, it is advisable to choose the arithmetic mean of [`LOG_MAPPABLE_BYTES`] -/// and [`LOG_MMAP_CHUNK_BYTES`] in order to make [`MMAP_SLAB_BYTES`] the geometric mean of -/// [`MAPPABLE_BYTES`] and [`MMAP_CHUNK_BYTES`]. This will balance the array size of -/// [`TwoLevelMmapper::slabs`] and [`Slab`]. -/// -/// TODO: Use `usize::midpoint` after bumping MSRV to 1.85 -const LOG_MMAP_SLAB_BYTES: usize = - LOG_MMAP_CHUNK_BYTES + (LOG_MAPPABLE_BYTES - LOG_MMAP_CHUNK_BYTES) / 2; -/// Number of bytes per slab. -const MMAP_SLAB_BYTES: usize = 1 << LOG_MMAP_SLAB_BYTES; - -/// Log number of chunks per slab. -const LOG_MMAP_CHUNKS_PER_SLAB: usize = LOG_MMAP_SLAB_BYTES - LOG_MMAP_CHUNK_BYTES; -/// Number of chunks per slab. -const MMAP_CHUNKS_PER_SLAB: usize = 1 << LOG_MMAP_CHUNKS_PER_SLAB; - -/// Mask for getting in-slab bits from an address. -/// Invert this to get out-of-slab bits. -const MMAP_SLAB_MASK: usize = (1 << LOG_MMAP_SLAB_BYTES) - 1; - -/// Logarithm of maximum number of slabs. -const LOG_MAX_SLABS: usize = LOG_MAPPABLE_BYTES - LOG_MMAP_SLAB_BYTES; -/// maximum number of slabs. -const MAX_SLABS: usize = 1 << LOG_MAX_SLABS; - -/// The slab type. Each slab holds the `MapState` of multiple chunks. -type Slab = [Atomic; MMAP_CHUNKS_PER_SLAB]; - -/// A two-level implementation of `Mmapper`. -/// -/// It is essentially a lazily initialized array of [`Atomic`]. Because it is designed to -/// govern a large address range, and the array is sparse, we use a two-level design. The higher -/// level holds a vector of slabs, and each slab holds an array of [`Atomic`]. Each slab -/// governs an aligned region of [`MMAP_CHUNKS_PER_SLAB`] chunks. Slabs are lazily created when the -/// user intends to write into one of its `MapState`. -pub struct TwoLevelMmapper { - /// Lock for transitioning map states. - /// - /// FIXME: We only need this lock when transitioning map states. - /// The `TwoLevelMmapper` itself is completely lock-free even when allocating new slabs. - /// We should move the lock one leve above, to `MapState`. - transition_lock: Mutex<()>, - /// Slabs - slabs: Vec>, -} - -unsafe impl Send for TwoLevelMmapper {} -unsafe impl Sync for TwoLevelMmapper {} - -impl fmt::Debug for TwoLevelMmapper { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "TwoLevelMapper({})", 1 << LOG_MAX_SLABS) - } -} - -impl Mmapper for TwoLevelMmapper { - fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]) {} - - fn mark_as_mapped(&self, mut start: Address, bytes: usize) { - let end = start + bytes; - // Iterate over the slabs covered - while start < end { - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - for entry in mapped.iter().take(end_chunk).skip(start_chunk) { - entry.store(MapState::Mapped, Ordering::Relaxed); - } - start = high; - } - } - - fn quarantine_address_range( - &self, - mut start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - debug_assert!(start.is_aligned_to(BYTES_IN_PAGE)); - - let end = start + conversions::pages_to_bytes(pages); - - // Each `MapState` entry governs a chunk. - // Align down to the chunk start because we only mmap multiples of whole chunks. - let mmap_start = conversions::mmap_chunk_align_down(start); - - // We collect the chunk states from slabs to process them in bulk. - let mut state_slices = vec![]; - - // Iterate over the slabs covered - while start < end { - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - state_slices.push(&mapped[start_chunk..end_chunk]); - - start = high; - } - - #[cfg(debug_assertions)] - { - // Check if the number of entries are normal. - let mmap_end = conversions::mmap_chunk_align_up(end); - let num_slices = state_slices.iter().map(|s| s.len()).sum::(); - - debug_assert_eq!(mmap_start + BYTES_IN_CHUNK * num_slices, mmap_end); - } - - // Transition the chunks in bulk. - { - let _guard = self.transition_lock.lock().unwrap(); - MapState::bulk_transition_to_quarantined( - state_slices.as_slice(), - mmap_start, - strategy, - anno, - )?; - } - - Ok(()) - } - - fn ensure_mapped( - &self, - mut start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - let end = start + conversions::pages_to_bytes(pages); - // Iterate over the slabs covered - while start < end { - let base = Self::slab_align_down(start); - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - - /* Iterate over the chunks within the slab */ - for (chunk, entry) in mapped.iter().enumerate().take(end_chunk).skip(start_chunk) { - if matches!(entry.load(Ordering::Relaxed), MapState::Mapped) { - continue; - } - - let mmap_start = Self::chunk_index_to_address(base, chunk); - let _guard = self.transition_lock.lock().unwrap(); - MapState::transition_to_mapped(entry, mmap_start, strategy, anno)?; - } - start = high; - } - Ok(()) - } - - /** - * Return {@code true} if the given address has been mmapped - * - * @param addr The address in question. - * @return {@code true} if the given address has been mmapped - */ - fn is_mapped_address(&self, addr: Address) -> bool { - let mapped = self.slab_table(addr); - match mapped { - Some(mapped) => { - mapped[Self::chunk_index(Self::slab_align_down(addr), addr)].load(Ordering::Relaxed) - == MapState::Mapped - } - _ => false, - } - } - - fn protect(&self, mut start: Address, pages: usize) { - let end = start + conversions::pages_to_bytes(pages); - let _guard = self.transition_lock.lock().unwrap(); - // Iterate over the slabs covered - while start < end { - let base = Self::slab_align_down(start); - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - - for (chunk, entry) in mapped.iter().enumerate().take(end_chunk).skip(start_chunk) { - let mmap_start = Self::chunk_index_to_address(base, chunk); - MapState::transition_to_protected(entry, mmap_start).unwrap(); - } - start = high; - } - } -} - -impl TwoLevelMmapper { - pub fn new() -> Self { - Self { - transition_lock: Default::default(), - slabs: unsafe { crate::util::rust_util::zeroed_alloc::new_zeroed_vec(MAX_SLABS) }, - } - } - - fn new_slab() -> Slab { - std::array::from_fn(|_| Atomic::new(MapState::Unmapped)) - } - - fn slab_table(&self, addr: Address) -> Option<&Slab> { - let index: usize = addr >> LOG_MMAP_SLAB_BYTES; - let slot = self.slabs.get(index)?; - // Note: We don't need acquire here. See `get_or_allocate_slab_table`. - slot.get(Ordering::Relaxed) - } - - fn get_or_allocate_slab_table(&self, addr: Address) -> &Slab { - let index: usize = addr >> LOG_MMAP_SLAB_BYTES; - let Some(slot) = self.slabs.get(index) else { - panic!("Cannot allocate slab for address: {addr}"); - }; - // Note: We set both order_load and order_store to `Relaxed` because we never populate the - // content of the slab before making the `OnceOptionBox` point to the new slab. For this - // reason, the release-acquire relation is not needed here. - slot.get_or_init(Ordering::Relaxed, Ordering::Relaxed, Self::new_slab) - } - - fn chunk_index_to_address(base: Address, chunk: usize) -> Address { - base + (chunk << LOG_MMAP_CHUNK_BYTES) - } - - /// Align `addr` down to slab size. - fn slab_align_down(addr: Address) -> Address { - addr.align_down(MMAP_SLAB_BYTES) - } - - /// Get the base address of the next slab after the slab that contains `addr`. - fn slab_limit(addr: Address) -> Address { - Self::slab_align_down(addr) + MMAP_SLAB_BYTES - } - - /// Return the index of the chunk that contains `addr` within the slab starting at `slab`. - /// If `addr` is beyond the end of the slab, the result could be beyond the end of the slab. - fn chunk_index(slab: Address, addr: Address) -> usize { - let delta = addr - slab; - delta >> LOG_MMAP_CHUNK_BYTES - } -} - -impl Default for TwoLevelMmapper { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::mmap_anno_test; - use crate::util::constants::LOG_BYTES_IN_PAGE; - use crate::util::heap::layout::vm_layout::MMAP_CHUNK_BYTES; - use crate::util::memory; - use crate::util::test_util::FRAGMENTED_MMAPPER_TEST_REGION; - use crate::util::test_util::{serial_test, with_cleanup}; - use crate::util::{conversions, Address}; - - const FIXED_ADDRESS: Address = FRAGMENTED_MMAPPER_TEST_REGION.start; - const MAX_BYTES: usize = FRAGMENTED_MMAPPER_TEST_REGION.size; - - fn pages_to_chunks_up(pages: usize) -> usize { - conversions::raw_align_up(pages, MMAP_CHUNK_BYTES) / MMAP_CHUNK_BYTES - } - - fn get_chunk_map_state(mmapper: &TwoLevelMmapper, chunk: Address) -> Option { - assert_eq!(conversions::mmap_chunk_align_up(chunk), chunk); - let mapped = mmapper.slab_table(chunk); - mapped.map(|m| { - m[TwoLevelMmapper::chunk_index(TwoLevelMmapper::slab_align_down(chunk), chunk)] - .load(Ordering::Relaxed) - }) - } - - #[test] - fn ensure_mapped_1page() { - serial_test(|| { - let pages = 1; - with_cleanup( - || { - let mmapper = TwoLevelMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let chunks = pages_to_chunks_up(pages); - for i in 0..chunks { - assert_eq!( - get_chunk_map_state( - &mmapper, - FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) - ), - Some(MapState::Mapped) - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - #[test] - fn ensure_mapped_1chunk() { - serial_test(|| { - let pages = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - with_cleanup( - || { - let mmapper = TwoLevelMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let chunks = pages_to_chunks_up(pages); - for i in 0..chunks { - assert_eq!( - get_chunk_map_state( - &mmapper, - FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) - ), - Some(MapState::Mapped) - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_more_than_1chunk() { - serial_test(|| { - let pages = (MMAP_CHUNK_BYTES + MMAP_CHUNK_BYTES / 2) >> LOG_BYTES_IN_PAGE as usize; - with_cleanup( - || { - let mmapper = TwoLevelMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let chunks = pages_to_chunks_up(pages); - for i in 0..chunks { - assert_eq!( - get_chunk_map_state( - &mmapper, - FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) - ), - Some(MapState::Mapped) - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - - #[test] - fn protect() { - serial_test(|| { - with_cleanup( - || { - // map 2 chunks - let mmapper = TwoLevelMmapper::new(); - let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - mmapper - .ensure_mapped( - FIXED_ADDRESS, - pages_per_chunk * 2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - - // protect 1 chunk - mmapper.protect(FIXED_ADDRESS, pages_per_chunk); - - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS), - Some(MapState::Protected) - ); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), - Some(MapState::Mapped) - ); - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_on_protected_chunks() { - serial_test(|| { - with_cleanup( - || { - // map 2 chunks - let mmapper = TwoLevelMmapper::new(); - let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - mmapper - .ensure_mapped( - FIXED_ADDRESS, - pages_per_chunk * 2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - - // protect 1 chunk - mmapper.protect(FIXED_ADDRESS, pages_per_chunk); - - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS), - Some(MapState::Protected) - ); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), - Some(MapState::Mapped) - ); - - // ensure mapped - this will unprotect the previously protected chunk - mmapper - .ensure_mapped( - FIXED_ADDRESS, - pages_per_chunk * 2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS), - Some(MapState::Mapped) - ); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), - Some(MapState::Mapped) - ); - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } -}