diff --git a/benches/mock_bench/mmapper.rs b/benches/mock_bench/mmapper.rs index d9d8214571..ba3216581f 100644 --- a/benches/mock_bench/mmapper.rs +++ b/benches/mock_bench/mmapper.rs @@ -1,8 +1,8 @@ pub use criterion::Criterion; use mmtk::{ - memory_manager, - util::{test_util::fixtures::*, Address}, + memory_manager, mmap_anno_test, + util::{constants::BYTES_IN_PAGE, memory::MmapStrategy, test_util::fixtures::*, Address}, }; pub fn bench(c: &mut Criterion) { @@ -69,4 +69,16 @@ pub fn bench(c: &mut Criterion) { } }) }); + + c.bench_function("ensure_mapped_regular", |b| { + let start = regular.align_down(BYTES_IN_PAGE); + assert!(start.is_mapped()); + let strategy = MmapStrategy::new(false, mmtk::util::memory::MmapProtection::ReadWrite); + let anno = mmap_anno_test!(); + b.iter(|| { + mmtk::MMAPPER + .ensure_mapped(start, 1, strategy, anno) + .unwrap(); + }) + }); } diff --git a/docs/userguide/src/migration/prefix.md b/docs/userguide/src/migration/prefix.md index ee405efa40..cca4364a04 100644 --- a/docs/userguide/src/migration/prefix.md +++ b/docs/userguide/src/migration/prefix.md @@ -32,6 +32,22 @@ Notes for the mmtk-core developers: ## 0.32.0 +### Removed the notion of "mmap chunk" + +```admonish tldr +Constants such as `MMAP_CHUNK_BYTES` were related to the implementation details of the memory +mapper, and should not have been exposed. +``` + +API changes: + +- module `util::conversions` + + `mmap_chunk_align_down`: Removed. + + `mmap_chunk_align_up`: Removed. +- module `util::heap::vm_layout` + + `LOG_MMAP_CHUNK_BYTES`: Removed. + + `MMAP_CHUNK_BYTES`: Removed. + ### `Options` no longer differentiates between environment variables and command line arguments. ```admonish tldr diff --git a/src/lib.rs b/src/lib.rs index afe094885f..44e155eda6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,7 @@ extern crate probe; mod mmtk; pub use mmtk::MMTKBuilder; -pub(crate) use mmtk::MMAPPER; +pub use mmtk::MMAPPER; pub use mmtk::MMTK; mod global_state; diff --git a/src/mmtk.rs b/src/mmtk.rs index 2916f98f82..571e51274d 100644 --- a/src/mmtk.rs +++ b/src/mmtk.rs @@ -48,7 +48,7 @@ lazy_static! { pub static ref VM_MAP: Box = layout::create_vm_map(); /// A global Mmapper for mmaping and protection of virtual memory. - pub static ref MMAPPER: Box = layout::create_mmapper(); + pub static ref MMAPPER: Box = layout::create_mmapper(); } use crate::util::rust_util::InitializeOnce; diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index abe7976082..33cb3230a9 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -105,12 +105,14 @@ impl SFT for LargeObjectSpace { ptr: Address, max_search_bytes: usize, ) -> Option { - use crate::util::heap::vm_layout::MMAP_CHUNK_BYTES; - use crate::util::metadata::vo_bit; + use crate::{util::metadata::vo_bit, MMAPPER}; - // We need to check if metadata address is mapped or not. But we only check at chunk granularity. - // This records the start of a chunk that is tested to be mapped. - let mut mapped_chunk = Address::MAX; + let mmap_granularity = MMAPPER.granularity(); + + // We need to check if metadata address is mapped or not. But we make use of the granularity of + // the `Mmapper` to reduce the number of checks. This records the start of a grain that is + // tested to be mapped. + let mut mapped_grain = Address::MAX; // For large object space, it is a bit special. We only need to check VO bit for each page. let mut cur_page = ptr.align_down(BYTES_IN_PAGE); @@ -118,13 +120,13 @@ impl SFT for LargeObjectSpace { .saturating_sub(max_search_bytes) .align_down(BYTES_IN_PAGE); while cur_page >= low_page { - if cur_page < mapped_chunk { + if cur_page < mapped_grain { if !cur_page.is_mapped() { // If the page start is not mapped, there can't be an object in it. return None; } // This is mapped. No need to check for this chunk. - mapped_chunk = cur_page.align_down(MMAP_CHUNK_BYTES); + mapped_grain = cur_page.align_down(mmap_granularity); } // For performance, we only check the first word which maps to the first 512 bytes in the page. // In almost all the cases, it should be sufficient. diff --git a/src/util/conversions.rs b/src/util/conversions.rs index e62dff0392..f8077ec689 100644 --- a/src/util/conversions.rs +++ b/src/util/conversions.rs @@ -27,16 +27,6 @@ pub const fn chunk_align_down(addr: Address) -> Address { addr.align_down(BYTES_IN_CHUNK) } -/// Align up an address to the nearest chunk at which granularity we mmap memory. -pub const fn mmap_chunk_align_up(addr: Address) -> Address { - addr.align_up(MMAP_CHUNK_BYTES) -} - -/// Align down an address to the nearest chunk at which granularity we mmap memory. -pub const fn mmap_chunk_align_down(addr: Address) -> Address { - addr.align_down(MMAP_CHUNK_BYTES) -} - /// Convert size in bytes to the number of chunks (aligned up). pub fn bytes_to_chunks_up(bytes: usize) -> usize { (bytes + BYTES_IN_CHUNK - 1) >> LOG_BYTES_IN_CHUNK diff --git a/src/util/heap/layout/byte_map_mmapper.rs b/src/util/heap/layout/byte_map_mmapper.rs deleted file mode 100644 index 412e53ddcb..0000000000 --- a/src/util/heap/layout/byte_map_mmapper.rs +++ /dev/null @@ -1,428 +0,0 @@ -use super::mmapper::MapState; -use super::Mmapper; -use crate::util::memory::MmapAnnotation; -use crate::util::Address; - -use crate::util::constants::*; -use crate::util::conversions::pages_to_bytes; -use crate::util::heap::layout::vm_layout::*; -use crate::util::memory::MmapStrategy; -use std::fmt; -use std::sync::atomic::Ordering; -use std::sync::Mutex; - -use atomic::Atomic; -use std::io::Result; - -const MMAP_NUM_CHUNKS: usize = if LOG_BYTES_IN_ADDRESS_SPACE == 32 { - 1 << (LOG_BYTES_IN_ADDRESS_SPACE as usize - LOG_MMAP_CHUNK_BYTES) -} else { - 1 << (33 - LOG_MMAP_CHUNK_BYTES) -}; -pub const VERBOSE: bool = true; - -pub struct ByteMapMmapper { - lock: Mutex<()>, - mapped: [Atomic; MMAP_NUM_CHUNKS], -} - -impl fmt::Debug for ByteMapMmapper { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "ByteMapMmapper({})", MMAP_NUM_CHUNKS) - } -} - -impl Mmapper for ByteMapMmapper { - fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]) { - unimplemented!() - } - - fn mark_as_mapped(&self, start: Address, bytes: usize) { - let start_chunk = Self::address_to_mmap_chunks_down(start); - let end_chunk = Self::address_to_mmap_chunks_up(start + bytes) - 1; - for i in start_chunk..=end_chunk { - self.mapped[i].store(MapState::Mapped, Ordering::Relaxed); - } - } - - fn ensure_mapped( - &self, - start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - let start_chunk = Self::address_to_mmap_chunks_down(start); - let end_chunk = Self::address_to_mmap_chunks_up(start + pages_to_bytes(pages)); - trace!( - "Calling ensure_mapped with start={:?} and {} pages, {}-{}", - start, - pages, - Self::mmap_chunks_to_address(start_chunk), - Self::mmap_chunks_to_address(end_chunk) - ); - - for chunk in start_chunk..end_chunk { - if self.mapped[chunk].load(Ordering::Relaxed) == MapState::Mapped { - continue; - } - - let mmap_start = Self::mmap_chunks_to_address(chunk); - let _guard = self.lock.lock().unwrap(); - MapState::transition_to_mapped(&self.mapped[chunk], mmap_start, strategy, anno) - .unwrap(); - } - - Ok(()) - } - - fn quarantine_address_range( - &self, - start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - let start_chunk = Self::address_to_mmap_chunks_down(start); - let end_chunk = Self::address_to_mmap_chunks_up(start + pages_to_bytes(pages)); - trace!( - "Calling quarantine_address_range with start={:?} and {} pages, {}-{}", - start, - pages, - Self::mmap_chunks_to_address(start_chunk), - Self::mmap_chunks_to_address(end_chunk) - ); - - for chunk in start_chunk..end_chunk { - if self.mapped[chunk].load(Ordering::Relaxed) == MapState::Mapped { - continue; - } - - let mmap_start = Self::mmap_chunks_to_address(chunk); - let _guard = self.lock.lock().unwrap(); - MapState::transition_to_quarantined(&self.mapped[chunk], mmap_start, strategy, anno) - .unwrap(); - } - - Ok(()) - } - - /** - * Return {@code true} if the given address has been mmapped - * - * @param addr The address in question. - * @return {@code true} if the given address has been mmapped - */ - fn is_mapped_address(&self, addr: Address) -> bool { - let chunk = Self::address_to_mmap_chunks_down(addr); - self.mapped[chunk].load(Ordering::Relaxed) == MapState::Mapped - } - - fn protect(&self, start: Address, pages: usize) { - let start_chunk = Self::address_to_mmap_chunks_down(start); - let chunks = Self::pages_to_mmap_chunks_up(pages); - let end_chunk = start_chunk + chunks; - let _guard = self.lock.lock().unwrap(); - - for chunk in start_chunk..end_chunk { - let mmap_start = Self::mmap_chunks_to_address(chunk); - MapState::transition_to_protected(&self.mapped[chunk], mmap_start).unwrap(); - } - } -} - -impl ByteMapMmapper { - pub fn new() -> Self { - // Because AtomicU8 does not implement Copy, it is a compilation error to usen the - // expression `[Atomic::new(MapState::Unmapped); MMAP_NUM_CHUNKS]` because that involves - // copying. We must define a constant for it. - // - // TODO: Use the inline const expression `const { Atomic::new(MapState::Unmapped) }` after - // we bump MSRV to 1.79. - - // If we declare a const Atomic, Clippy will warn about const items being interior mutable. - // Using inline const expression will eliminate this warning, but that is experimental until - // 1.79. Fix it after we bump MSRV. - #[allow(clippy::declare_interior_mutable_const)] - const INITIAL_ENTRY: Atomic = Atomic::new(MapState::Unmapped); - - ByteMapMmapper { - lock: Mutex::new(()), - mapped: [INITIAL_ENTRY; MMAP_NUM_CHUNKS], - } - } - - fn bytes_to_mmap_chunks_up(bytes: usize) -> usize { - (bytes + MMAP_CHUNK_BYTES - 1) >> LOG_MMAP_CHUNK_BYTES - } - - fn pages_to_mmap_chunks_up(pages: usize) -> usize { - Self::bytes_to_mmap_chunks_up(pages_to_bytes(pages)) - } - - fn address_to_mmap_chunks_down(addr: Address) -> usize { - addr >> LOG_MMAP_CHUNK_BYTES - } - - fn mmap_chunks_to_address(chunk: usize) -> Address { - unsafe { Address::from_usize(chunk << LOG_MMAP_CHUNK_BYTES) } - } - - fn address_to_mmap_chunks_up(addr: Address) -> usize { - (addr + MMAP_CHUNK_BYTES - 1) >> LOG_MMAP_CHUNK_BYTES - } -} - -impl Default for ByteMapMmapper { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::ByteMapMmapper; - use crate::mmap_anno_test; - use crate::util::heap::layout::Mmapper; - use crate::util::Address; - - use crate::util::constants::LOG_BYTES_IN_PAGE; - use crate::util::conversions::pages_to_bytes; - use crate::util::heap::layout::mmapper::MapState; - use crate::util::heap::layout::vm_layout::MMAP_CHUNK_BYTES; - use crate::util::memory::{self, MmapStrategy}; - use crate::util::test_util::BYTE_MAP_MMAPPER_TEST_REGION; - use crate::util::test_util::{serial_test, with_cleanup}; - use std::sync::atomic::Ordering; - - const CHUNK_SIZE: usize = 1 << 22; - const FIXED_ADDRESS: Address = BYTE_MAP_MMAPPER_TEST_REGION.start; - const MAX_SIZE: usize = BYTE_MAP_MMAPPER_TEST_REGION.size; - - #[test] - fn address_to_mmap_chunks() { - for i in 0..10 { - unsafe { - let start = CHUNK_SIZE * i; - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_up(Address::from_usize(start)), - i - ); - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_down(Address::from_usize(start)), - i - ); - - let middle = start + 8; - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_up(Address::from_usize(middle)), - i + 1 - ); - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_down(Address::from_usize(middle)), - i - ); - - let end = start + CHUNK_SIZE; - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_up(Address::from_usize(end)), - i + 1 - ); - assert_eq!( - ByteMapMmapper::address_to_mmap_chunks_down(Address::from_usize(end)), - i + 1 - ); - } - } - } - - #[test] - fn ensure_mapped_1page() { - serial_test(|| { - let pages = 1; - let start_chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - let end_chunk = - ByteMapMmapper::address_to_mmap_chunks_up(FIXED_ADDRESS + pages_to_bytes(pages)); - let test_memory_bytes = (end_chunk - start_chunk) * MMAP_CHUNK_BYTES; - with_cleanup( - || { - let mmapper = ByteMapMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - for chunk in start_chunk..end_chunk { - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Mapped - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, test_memory_bytes).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_1chunk() { - serial_test(|| { - let pages = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - let start_chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - let end_chunk = - ByteMapMmapper::address_to_mmap_chunks_up(FIXED_ADDRESS + pages_to_bytes(pages)); - let test_memory_bytes = (end_chunk - start_chunk) * MMAP_CHUNK_BYTES; - with_cleanup( - || { - let mmapper = ByteMapMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - for chunk in start_chunk..end_chunk { - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Mapped - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, test_memory_bytes).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_more_than_1chunk() { - serial_test(|| { - let pages = (MMAP_CHUNK_BYTES + MMAP_CHUNK_BYTES / 2) >> LOG_BYTES_IN_PAGE as usize; - let start_chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - let end_chunk = - ByteMapMmapper::address_to_mmap_chunks_up(FIXED_ADDRESS + pages_to_bytes(pages)); - let test_memory_bytes = (end_chunk - start_chunk) * MMAP_CHUNK_BYTES; - with_cleanup( - || { - let mmapper = ByteMapMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let start_chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - let end_chunk = ByteMapMmapper::address_to_mmap_chunks_up( - FIXED_ADDRESS + pages_to_bytes(pages), - ); - assert_eq!(end_chunk - start_chunk, 2); - for chunk in start_chunk..end_chunk { - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Mapped - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, test_memory_bytes).unwrap(); - }, - ) - }) - } - - #[test] - fn protect() { - serial_test(|| { - let test_memory_bytes = MMAP_CHUNK_BYTES * 2; - let test_memory_pages = test_memory_bytes >> LOG_BYTES_IN_PAGE; - let protect_memory_bytes = MMAP_CHUNK_BYTES; - let protect_memory_pages = protect_memory_bytes >> LOG_BYTES_IN_PAGE; - with_cleanup( - || { - // map 2 chunks - let mmapper = ByteMapMmapper::new(); - mmapper - .ensure_mapped( - FIXED_ADDRESS, - test_memory_pages, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - - // protect 1 chunk - mmapper.protect(FIXED_ADDRESS, protect_memory_pages); - - let chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Protected - ); - assert_eq!( - mmapper.mapped[chunk + 1].load(Ordering::Relaxed), - MapState::Mapped - ); - }, - || { - memory::munmap(FIXED_ADDRESS, test_memory_bytes).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_on_protected_chunks() { - serial_test(|| { - let test_memory_bytes = MMAP_CHUNK_BYTES * 2; - let test_memory_pages = test_memory_bytes >> LOG_BYTES_IN_PAGE; - let protect_memory_pages_1 = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE; // protect one chunk in the first protect - let protect_memory_pages_2 = test_memory_pages; // protect both chunks in the second protect - with_cleanup( - || { - // map 2 chunks - let mmapper = ByteMapMmapper::new(); - mmapper - .ensure_mapped( - FIXED_ADDRESS, - test_memory_pages, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - - // protect 1 chunk - mmapper.protect(FIXED_ADDRESS, protect_memory_pages_1); - - let chunk = ByteMapMmapper::address_to_mmap_chunks_down(FIXED_ADDRESS); - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Protected - ); - assert_eq!( - mmapper.mapped[chunk + 1].load(Ordering::Relaxed), - MapState::Mapped - ); - - // ensure mapped - this will unprotect the previously protected chunk - mmapper - .ensure_mapped( - FIXED_ADDRESS, - protect_memory_pages_2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - assert_eq!( - mmapper.mapped[chunk].load(Ordering::Relaxed), - MapState::Mapped - ); - assert_eq!( - mmapper.mapped[chunk + 1].load(Ordering::Relaxed), - MapState::Mapped - ); - }, - || { - memory::munmap(FIXED_ADDRESS, test_memory_bytes).unwrap(); - }, - ) - }) - } -} diff --git a/src/util/heap/layout/mmapper.rs b/src/util/heap/layout/mmapper.rs deleted file mode 100644 index 364cc0bc0d..0000000000 --- a/src/util/heap/layout/mmapper.rs +++ /dev/null @@ -1,240 +0,0 @@ -use crate::util::heap::layout::vm_layout::*; -use crate::util::memory::*; -use crate::util::rust_util::rev_group::RevisitableGroupByForIterator; -use crate::util::Address; -use atomic::{Atomic, Ordering}; -use bytemuck::NoUninit; -use std::io::Result; - -/// Generic mmap and protection functionality -pub trait Mmapper: Sync { - /// Given an address array describing the regions of virtual memory to be used - /// by MMTk, demand zero map all of them if they are not already mapped. - /// - /// Arguments: - /// * `spaceMap`: An address array containing a pairs of start and end - /// addresses for each of the regions to be mapped - fn eagerly_mmap_all_spaces(&self, space_map: &[Address]); - - /// Mark a number of pages as mapped, without making any - /// request to the operating system. Used to mark pages - /// that the VM has already mapped. - /// - /// Arguments: - /// * `start`: Address of the first page to be mapped - /// * `bytes`: Number of bytes to ensure mapped - fn mark_as_mapped(&self, start: Address, bytes: usize); - - /// Quarantine/reserve address range. We mmap from the OS with no reserve and with PROT_NONE, - /// which should be little overhead. This ensures that we can reserve certain address range that - /// we can use if needed. Quarantined memory needs to be mapped before it can be used. - /// - /// Arguments: - /// * `start`: Address of the first page to be quarantined - /// * `bytes`: Number of bytes to quarantine from the start - /// * `strategy`: The mmap strategy. The `prot` field is ignored because we always use - /// `PROT_NONE`. - /// * `anno`: Human-readable annotation to apply to newly mapped memory ranges. - fn quarantine_address_range( - &self, - start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()>; - - /// Ensure that a range of pages is mmapped (or equivalent). If the - /// pages are not yet mapped, demand-zero map them. Note that mapping - /// occurs at chunk granularity, not page granularity. - /// - /// Arguments: - /// * `start`: The start of the range to be mapped. - /// * `pages`: The size of the range to be mapped, in pages - /// * `strategy`: The mmap strategy. - /// * `anno`: Human-readable annotation to apply to newly mapped memory ranges. - // NOTE: There is a monotonicity assumption so that only updates require lock - // acquisition. - // TODO: Fix the above to support unmapping. - fn ensure_mapped( - &self, - start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()>; - - /// Is the page pointed to by this address mapped? Returns true if - /// the page at the given address is mapped. - /// - /// Arguments: - /// * `addr`: Address in question - fn is_mapped_address(&self, addr: Address) -> bool; - - /// Mark a number of pages as inaccessible. - /// - /// Arguments: - /// * `start`: Address of the first page to be protected - /// * `pages`: Number of pages to be protected - fn protect(&self, start: Address, pages: usize); -} - -/// The mmap state of a mmap chunk. -#[repr(u8)] -#[derive(Copy, Clone, PartialEq, Eq, Debug, NoUninit)] -pub(super) enum MapState { - /// The chunk is unmapped and not managed by MMTk. - Unmapped, - /// The chunk is reserved for future use. MMTk reserved the address range but hasn't used it yet. - /// We have reserved the addresss range with mmap_noreserve with PROT_NONE. - Quarantined, - /// The chunk is mapped by MMTk and is in use. - Mapped, - /// The chunk is mapped and is also protected by MMTk. - Protected, -} - -impl MapState { - /// Check the current MapState of the chunk, and transition the chunk to MapState::Mapped. - /// The caller should hold a lock before invoking this method. - pub(super) fn transition_to_mapped( - state: &Atomic, - mmap_start: Address, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - trace!( - "Trying to map {} - {}", - mmap_start, - mmap_start + MMAP_CHUNK_BYTES - ); - let res = match state.load(Ordering::Relaxed) { - MapState::Unmapped => dzmmap_noreplace(mmap_start, MMAP_CHUNK_BYTES, strategy, anno), - MapState::Protected => munprotect(mmap_start, MMAP_CHUNK_BYTES, strategy.prot), - MapState::Quarantined => unsafe { - dzmmap(mmap_start, MMAP_CHUNK_BYTES, strategy, anno) - }, - // might have become MapState::Mapped here - MapState::Mapped => Ok(()), - }; - if res.is_ok() { - state.store(MapState::Mapped, Ordering::Relaxed); - } - res - } - - /// Check the current MapState of the chunk, and transition the chunk to MapState::Quarantined. - /// The caller should hold a lock before invoking this method. - pub(super) fn transition_to_quarantined( - state: &Atomic, - mmap_start: Address, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - trace!( - "Trying to quarantine {} - {}", - mmap_start, - mmap_start + MMAP_CHUNK_BYTES - ); - let res = match state.load(Ordering::Relaxed) { - MapState::Unmapped => mmap_noreserve(mmap_start, MMAP_CHUNK_BYTES, strategy, anno), - MapState::Quarantined => Ok(()), - MapState::Mapped => { - // If a chunk is mapped by us and we try to quarantine it, we simply don't do anything. - // We allow this as it is possible to have a situation like this: - // we have global side metadata S, and space A and B. We quarantine memory X for S for A, then map - // X for A, and then we quarantine memory Y for S for B. It is possible that X and Y is the same chunk, - // so the chunk is already mapped for A, and we try quarantine it for B. We simply allow this transition. - return Ok(()); - } - MapState::Protected => panic!("Cannot quarantine protected memory"), - }; - if res.is_ok() { - state.store(MapState::Quarantined, Ordering::Relaxed); - } - res - } - - /// Equivalent to calling `transition_to_quarantined` on each element of `states`, but faster. - /// The caller should hold a lock before invoking this method. - /// - /// The memory region to transition starts from `mmap_start`. The size is the chunk size - /// multiplied by the total number of `Atomic` in the nested slice of slices. - /// - /// This function is introduced to speed up initialization of MMTk. MMTk tries to quarantine - /// very large amount of memory during start-up. If we quarantine one chunk at a time, it will - /// require thousands of `mmap` calls to cover gigabytes of quarantined memory, introducing a - /// noticeable delay. - /// - /// Arguments: - /// - /// * `state_slices`: A slice of slices. Each inner slice is a part of a `Slab`. - /// * `mmap_start`: The start of the region to transition. - /// * `strategy`: The mmap strategy. - /// * `anno`: Human-readable annotation to apply to newly mapped memory ranges. - pub(super) fn bulk_transition_to_quarantined( - state_slices: &[&[Atomic]], - mmap_start: Address, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - trace!( - "Trying to bulk-quarantine {} - {}", - mmap_start, - mmap_start + MMAP_CHUNK_BYTES * state_slices.iter().map(|s| s.len()).sum::(), - ); - - let mut start_index = 0; - - for group in state_slices - .iter() - .copied() - .flatten() - .revisitable_group_by(|s| s.load(Ordering::Relaxed)) - { - let end_index = start_index + group.len; - let start_addr = mmap_start + MMAP_CHUNK_BYTES * start_index; - let end_addr = mmap_start + MMAP_CHUNK_BYTES * end_index; - - match group.key { - MapState::Unmapped => { - trace!("Trying to quarantine {} - {}", start_addr, end_addr); - mmap_noreserve(start_addr, end_addr - start_addr, strategy, anno)?; - - for state in group { - state.store(MapState::Quarantined, Ordering::Relaxed); - } - } - MapState::Quarantined => { - trace!("Already quarantine {} - {}", start_addr, end_addr); - } - MapState::Mapped => { - trace!("Already mapped {} - {}", start_addr, end_addr); - } - MapState::Protected => { - panic!("Cannot quarantine protected memory") - } - } - - start_index = end_index; - } - - Ok(()) - } - - /// Check the current MapState of the chunk, and transition the chunk to MapState::Protected. - /// The caller should hold a lock before invoking this method. - pub(super) fn transition_to_protected( - state: &Atomic, - mmap_start: Address, - ) -> Result<()> { - match state.load(Ordering::Relaxed) { - MapState::Mapped => { - crate::util::memory::mprotect(mmap_start, MMAP_CHUNK_BYTES).unwrap(); - state.store(MapState::Protected, Ordering::Relaxed); - } - MapState::Protected => {} - _ => panic!("Cannot transition {:?} to protected", mmap_start), - } - Ok(()) - } -} diff --git a/src/util/heap/layout/mmapper/csm/byte_map_storage.rs b/src/util/heap/layout/mmapper/csm/byte_map_storage.rs new file mode 100644 index 0000000000..8205b9ecd2 --- /dev/null +++ b/src/util/heap/layout/mmapper/csm/byte_map_storage.rs @@ -0,0 +1,124 @@ +use super::MapState; +use crate::util::heap::layout::mmapper::csm::ChunkRange; +use crate::util::heap::layout::mmapper::csm::MapStateStorage; +use crate::util::rust_util::rev_group::RevisitableGroupByForIterator; +use crate::util::Address; + +use crate::util::heap::layout::vm_layout::*; +use std::fmt; +use std::sync::atomic::Ordering; +use std::sync::Mutex; + +use atomic::Atomic; +use std::io::Result; + +/// For now, we only use `ByteMapStateStorage` for 32-bit address range. +const MMAP_NUM_CHUNKS: usize = 1 << (32 - LOG_BYTES_IN_CHUNK); + +/// A [`MapStateStorage`] implementation based on a simple array. +/// +/// Currently it is sized to cover a 32-bit address range. +pub struct ByteMapStateStorage { + lock: Mutex<()>, + mapped: [Atomic; MMAP_NUM_CHUNKS], +} + +impl fmt::Debug for ByteMapStateStorage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ByteMapStateStorage({})", MMAP_NUM_CHUNKS) + } +} + +impl MapStateStorage for ByteMapStateStorage { + fn get_state(&self, chunk: Address) -> MapState { + let index = chunk >> LOG_BYTES_IN_CHUNK; + let Some(slot) = self.mapped.get(index) else { + return MapState::Unmapped; + }; + slot.load(Ordering::Relaxed) + } + + fn bulk_set_state(&self, range: ChunkRange, state: MapState) { + if range.is_empty() { + return; + } + + let index_start = range.start >> LOG_BYTES_IN_CHUNK; + let index_limit = range.limit() >> LOG_BYTES_IN_CHUNK; + for index in index_start..index_limit { + self.mapped[index].store(state, Ordering::Relaxed); + } + } + + fn bulk_transition_state(&self, range: ChunkRange, mut transformer: F) -> Result<()> + where + F: FnMut(ChunkRange, MapState) -> Result>, + { + if range.is_empty() { + return Ok(()); + } + + if range.is_single_chunk() { + let chunk = range.start; + let index = chunk >> LOG_BYTES_IN_CHUNK; + let slot: &Atomic = &self.mapped[index]; + let state = slot.load(Ordering::Relaxed); + if let Some(new_state) = transformer(range, state)? { + slot.store(new_state, Ordering::Relaxed); + } + return Ok(()); + } + + let index_start = range.start >> LOG_BYTES_IN_CHUNK; + let index_limit = range.limit() >> LOG_BYTES_IN_CHUNK; + + let mut group_start = index_start; + for group in self.mapped.as_slice()[index_start..index_limit] + .iter() + .revisitable_group_by(|s| s.load(Ordering::Relaxed)) + { + let state = group.key; + let group_end = group_start + group.len; + let group_start_addr = + unsafe { Address::from_usize(group_start << LOG_BYTES_IN_CHUNK) }; + let group_bytes = group.len << LOG_BYTES_IN_CHUNK; + let group_range = ChunkRange::new_aligned(group_start_addr, group_bytes); + if let Some(new_state) = transformer(group_range, state)? { + for index in group_start..group_end { + self.mapped[index].store(new_state, Ordering::Relaxed); + } + } + group_start = group_end; + } + + Ok(()) + } +} + +impl ByteMapStateStorage { + pub fn new() -> Self { + // Because AtomicU8 does not implement Copy, it is a compilation error to usen the + // expression `[Atomic::new(MapState::Unmapped); MMAP_NUM_CHUNKS]` because that involves + // copying. We must define a constant for it. + // + // TODO: Use the inline const expression `const { Atomic::new(MapState::Unmapped) }` after + // we bump MSRV to 1.79. + + // If we declare a const Atomic, Clippy will warn about const items being interior mutable. + // Using inline const expression will eliminate this warning, but that is experimental until + // 1.79. Fix it after we bump MSRV. + #[allow(clippy::declare_interior_mutable_const)] + const INITIAL_ENTRY: Atomic = Atomic::new(MapState::Unmapped); + + ByteMapStateStorage { + lock: Mutex::new(()), + mapped: [INITIAL_ENTRY; MMAP_NUM_CHUNKS], + } + } +} + +impl Default for ByteMapStateStorage { + fn default() -> Self { + Self::new() + } +} diff --git a/src/util/heap/layout/mmapper/csm/mod.rs b/src/util/heap/layout/mmapper/csm/mod.rs new file mode 100644 index 0000000000..b7069ec6bb --- /dev/null +++ b/src/util/heap/layout/mmapper/csm/mod.rs @@ -0,0 +1,466 @@ +use crate::util::constants::LOG_BYTES_IN_PAGE; +use crate::util::conversions::raw_is_aligned; +use crate::util::heap::layout::vm_layout::*; +use crate::util::heap::layout::Mmapper; +use crate::util::memory::*; +use crate::util::Address; +use bytemuck::NoUninit; +use std::{io::Result, sync::Mutex}; + +mod byte_map_storage; +#[cfg(target_pointer_width = "64")] +mod two_level_storage; + +#[cfg(target_pointer_width = "32")] +type ChosenMapStateStorage = byte_map_storage::ByteMapStateStorage; +#[cfg(target_pointer_width = "64")] +type ChosenMapStateStorage = two_level_storage::TwoLevelStateStorage; + +/// A range of whole chunks. Always aligned. +/// +/// This type is used internally by the chunk state mmapper and its storage backends. +#[derive(Clone, Copy)] +struct ChunkRange { + start: Address, + bytes: usize, +} + +impl ChunkRange { + fn new_aligned(start: Address, bytes: usize) -> Self { + debug_assert!( + start.is_aligned_to(BYTES_IN_CHUNK), + "start {start} is not chunk-aligned" + ); + debug_assert!( + raw_is_aligned(bytes, BYTES_IN_CHUNK), + "bytes 0x{bytes:x} is not a multiple of chunks" + ); + Self { start, bytes } + } + + fn new_unaligned(start: Address, bytes: usize) -> Self { + let start_aligned = start.align_down(BYTES_IN_CHUNK); + let end_aligned = (start + bytes).align_up(BYTES_IN_CHUNK); + Self::new_aligned(start_aligned, end_aligned - start_aligned) + } + + fn limit(&self) -> Address { + self.start + self.bytes + } + + fn is_within_limit(&self, limit: Address) -> bool { + self.limit() <= limit + } + + fn is_empty(&self) -> bool { + self.bytes == 0 + } + + fn is_single_chunk(&self) -> bool { + self.bytes == BYTES_IN_CHUNK + } +} + +impl std::fmt::Display for ChunkRange { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}-{}", self.start, self.limit()) + } +} + +/// The back-end storage of [`ChunkStateMmapper`]. It is responsible for holding the states of each +/// chunk (eagerly or lazily) and transitioning the states in bulk. +trait MapStateStorage { + /// Return the state of a given `chunk` (must be aligned). + /// + /// Note that all chunks are logically `MapState::Unmapped` before the states are stored. They + /// include chunks outside the mappable address range. + fn get_state(&self, chunk: Address) -> MapState; + + /// Set all chunks within `range` to `state`. + fn bulk_set_state(&self, range: ChunkRange, state: MapState); + + /// Visit the chunk states within `range` and allow the `transformer` callback to inspect and + /// change the states. + /// + /// It visits chunks from low to high addresses, and calls `transformer(group_range, + /// group_state)` for each contiguous chunk range `group_range` that have the same state + /// `group_state`. `transformer` can take actions accordingly and return one of the three + /// values: + /// - `Err(err)`: Stop visiting and return `Err(err)` from `bulk_transition_state` + /// immediately. + /// - `Ok(None)`: Continue visiting the next chunk range without changing chunk states. + /// - `Ok(Some(new_state))`: Set the state of all chunks within `group_range` to `new_state`. + /// + /// Return `Ok(())` if finished visiting all chunks normally. + fn bulk_transition_state(&self, range: ChunkRange, transformer: F) -> Result<()> + where + F: FnMut(ChunkRange, MapState) -> Result>; +} + +/// A [`Mmapper`] implementation based on a logical array of chunk states. +/// +/// The [`ChunkStateMmapper::storage`] field holds the state of each chunk, and the +/// [`ChunkStateMmapper`] itself actually makes system calls to manage the memory mapping. +/// +/// As the name suggests, this implementation of [`Mmapper`] operates at the granularity of chunks. +pub struct ChunkStateMmapper { + /// Lock for transitioning map states. + transition_lock: Mutex<()>, + /// This holds the [`MapState`] for each chunk. + storage: ChosenMapStateStorage, +} + +impl ChunkStateMmapper { + pub fn new() -> Self { + Self { + transition_lock: Default::default(), + storage: ChosenMapStateStorage::new(), + } + } + + #[cfg(test)] + fn get_state(&self, chunk: Address) -> MapState { + self.storage.get_state(chunk) + } +} + +/// Generic mmap and protection functionality +impl Mmapper for ChunkStateMmapper { + fn log_granularity(&self) -> u8 { + LOG_BYTES_IN_CHUNK as u8 + } + + fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]) { + unimplemented!() + } + + fn mark_as_mapped(&self, start: Address, bytes: usize) { + let _guard = self.transition_lock.lock().unwrap(); + + let range = ChunkRange::new_unaligned(start, bytes); + self.storage.bulk_set_state(range, MapState::Mapped); + } + + fn quarantine_address_range( + &self, + start: Address, + pages: usize, + strategy: MmapStrategy, + anno: &MmapAnnotation, + ) -> Result<()> { + let _guard = self.transition_lock.lock().unwrap(); + + let bytes = pages << LOG_BYTES_IN_PAGE; + let range = ChunkRange::new_unaligned(start, bytes); + + self.storage + .bulk_transition_state(range, |group_range, state| { + let group_start: Address = group_range.start; + let group_bytes = group_range.bytes; + + match state { + MapState::Unmapped => { + trace!("Trying to quarantine {group_range}"); + mmap_noreserve(group_start, group_bytes, strategy, anno)?; + Ok(Some(MapState::Quarantined)) + } + MapState::Quarantined => { + trace!("Already quarantine {group_range}"); + Ok(None) + } + MapState::Mapped => { + trace!("Already mapped {group_range}"); + Ok(None) + } + MapState::Protected => { + panic!("Cannot quarantine protected memory") + } + } + }) + } + + fn ensure_mapped( + &self, + start: Address, + pages: usize, + strategy: MmapStrategy, + anno: &MmapAnnotation, + ) -> Result<()> { + let _guard = self.transition_lock.lock().unwrap(); + + let bytes = pages << LOG_BYTES_IN_PAGE; + let range = ChunkRange::new_unaligned(start, bytes); + + self.storage + .bulk_transition_state(range, |group_range, state| { + let group_start: Address = group_range.start; + let group_bytes = group_range.bytes; + + match state { + MapState::Unmapped => { + dzmmap_noreplace(group_start, group_bytes, strategy, anno)?; + Ok(Some(MapState::Mapped)) + } + MapState::Protected => { + munprotect(group_start, group_bytes, strategy.prot)?; + Ok(Some(MapState::Mapped)) + } + MapState::Quarantined => { + unsafe { dzmmap(group_start, group_bytes, strategy, anno) }?; + Ok(Some(MapState::Mapped)) + } + MapState::Mapped => Ok(None), + } + }) + } + + fn is_mapped_address(&self, addr: Address) -> bool { + self.storage.get_state(addr) == MapState::Mapped + } + + fn protect(&self, start: Address, pages: usize) { + let _guard = self.transition_lock.lock().unwrap(); + + let bytes = pages << LOG_BYTES_IN_PAGE; + let range = ChunkRange::new_unaligned(start, bytes); + + self.storage + .bulk_transition_state(range, |group_range, state| { + let group_start: Address = group_range.start; + let group_bytes = group_range.bytes; + + match state { + MapState::Mapped => { + crate::util::memory::mprotect(group_start, group_bytes).unwrap(); + Ok(Some(MapState::Protected)) + } + MapState::Protected => Ok(None), + _ => panic!("Cannot transition {group_range} to protected",), + } + }) + .unwrap(); + } +} + +/// The mmap state of a mmap chunk. +#[repr(u8)] +#[derive(Copy, Clone, PartialEq, Eq, Debug, NoUninit)] +enum MapState { + /// The chunk is unmapped and not managed by MMTk. + Unmapped, + /// The chunk is reserved for future use. MMTk reserved the address range but hasn't used it yet. + /// We have reserved the addresss range with mmap_noreserve with PROT_NONE. + Quarantined, + /// The chunk is mapped by MMTk and is in use. + Mapped, + /// The chunk is mapped and is also protected by MMTk. + Protected, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mmap_anno_test; + use crate::util::constants::LOG_BYTES_IN_PAGE; + use crate::util::memory; + use crate::util::test_util::CHUNK_STATE_MMAPPER_TEST_REGION; + use crate::util::test_util::{serial_test, with_cleanup}; + use crate::util::{conversions, Address}; + + const FIXED_ADDRESS: Address = CHUNK_STATE_MMAPPER_TEST_REGION.start; + const MAX_BYTES: usize = CHUNK_STATE_MMAPPER_TEST_REGION.size; + + fn pages_to_chunks_up(pages: usize) -> usize { + conversions::raw_align_up(pages, BYTES_IN_CHUNK) / BYTES_IN_CHUNK + } + + fn get_chunk_map_state(mmapper: &ChunkStateMmapper, chunk: Address) -> MapState { + chunk.is_aligned_to(BYTES_IN_CHUNK); + mmapper.get_state(chunk) + } + + #[test] + fn ensure_mapped_1page() { + serial_test(|| { + let pages = 1; + with_cleanup( + || { + let mmapper = ChunkStateMmapper::new(); + mmapper + .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .unwrap(); + + let chunks = pages_to_chunks_up(pages); + for i in 0..chunks { + assert_eq!( + get_chunk_map_state( + &mmapper, + FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) + ), + MapState::Mapped + ); + } + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + #[test] + fn ensure_mapped_1chunk() { + serial_test(|| { + let pages = BYTES_IN_CHUNK >> LOG_BYTES_IN_PAGE as usize; + with_cleanup( + || { + let mmapper = ChunkStateMmapper::new(); + mmapper + .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .unwrap(); + + let chunks = pages_to_chunks_up(pages); + for i in 0..chunks { + assert_eq!( + get_chunk_map_state( + &mmapper, + FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) + ), + MapState::Mapped + ); + } + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + + #[test] + fn ensure_mapped_more_than_1chunk() { + serial_test(|| { + let pages = (BYTES_IN_CHUNK + BYTES_IN_CHUNK / 2) >> LOG_BYTES_IN_PAGE as usize; + with_cleanup( + || { + let mmapper = ChunkStateMmapper::new(); + mmapper + .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .unwrap(); + + let chunks = pages_to_chunks_up(pages); + for i in 0..chunks { + assert_eq!( + get_chunk_map_state( + &mmapper, + FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) + ), + MapState::Mapped + ); + } + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + + #[test] + fn protect() { + serial_test(|| { + with_cleanup( + || { + // map 2 chunks + let mmapper = ChunkStateMmapper::new(); + let pages_per_chunk = BYTES_IN_CHUNK >> LOG_BYTES_IN_PAGE as usize; + mmapper + .ensure_mapped( + FIXED_ADDRESS, + pages_per_chunk * 2, + MmapStrategy::TEST, + mmap_anno_test!(), + ) + .unwrap(); + + // protect 1 chunk + mmapper.protect(FIXED_ADDRESS, pages_per_chunk); + + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + MapState::Protected + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + BYTES_IN_CHUNK), + MapState::Mapped + ); + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } + + #[test] + fn ensure_mapped_on_protected_chunks() { + serial_test(|| { + with_cleanup( + || { + // map 2 chunks + let mmapper = ChunkStateMmapper::new(); + let pages_per_chunk = BYTES_IN_CHUNK >> LOG_BYTES_IN_PAGE as usize; + mmapper + .ensure_mapped( + FIXED_ADDRESS, + pages_per_chunk * 2, + MmapStrategy::TEST, + mmap_anno_test!(), + ) + .unwrap(); + + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + MapState::Mapped + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + BYTES_IN_CHUNK), + MapState::Mapped + ); + + // protect 1 chunk + mmapper.protect(FIXED_ADDRESS, pages_per_chunk); + + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + MapState::Protected + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + BYTES_IN_CHUNK), + MapState::Mapped + ); + + // ensure mapped - this will unprotect the previously protected chunk + mmapper + .ensure_mapped( + FIXED_ADDRESS, + pages_per_chunk * 2, + MmapStrategy::TEST, + mmap_anno_test!(), + ) + .unwrap(); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS), + MapState::Mapped + ); + assert_eq!( + get_chunk_map_state(&mmapper, FIXED_ADDRESS + BYTES_IN_CHUNK), + MapState::Mapped + ); + }, + || { + memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + }, + ) + }) + } +} diff --git a/src/util/heap/layout/mmapper/csm/two_level_storage.rs b/src/util/heap/layout/mmapper/csm/two_level_storage.rs new file mode 100644 index 0000000000..0902023cd1 --- /dev/null +++ b/src/util/heap/layout/mmapper/csm/two_level_storage.rs @@ -0,0 +1,253 @@ +//! This module contains [`TwoLevelStateStorage`], an implementation of [`MapStateStorage`] that is +//! designed to work well on 64-bit machines. Currently it supports 48-bit address spaces, and many +//! constants and data structures (such as [`Slab`]) are larger than `i32::MAX`. For this reason, +//! this module is only available on 64-bit machines. + +use super::MapState; +use crate::util::heap::layout::mmapper::csm::{ChunkRange, MapStateStorage}; +use crate::util::heap::layout::vm_layout::*; +use crate::util::rust_util::atomic_box::OnceOptionBox; +use crate::util::rust_util::rev_group::RevisitableGroupByForIterator; +use crate::util::rust_util::zeroed_alloc::new_zeroed_vec; +use crate::util::Address; +use atomic::{Atomic, Ordering}; +use std::fmt; +use std::io::Result; + +/// Logarithm of the address space size a user-space program is allowed to use. +/// This is enough for ARM64, x86_64 and some other architectures. +/// Feel free to increase it if we plan to support larger address spaces. +const LOG_MAPPABLE_BYTES: usize = 48; +/// Address space size a user-space program is allowed to use. +const MAPPABLE_BYTES: usize = 1 << LOG_MAPPABLE_BYTES; +/// The limit of mappable address +const MAPPABLE_ADDRESS_LIMIT: Address = unsafe { Address::from_usize(MAPPABLE_BYTES) }; + +/// Log number of bytes per slab. For a two-level array, it is advisable to choose the arithmetic +/// mean of [`LOG_MAPPABLE_BYTES`] and [`LOG_BYTES_IN_CHUNK`] in order to make [`MMAP_SLAB_BYTES`] +/// the geometric mean of [`MAPPABLE_BYTES`] and [`BYTES_IN_CHUNK`]. This will balance the array +/// size of [`TwoLevelStateStorage::slabs`] and [`Slab`]. +/// +/// TODO: Use `usize::midpoint` after bumping MSRV to 1.85 +const LOG_MMAP_SLAB_BYTES: usize = + LOG_BYTES_IN_CHUNK + (LOG_MAPPABLE_BYTES - LOG_BYTES_IN_CHUNK) / 2; +/// Number of bytes per slab. +const MMAP_SLAB_BYTES: usize = 1 << LOG_MMAP_SLAB_BYTES; + +/// Log number of chunks per slab. +const LOG_MMAP_CHUNKS_PER_SLAB: usize = LOG_MMAP_SLAB_BYTES - LOG_BYTES_IN_CHUNK; +/// Number of chunks per slab. +const MMAP_CHUNKS_PER_SLAB: usize = 1 << LOG_MMAP_CHUNKS_PER_SLAB; + +/// Mask for getting in-slab bits from an address. +/// Invert this to get out-of-slab bits. +const MMAP_SLAB_MASK: usize = (1 << LOG_MMAP_SLAB_BYTES) - 1; + +/// Logarithm of maximum number of slabs. +const LOG_MAX_SLABS: usize = LOG_MAPPABLE_BYTES - LOG_MMAP_SLAB_BYTES; +/// maximum number of slabs. +const MAX_SLABS: usize = 1 << LOG_MAX_SLABS; + +/// The slab type. Each slab holds the `MapState` of multiple chunks. +type Slab = [Atomic; MMAP_CHUNKS_PER_SLAB]; + +/// A two-level implementation of `MapStateStorage`. +/// +/// It is essentially a lazily initialized array of [`Atomic`]. Because it is designed to +/// govern a large address range, and the array is sparse, we use a two-level design. The higher +/// level holds a vector of slabs, and each slab holds an array of [`Atomic`]. Each slab +/// governs an aligned region of [`MMAP_CHUNKS_PER_SLAB`] chunks. Slabs are lazily created when the +/// user intends to write into one of its `MapState`. +pub struct TwoLevelStateStorage { + /// Slabs + slabs: Vec>, +} + +unsafe impl Send for TwoLevelStateStorage {} +unsafe impl Sync for TwoLevelStateStorage {} + +impl fmt::Debug for TwoLevelStateStorage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "TwoLevelMapper({})", 1 << LOG_MAX_SLABS) + } +} + +impl MapStateStorage for TwoLevelStateStorage { + fn get_state(&self, chunk: Address) -> MapState { + let Some(slab) = self.slab_table(chunk) else { + return MapState::Unmapped; + }; + slab[Self::in_slab_index(chunk)].load(Ordering::Relaxed) + } + + fn bulk_set_state(&self, range: ChunkRange, state: MapState) { + if range.is_empty() { + return; + } + + if range.is_single_chunk() { + let addr = range.start; + let slab = self.get_or_allocate_slab_table(addr); + slab[Self::in_slab_index(addr)].store(state, Ordering::Relaxed); + return; + } + + self.foreach_slab_slice_for_write(range, |slice| { + for slot in slice { + slot.store(state, Ordering::Relaxed); + } + }); + } + + fn bulk_transition_state(&self, range: ChunkRange, mut transformer: F) -> Result<()> + where + F: FnMut(ChunkRange, MapState) -> Result>, + { + if range.is_empty() { + return Ok(()); + } + + if range.is_single_chunk() { + let addr = range.start; + let slab = self.get_or_allocate_slab_table(addr); + let slot = &slab[Self::in_slab_index(addr)]; + + let old_state = slot.load(Ordering::Relaxed); + if let Some(new_state) = transformer(range, old_state)? { + slot.store(new_state, Ordering::Relaxed); + }; + + return Ok(()); + } + + let mut slice_indices = Vec::new(); + + self.foreach_slab_slice_for_write(range, |slice| { + slice_indices.push(slice); + }); + + let start = range.start; + // Chunk index from `start`. + let mut start_index: usize = 0usize; + + for group in slice_indices + .iter() + .copied() + .flatten() + .revisitable_group_by(|s| s.load(Ordering::Relaxed)) + { + let state = group.key; + let end_index = start_index + group.len; + let group_start = start + (start_index << LOG_BYTES_IN_CHUNK); + let group_bytes = group.len << LOG_BYTES_IN_CHUNK; + let group_range = ChunkRange::new_aligned(group_start, group_bytes); + + if let Some(new_state) = transformer(group_range, state)? { + for slot in group { + slot.store(new_state, Ordering::Relaxed); + } + } + + start_index = end_index; + } + + Ok(()) + } +} + +impl TwoLevelStateStorage { + pub fn new() -> Self { + Self { + slabs: new_zeroed_vec(MAX_SLABS), + } + } + + fn new_slab() -> Slab { + std::array::from_fn(|_| Atomic::new(MapState::Unmapped)) + } + + fn slab_table(&self, addr: Address) -> Option<&Slab> { + let index: usize = Self::slab_index(addr); + let slot = self.slabs.get(index)?; + // Note: We don't need acquire here. See `get_or_allocate_slab_table`. + slot.get(Ordering::Relaxed) + } + + fn get_or_allocate_slab_table(&self, addr: Address) -> &Slab { + let index: usize = Self::slab_index(addr); + let Some(slot) = self.slabs.get(index) else { + panic!("Cannot allocate slab for address: {addr}"); + }; + // Note: We set both order_load and order_store to `Relaxed` because we never populate the + // content of the slab before making the `OnceOptionBox` point to the new slab. For this + // reason, the release-acquire relation is not needed here. + slot.get_or_init(Ordering::Relaxed, Ordering::Relaxed, Self::new_slab) + } + + fn slab_index(addr: Address) -> usize { + addr >> LOG_MMAP_SLAB_BYTES + } + + fn in_slab_index(addr: Address) -> usize { + (addr & MMAP_SLAB_MASK) >> LOG_BYTES_IN_CHUNK + } + + /// Visit all slabs that overlap with the `range` from low to high address. For each slab, call + /// `f` with the slice of the slab that overlap with the chunks within the `range`. + fn foreach_slab_slice_for_write<'s, F>(&'s self, range: ChunkRange, mut f: F) + where + F: FnMut(&'s [Atomic]), + { + debug_assert!( + range.is_within_limit(MAPPABLE_ADDRESS_LIMIT), + "range {range} out of bound" + ); + + let limit = range.limit(); + let mut low = range.start; + while low < limit { + let high = (low + MMAP_SLAB_BYTES) + .align_down(MMAP_SLAB_BYTES) + .min(limit); + + let slab = self.get_or_allocate_slab_table(low); + let low_index = Self::in_slab_index(low); + let high_index = Self::in_slab_index(high); + let ub_index = if high_index == 0 { + MMAP_CHUNKS_PER_SLAB + } else { + high_index + }; + f(&slab[low_index..ub_index]); + + low = high; + } + } + + fn chunk_index_to_address(base: Address, chunk: usize) -> Address { + base + (chunk << LOG_BYTES_IN_CHUNK) + } + + /// Align `addr` down to slab size. + fn slab_align_down(addr: Address) -> Address { + addr.align_down(MMAP_SLAB_BYTES) + } + + /// Get the base address of the next slab after the slab that contains `addr`. + fn slab_limit(addr: Address) -> Address { + Self::slab_align_down(addr) + MMAP_SLAB_BYTES + } + + /// Return the index of the chunk that contains `addr` within the slab starting at `slab`. + /// If `addr` is beyond the end of the slab, the result could be beyond the end of the slab. + fn chunk_index(slab: Address, addr: Address) -> usize { + let delta = addr - slab; + delta >> LOG_BYTES_IN_CHUNK + } +} + +impl Default for TwoLevelStateStorage { + fn default() -> Self { + Self::new() + } +} diff --git a/src/util/heap/layout/mmapper/mod.rs b/src/util/heap/layout/mmapper/mod.rs new file mode 100644 index 0000000000..b7444e51c3 --- /dev/null +++ b/src/util/heap/layout/mmapper/mod.rs @@ -0,0 +1,100 @@ +use crate::util::{ + memory::{MmapAnnotation, MmapStrategy}, + Address, +}; +use std::io::Result; + +#[allow(unused)] // Used in doc comment. +use crate::util::constants::LOG_BYTES_IN_PAGE; + +pub mod csm; + +/// An `Mmapper` manages the mmap state of memory used by the heap and side metadata of MMTk. +/// +/// For the efficiency of implementation, an `Mmapper` operates at the granularity of +/// [`Mmapper::granularity()`]. Methods that take memory ranges as arguments will round the range +/// to the overlapping chunks. +pub trait Mmapper: Sync { + /// The logarithm of granularity of this `Mmapper`, in bytes. Must be at least + /// [`LOG_BYTES_IN_PAGE`]. + /// + /// See trait-level doc for [`Mmapper`] for details. + fn log_granularity(&self) -> u8; + + /// The granularity of `Mmapper`. Don't override this method. Override + /// [`Mmapper::log_granularity`] instead. + /// + /// See trait-level doc for [`Mmapper`] for details. + fn granularity(&self) -> usize { + 1 << self.log_granularity() + } + + /// Given an address array describing the regions of virtual memory to be used + /// by MMTk, demand zero map all of them if they are not already mapped. + /// + /// Arguments: + /// * `spaceMap`: An address array containing a pairs of start and end + /// addresses for each of the regions to be mapped + fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]); + + /// Mark a number of pages as mapped, without making any + /// request to the operating system. Used to mark pages + /// that the VM has already mapped. + /// + /// Arguments: + /// * `start`: Address of the first page to be mapped + /// * `bytes`: Number of bytes to ensure mapped + fn mark_as_mapped(&self, start: Address, bytes: usize); + + /// Quarantine/reserve address range. We mmap from the OS with no reserve and with PROT_NONE, + /// which should be little overhead. This ensures that we can reserve certain address range that + /// we can use if needed. Quarantined memory needs to be mapped before it can be used. + /// + /// Arguments: + /// * `start`: Address of the first page to be quarantined + /// * `pages`: Number of pages to quarantine from the start + /// * `strategy`: The mmap strategy. The `prot` field is ignored because we always use + /// `PROT_NONE`. + /// * `anno`: Human-readable annotation to apply to newly mapped memory ranges. + fn quarantine_address_range( + &self, + start: Address, + pages: usize, + strategy: MmapStrategy, + anno: &MmapAnnotation, + ) -> Result<()>; + + /// Ensure that a range of pages is mmapped (or equivalent). If the + /// pages are not yet mapped, demand-zero map them. Note that mapping + /// occurs at chunk granularity, not page granularity. + /// + /// Arguments: + /// * `start`: The start of the range to be mapped. + /// * `pages`: The size of the range to be mapped, in pages + /// * `strategy`: The mmap strategy. + /// * `anno`: Human-readable annotation to apply to newly mapped memory ranges. + // NOTE: There is a monotonicity assumption so that only updates require lock + // acquisition. + // TODO: Fix the above to support unmapping. + fn ensure_mapped( + &self, + start: Address, + pages: usize, + strategy: MmapStrategy, + anno: &MmapAnnotation, + ) -> Result<()>; + + /// Is the page pointed to by this address mapped? Returns true if + /// the page at the given address is mapped. + /// + /// Arguments: + /// * `addr`: Address in question + fn is_mapped_address(&self, addr: Address) -> bool; + + /// Mark a number of pages as inaccessible. + /// + /// Arguments: + /// * `start`: Address of the first page to be protected + /// * `pages`: Number of pages to be protected + fn protect(&self, start: Address, pages: usize); +} diff --git a/src/util/heap/layout/mod.rs b/src/util/heap/layout/mod.rs index 5412c280ea..6539274c6f 100644 --- a/src/util/heap/layout/mod.rs +++ b/src/util/heap/layout/mod.rs @@ -3,9 +3,6 @@ pub mod vm_layout; mod mmapper; pub use self::mmapper::Mmapper; -mod byte_map_mmapper; -#[cfg(target_pointer_width = "64")] -mod two_level_mmapper; mod map; pub(crate) use self::map::CreateFreeListResult; @@ -29,15 +26,12 @@ pub fn create_vm_map() -> Box { } } -#[cfg(target_pointer_width = "32")] -pub fn create_mmapper() -> Box { - Box::new(byte_map_mmapper::ByteMapMmapper::new()) -} +pub fn create_mmapper() -> Box { + // TODO: Select a MapStateStorage based on the actuall address space size. + // For example, choose ByteMapStateStorage for 39-bit or less virtual space. -#[cfg(target_pointer_width = "64")] -pub fn create_mmapper() -> Box { - // TODO: ByteMapMmapper for 39-bit or less virtual space - Box::new(two_level_mmapper::TwoLevelMmapper::new()) + use crate::util::heap::layout::mmapper::csm::ChunkStateMmapper; + Box::new(ChunkStateMmapper::new()) } use crate::util::Address; diff --git a/src/util/heap/layout/two_level_mmapper.rs b/src/util/heap/layout/two_level_mmapper.rs deleted file mode 100644 index 57f86cca9f..0000000000 --- a/src/util/heap/layout/two_level_mmapper.rs +++ /dev/null @@ -1,508 +0,0 @@ -//! This module contains [`TwoLevelMmapper`], an implementation of [`Mmapper`] that is designed to -//! work well on 64-bit machines. Currently it supports 48-bit address spaces, and many constants -//! and data structures (such as [`Slab`]) are larger than `i32::MAX`. For this reason, this module -//! is only available on 64-bit machines. - -use super::mmapper::MapState; -use super::Mmapper; -use crate::util::constants::BYTES_IN_PAGE; -use crate::util::conversions; -use crate::util::heap::layout::vm_layout::*; -use crate::util::memory::{MmapAnnotation, MmapStrategy}; -use crate::util::rust_util::atomic_box::OnceOptionBox; -use crate::util::rust_util::zeroed_alloc::new_zeroed_vec; -use crate::util::Address; -use atomic::{Atomic, Ordering}; -use std::fmt; -use std::io::Result; -use std::sync::Mutex; - -/// Logarithm of the address space size a user-space program is allowed to use. -/// This is enough for ARM64, x86_64 and some other architectures. -/// Feel free to increase it if we plan to support larger address spaces. -const LOG_MAPPABLE_BYTES: usize = 48; -/// Address space size a user-space program is allowed to use. -const MAPPABLE_BYTES: usize = 1 << LOG_MAPPABLE_BYTES; - -/// Log number of bytes per slab. -/// For a two-level array, it is advisable to choose the arithmetic mean of [`LOG_MAPPABLE_BYTES`] -/// and [`LOG_MMAP_CHUNK_BYTES`] in order to make [`MMAP_SLAB_BYTES`] the geometric mean of -/// [`MAPPABLE_BYTES`] and [`MMAP_CHUNK_BYTES`]. This will balance the array size of -/// [`TwoLevelMmapper::slabs`] and [`Slab`]. -/// -/// TODO: Use `usize::midpoint` after bumping MSRV to 1.85 -const LOG_MMAP_SLAB_BYTES: usize = - LOG_MMAP_CHUNK_BYTES + (LOG_MAPPABLE_BYTES - LOG_MMAP_CHUNK_BYTES) / 2; -/// Number of bytes per slab. -const MMAP_SLAB_BYTES: usize = 1 << LOG_MMAP_SLAB_BYTES; - -/// Log number of chunks per slab. -const LOG_MMAP_CHUNKS_PER_SLAB: usize = LOG_MMAP_SLAB_BYTES - LOG_MMAP_CHUNK_BYTES; -/// Number of chunks per slab. -const MMAP_CHUNKS_PER_SLAB: usize = 1 << LOG_MMAP_CHUNKS_PER_SLAB; - -/// Mask for getting in-slab bits from an address. -/// Invert this to get out-of-slab bits. -const MMAP_SLAB_MASK: usize = (1 << LOG_MMAP_SLAB_BYTES) - 1; - -/// Logarithm of maximum number of slabs. -const LOG_MAX_SLABS: usize = LOG_MAPPABLE_BYTES - LOG_MMAP_SLAB_BYTES; -/// maximum number of slabs. -const MAX_SLABS: usize = 1 << LOG_MAX_SLABS; - -/// The slab type. Each slab holds the `MapState` of multiple chunks. -type Slab = [Atomic; MMAP_CHUNKS_PER_SLAB]; - -/// A two-level implementation of `Mmapper`. -/// -/// It is essentially a lazily initialized array of [`Atomic`]. Because it is designed to -/// govern a large address range, and the array is sparse, we use a two-level design. The higher -/// level holds a vector of slabs, and each slab holds an array of [`Atomic`]. Each slab -/// governs an aligned region of [`MMAP_CHUNKS_PER_SLAB`] chunks. Slabs are lazily created when the -/// user intends to write into one of its `MapState`. -pub struct TwoLevelMmapper { - /// Lock for transitioning map states. - /// - /// FIXME: We only need this lock when transitioning map states. - /// The `TwoLevelMmapper` itself is completely lock-free even when allocating new slabs. - /// We should move the lock one leve above, to `MapState`. - transition_lock: Mutex<()>, - /// Slabs - slabs: Vec>, -} - -unsafe impl Send for TwoLevelMmapper {} -unsafe impl Sync for TwoLevelMmapper {} - -impl fmt::Debug for TwoLevelMmapper { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "TwoLevelMapper({})", 1 << LOG_MAX_SLABS) - } -} - -impl Mmapper for TwoLevelMmapper { - fn eagerly_mmap_all_spaces(&self, _space_map: &[Address]) {} - - fn mark_as_mapped(&self, mut start: Address, bytes: usize) { - let end = start + bytes; - // Iterate over the slabs covered - while start < end { - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - for entry in mapped.iter().take(end_chunk).skip(start_chunk) { - entry.store(MapState::Mapped, Ordering::Relaxed); - } - start = high; - } - } - - fn quarantine_address_range( - &self, - mut start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - debug_assert!(start.is_aligned_to(BYTES_IN_PAGE)); - - let end = start + conversions::pages_to_bytes(pages); - - // Each `MapState` entry governs a chunk. - // Align down to the chunk start because we only mmap multiples of whole chunks. - let mmap_start = conversions::mmap_chunk_align_down(start); - - // We collect the chunk states from slabs to process them in bulk. - let mut state_slices = vec![]; - - // Iterate over the slabs covered - while start < end { - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - state_slices.push(&mapped[start_chunk..end_chunk]); - - start = high; - } - - #[cfg(debug_assertions)] - { - // Check if the number of entries are normal. - let mmap_end = conversions::mmap_chunk_align_up(end); - let num_slices = state_slices.iter().map(|s| s.len()).sum::(); - - debug_assert_eq!(mmap_start + BYTES_IN_CHUNK * num_slices, mmap_end); - } - - // Transition the chunks in bulk. - { - let _guard = self.transition_lock.lock().unwrap(); - MapState::bulk_transition_to_quarantined( - state_slices.as_slice(), - mmap_start, - strategy, - anno, - )?; - } - - Ok(()) - } - - fn ensure_mapped( - &self, - mut start: Address, - pages: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, - ) -> Result<()> { - let end = start + conversions::pages_to_bytes(pages); - // Iterate over the slabs covered - while start < end { - let base = Self::slab_align_down(start); - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - - /* Iterate over the chunks within the slab */ - for (chunk, entry) in mapped.iter().enumerate().take(end_chunk).skip(start_chunk) { - if matches!(entry.load(Ordering::Relaxed), MapState::Mapped) { - continue; - } - - let mmap_start = Self::chunk_index_to_address(base, chunk); - let _guard = self.transition_lock.lock().unwrap(); - MapState::transition_to_mapped(entry, mmap_start, strategy, anno)?; - } - start = high; - } - Ok(()) - } - - /** - * Return {@code true} if the given address has been mmapped - * - * @param addr The address in question. - * @return {@code true} if the given address has been mmapped - */ - fn is_mapped_address(&self, addr: Address) -> bool { - let mapped = self.slab_table(addr); - match mapped { - Some(mapped) => { - mapped[Self::chunk_index(Self::slab_align_down(addr), addr)].load(Ordering::Relaxed) - == MapState::Mapped - } - _ => false, - } - } - - fn protect(&self, mut start: Address, pages: usize) { - let end = start + conversions::pages_to_bytes(pages); - let _guard = self.transition_lock.lock().unwrap(); - // Iterate over the slabs covered - while start < end { - let base = Self::slab_align_down(start); - let high = if end > Self::slab_limit(start) && !Self::slab_limit(start).is_zero() { - Self::slab_limit(start) - } else { - end - }; - - let slab = Self::slab_align_down(start); - let start_chunk = Self::chunk_index(slab, start); - let end_chunk = Self::chunk_index(slab, conversions::mmap_chunk_align_up(high)); - - let mapped = self.get_or_allocate_slab_table(start); - - for (chunk, entry) in mapped.iter().enumerate().take(end_chunk).skip(start_chunk) { - let mmap_start = Self::chunk_index_to_address(base, chunk); - MapState::transition_to_protected(entry, mmap_start).unwrap(); - } - start = high; - } - } -} - -impl TwoLevelMmapper { - pub fn new() -> Self { - Self { - transition_lock: Default::default(), - slabs: new_zeroed_vec(MAX_SLABS), - } - } - - fn new_slab() -> Slab { - std::array::from_fn(|_| Atomic::new(MapState::Unmapped)) - } - - fn slab_table(&self, addr: Address) -> Option<&Slab> { - let index: usize = addr >> LOG_MMAP_SLAB_BYTES; - let slot = self.slabs.get(index)?; - // Note: We don't need acquire here. See `get_or_allocate_slab_table`. - slot.get(Ordering::Relaxed) - } - - fn get_or_allocate_slab_table(&self, addr: Address) -> &Slab { - let index: usize = addr >> LOG_MMAP_SLAB_BYTES; - let Some(slot) = self.slabs.get(index) else { - panic!("Cannot allocate slab for address: {addr}"); - }; - // Note: We set both order_load and order_store to `Relaxed` because we never populate the - // content of the slab before making the `OnceOptionBox` point to the new slab. For this - // reason, the release-acquire relation is not needed here. - slot.get_or_init(Ordering::Relaxed, Ordering::Relaxed, Self::new_slab) - } - - fn chunk_index_to_address(base: Address, chunk: usize) -> Address { - base + (chunk << LOG_MMAP_CHUNK_BYTES) - } - - /// Align `addr` down to slab size. - fn slab_align_down(addr: Address) -> Address { - addr.align_down(MMAP_SLAB_BYTES) - } - - /// Get the base address of the next slab after the slab that contains `addr`. - fn slab_limit(addr: Address) -> Address { - Self::slab_align_down(addr) + MMAP_SLAB_BYTES - } - - /// Return the index of the chunk that contains `addr` within the slab starting at `slab`. - /// If `addr` is beyond the end of the slab, the result could be beyond the end of the slab. - fn chunk_index(slab: Address, addr: Address) -> usize { - let delta = addr - slab; - delta >> LOG_MMAP_CHUNK_BYTES - } -} - -impl Default for TwoLevelMmapper { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::mmap_anno_test; - use crate::util::constants::LOG_BYTES_IN_PAGE; - use crate::util::heap::layout::vm_layout::MMAP_CHUNK_BYTES; - use crate::util::memory; - use crate::util::test_util::FRAGMENTED_MMAPPER_TEST_REGION; - use crate::util::test_util::{serial_test, with_cleanup}; - use crate::util::{conversions, Address}; - - const FIXED_ADDRESS: Address = FRAGMENTED_MMAPPER_TEST_REGION.start; - const MAX_BYTES: usize = FRAGMENTED_MMAPPER_TEST_REGION.size; - - fn pages_to_chunks_up(pages: usize) -> usize { - conversions::raw_align_up(pages, MMAP_CHUNK_BYTES) / MMAP_CHUNK_BYTES - } - - fn get_chunk_map_state(mmapper: &TwoLevelMmapper, chunk: Address) -> Option { - assert_eq!(conversions::mmap_chunk_align_up(chunk), chunk); - let mapped = mmapper.slab_table(chunk); - mapped.map(|m| { - m[TwoLevelMmapper::chunk_index(TwoLevelMmapper::slab_align_down(chunk), chunk)] - .load(Ordering::Relaxed) - }) - } - - #[test] - fn ensure_mapped_1page() { - serial_test(|| { - let pages = 1; - with_cleanup( - || { - let mmapper = TwoLevelMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let chunks = pages_to_chunks_up(pages); - for i in 0..chunks { - assert_eq!( - get_chunk_map_state( - &mmapper, - FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) - ), - Some(MapState::Mapped) - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - #[test] - fn ensure_mapped_1chunk() { - serial_test(|| { - let pages = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - with_cleanup( - || { - let mmapper = TwoLevelMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let chunks = pages_to_chunks_up(pages); - for i in 0..chunks { - assert_eq!( - get_chunk_map_state( - &mmapper, - FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) - ), - Some(MapState::Mapped) - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_more_than_1chunk() { - serial_test(|| { - let pages = (MMAP_CHUNK_BYTES + MMAP_CHUNK_BYTES / 2) >> LOG_BYTES_IN_PAGE as usize; - with_cleanup( - || { - let mmapper = TwoLevelMmapper::new(); - mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) - .unwrap(); - - let chunks = pages_to_chunks_up(pages); - for i in 0..chunks { - assert_eq!( - get_chunk_map_state( - &mmapper, - FIXED_ADDRESS + (i << LOG_BYTES_IN_CHUNK) - ), - Some(MapState::Mapped) - ); - } - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - - #[test] - fn protect() { - serial_test(|| { - with_cleanup( - || { - // map 2 chunks - let mmapper = TwoLevelMmapper::new(); - let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - mmapper - .ensure_mapped( - FIXED_ADDRESS, - pages_per_chunk * 2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - - // protect 1 chunk - mmapper.protect(FIXED_ADDRESS, pages_per_chunk); - - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS), - Some(MapState::Protected) - ); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), - Some(MapState::Mapped) - ); - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } - - #[test] - fn ensure_mapped_on_protected_chunks() { - serial_test(|| { - with_cleanup( - || { - // map 2 chunks - let mmapper = TwoLevelMmapper::new(); - let pages_per_chunk = MMAP_CHUNK_BYTES >> LOG_BYTES_IN_PAGE as usize; - mmapper - .ensure_mapped( - FIXED_ADDRESS, - pages_per_chunk * 2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - - // protect 1 chunk - mmapper.protect(FIXED_ADDRESS, pages_per_chunk); - - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS), - Some(MapState::Protected) - ); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), - Some(MapState::Mapped) - ); - - // ensure mapped - this will unprotect the previously protected chunk - mmapper - .ensure_mapped( - FIXED_ADDRESS, - pages_per_chunk * 2, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .unwrap(); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS), - Some(MapState::Mapped) - ); - assert_eq!( - get_chunk_map_state(&mmapper, FIXED_ADDRESS + MMAP_CHUNK_BYTES), - Some(MapState::Mapped) - ); - }, - || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); - }, - ) - }) - } -} diff --git a/src/util/heap/layout/vm_layout.rs b/src/util/heap/layout/vm_layout.rs index 720237e6fb..e9d3f24dc5 100644 --- a/src/util/heap/layout/vm_layout.rs +++ b/src/util/heap/layout/vm_layout.rs @@ -19,10 +19,6 @@ pub const BYTES_IN_CHUNK: usize = 1 << LOG_BYTES_IN_CHUNK; pub const CHUNK_MASK: usize = (1 << LOG_BYTES_IN_CHUNK) - 1; /// Coarsest unit of address space allocation, in pages pub const PAGES_IN_CHUNK: usize = 1 << (LOG_BYTES_IN_CHUNK - LOG_BYTES_IN_PAGE as usize); -/// log_2 of the granularity at which we map and unmap virtual address space in the heap -pub const LOG_MMAP_CHUNK_BYTES: usize = LOG_BYTES_IN_CHUNK; -/// Granularity at which we map and unmap virtual address space in the heap -pub const MMAP_CHUNK_BYTES: usize = 1 << LOG_MMAP_CHUNK_BYTES; /// Runtime-initialized virtual memory constants #[derive(Clone, Debug)] diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index adb56a28e1..24e5366dc8 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -252,12 +252,14 @@ pub fn find_last_non_zero_bit_in_metadata_bytes( meta_end: Address, ) -> FindMetaBitResult { use crate::util::constants::BYTES_IN_ADDRESS; - use crate::util::heap::vm_layout::MMAP_CHUNK_BYTES; + + let mmap_granularity = MMAPPER.granularity(); let mut cur = meta_end; - // We need to check if metadata address is mapped or not. But we only check at chunk granularity. - // This records the start of a chunk that is tested to be mapped. - let mut mapped_chunk = Address::MAX; + // We need to check if metadata address is mapped or not. But we make use of the granularity of + // the `Mmapper` to reduce the number of checks. This records the start of a grain that is + // tested to be mapped. + let mut mapped_grain = Address::MAX; while cur > meta_start { // If we can check the whole word, set step to word size. Otherwise, the step is 1 (byte) and we check byte. let step = if cur.is_aligned_to(BYTES_IN_ADDRESS) && cur - BYTES_IN_ADDRESS >= meta_start { @@ -277,10 +279,10 @@ pub fn find_last_non_zero_bit_in_metadata_bytes( ); // If we are looking at an address that is not in a mapped chunk, we need to check if the chunk if mapped. - if cur < mapped_chunk { + if cur < mapped_grain { if cur.is_mapped() { // This is mapped. No need to check for this chunk. - mapped_chunk = cur.align_down(MMAP_CHUNK_BYTES); + mapped_grain = cur.align_down(mmap_granularity); } else { return FindMetaBitResult::UnmappedMetadata; } diff --git a/src/util/test_util/mod.rs b/src/util/test_util/mod.rs index 15214fb499..4540cee3b8 100644 --- a/src/util/test_util/mod.rs +++ b/src/util/test_util/mod.rs @@ -54,18 +54,15 @@ const TEST_ADDRESS: Address = const TEST_ADDRESS: Address = crate::util::conversions::chunk_align_down(unsafe { Address::from_usize(0x2_0000_0000) }); -// util::heap::layout::fragmented_mmapper -pub(crate) const FRAGMENTED_MMAPPER_TEST_REGION: MmapTestRegion = - MmapTestRegion::reserve_before_address(TEST_ADDRESS, MMAP_CHUNK_BYTES * 2); -// util::heap::layout::byte_map_mmaper -pub(crate) const BYTE_MAP_MMAPPER_TEST_REGION: MmapTestRegion = - MmapTestRegion::reserve_before(FRAGMENTED_MMAPPER_TEST_REGION, MMAP_CHUNK_BYTES * 2); +// util::heap::layout::mmapper::csm +pub(crate) const CHUNK_STATE_MMAPPER_TEST_REGION: MmapTestRegion = + MmapTestRegion::reserve_before_address(TEST_ADDRESS, BYTES_IN_CHUNK * 2); // util::memory pub(crate) const MEMORY_TEST_REGION: MmapTestRegion = - MmapTestRegion::reserve_before(BYTE_MAP_MMAPPER_TEST_REGION, MMAP_CHUNK_BYTES); + MmapTestRegion::reserve_before(CHUNK_STATE_MMAPPER_TEST_REGION, BYTES_IN_CHUNK); // util::raw_memory_freelist pub(crate) const RAW_MEMORY_FREELIST_TEST_REGION: MmapTestRegion = - MmapTestRegion::reserve_before(MEMORY_TEST_REGION, MMAP_CHUNK_BYTES); + MmapTestRegion::reserve_before(MEMORY_TEST_REGION, BYTES_IN_CHUNK); // https://github.com/rust-lang/rfcs/issues/2798#issuecomment-552949300 pub fn panic_after(millis: u64, f: F) -> T