From 08ed6f21f7c9000293cc4b2c6a659f5375a58eab Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 1 Sep 2025 15:01:29 +0800 Subject: [PATCH 1/5] Allow requesting GC while over-committing. This is useful if the VM cannot trigger GC at most of its allocation sites. If allocation over-commits while still triggering GC, the GC thread can pause the mutators at the nearest safepoints. --- src/policy/space.rs | 39 ++++++++++++++++++++++++------------- src/util/alloc/allocator.rs | 23 +++++++++++++++------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/policy/space.rs b/src/policy/space.rs index 654bcb22a6..807fa13cc4 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -108,17 +108,22 @@ pub trait Space: 'static + SFT + Sync + Downcast { ); // Should we poll to attempt to GC? - // - If tls is collector, we cannot attempt a GC. - // - If gc is disabled, we cannot attempt a GC. - // - If overcommit is allowed, we don't attempt a GC. + // - If tls is collector, we shall not poll. + // - If gc is disabled, we shall not poll. + // - If the on_fail option does not allow polling, we shall not poll. let should_poll = VM::VMActivePlan::is_mutator(tls) && VM::VMCollection::is_collection_enabled() - && !alloc_options.on_fail.allow_overcommit(); - // Is a GC allowed here? If we should poll but are not allowed to poll, we will panic. - // initialize_collection() has to be called so we know GC is initialized. - let allow_gc = should_poll + && !alloc_options.on_fail.allow_polling(); + + // Can we continue to allocate even if GC is triggered? + let allow_overcommit = alloc_options.on_fail.allow_overcommit(); + + // Can we block for GC if polling triggers GC? + // - If the MMTk instance is not initialized, there is no GC workers, and we cannot block for GC. + // - If the on_fail option does not allow blocking, we do not block for GC, either. + let allow_blocking_for_gc = should_poll && self.common().global_state.is_initialized() - && alloc_options.on_fail.allow_gc(); + && alloc_options.on_fail.allow_blocking_for_gc(); trace!("Reserving pages"); let pr = self.get_page_resource(); @@ -126,18 +131,26 @@ pub trait Space: 'static + SFT + Sync + Downcast { trace!("Pages reserved"); trace!("Polling .."); - if should_poll && self.get_gc_trigger().poll(false, Some(self.as_space())) { + // Whether we should try to allocate. We should try to allocate if + // - we shouldn't poll, or + // - we polled, but GC was not triggered, or + // - GC is triggered, but we allow over-committing. + let should_try_to_allocate = !should_poll + || !self.get_gc_trigger().poll(false, Some(self.as_space())) + || allow_overcommit; + + if !should_try_to_allocate { // Clear the request pr.clear_request(pages_reserved); // If we do not want GC on fail, just return zero. - if !alloc_options.on_fail.allow_gc() { + if !allow_blocking_for_gc { return Address::ZERO; } // Otherwise do GC here debug!("Collection required"); - assert!(allow_gc, "GC is not allowed here: collection is not initialized (did you call initialize_collection()?)."); + assert!(allow_blocking_for_gc, "GC is not allowed here: collection is not initialized (did you call initialize_collection()?)."); // Inform GC trigger about the pending allocation. let meta_pages_reserved = self.estimate_side_meta_pages(pages_reserved); @@ -257,13 +270,13 @@ pub trait Space: 'static + SFT + Sync + Downcast { pr.clear_request(pages_reserved); // If we do not want GC on fail, just return zero. - if !alloc_options.on_fail.allow_gc() { + if !allow_blocking_for_gc { return Address::ZERO; } // We thought we had memory to allocate, but somehow failed the allocation. Will force a GC. assert!( - allow_gc, + allow_blocking_for_gc, "Physical allocation failed when GC is not allowed!" ); diff --git a/src/util/alloc/allocator.rs b/src/util/alloc/allocator.rs index 4e53ab953d..290579c6d2 100644 --- a/src/util/alloc/allocator.rs +++ b/src/util/alloc/allocator.rs @@ -32,25 +32,34 @@ pub enum AllocationError { #[repr(u8)] #[derive(Copy, Clone, Default, PartialEq, bytemuck::NoUninit, Debug)] pub enum OnAllocationFail { - /// Request the GC. This is the default behavior. + /// Request the GC and block until GC finishes. This is the default behavior. #[default] RequestGC, - /// Instead of requesting GC, the allocation request returns with a failure value. + /// Request the GC. But instead of blocking for GC, the allocation request returns with a + /// failure value. ReturnFailure, - /// Instead of requesting GC, the allocation request simply overcommits the memory, - /// and return a valid result at its best efforts. + /// Instead of requesting GC, the allocation request simply overcommits the memory, and return a + /// valid result at its best efforts. GC worker threads will not be notified about the + /// allocation failure. OverCommit, + /// Request the GC. But instead of blocking for GC, the allocating thread continues to + /// allocate, overcommitting the memory. GC will be scheduled asynchronously by the GC worker + /// threads, and the current mutator may stop at a safepoint as soon as possible. + RequestAndOverCommit, } impl OnAllocationFail { pub(crate) fn allow_oom_call(&self) -> bool { *self == Self::RequestGC } - pub(crate) fn allow_gc(&self) -> bool { - *self == Self::RequestGC + pub(crate) fn allow_polling(&self) -> bool { + *self != Self::OverCommit } pub(crate) fn allow_overcommit(&self) -> bool { - *self == Self::OverCommit + *self == Self::OverCommit || *self == Self::RequestAndOverCommit + } + pub(crate) fn allow_blocking_for_gc(&self) -> bool { + *self == Self::RequestGC } } From 77721b874c8e1b493637d1bf36d53fee0a5d6f59 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 1 Sep 2025 16:23:47 +0800 Subject: [PATCH 2/5] Fix typo --- src/policy/space.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/policy/space.rs b/src/policy/space.rs index 807fa13cc4..2e881b58b6 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -113,7 +113,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { // - If the on_fail option does not allow polling, we shall not poll. let should_poll = VM::VMActivePlan::is_mutator(tls) && VM::VMCollection::is_collection_enabled() - && !alloc_options.on_fail.allow_polling(); + && alloc_options.on_fail.allow_polling(); // Can we continue to allocate even if GC is triggered? let allow_overcommit = alloc_options.on_fail.allow_overcommit(); From 5949f62f4698aff6caa7868e3c15d5b1222d4939 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 1 Sep 2025 18:07:00 +0800 Subject: [PATCH 3/5] Remove OverCommit --- src/policy/space.rs | 4 +--- src/util/alloc/allocator.rs | 9 +-------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/policy/space.rs b/src/policy/space.rs index 2e881b58b6..1cb04193d5 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -110,10 +110,8 @@ pub trait Space: 'static + SFT + Sync + Downcast { // Should we poll to attempt to GC? // - If tls is collector, we shall not poll. // - If gc is disabled, we shall not poll. - // - If the on_fail option does not allow polling, we shall not poll. let should_poll = VM::VMActivePlan::is_mutator(tls) - && VM::VMCollection::is_collection_enabled() - && alloc_options.on_fail.allow_polling(); + && VM::VMCollection::is_collection_enabled(); // Can we continue to allocate even if GC is triggered? let allow_overcommit = alloc_options.on_fail.allow_overcommit(); diff --git a/src/util/alloc/allocator.rs b/src/util/alloc/allocator.rs index 290579c6d2..8c5831e5e7 100644 --- a/src/util/alloc/allocator.rs +++ b/src/util/alloc/allocator.rs @@ -38,10 +38,6 @@ pub enum OnAllocationFail { /// Request the GC. But instead of blocking for GC, the allocation request returns with a /// failure value. ReturnFailure, - /// Instead of requesting GC, the allocation request simply overcommits the memory, and return a - /// valid result at its best efforts. GC worker threads will not be notified about the - /// allocation failure. - OverCommit, /// Request the GC. But instead of blocking for GC, the allocating thread continues to /// allocate, overcommitting the memory. GC will be scheduled asynchronously by the GC worker /// threads, and the current mutator may stop at a safepoint as soon as possible. @@ -52,11 +48,8 @@ impl OnAllocationFail { pub(crate) fn allow_oom_call(&self) -> bool { *self == Self::RequestGC } - pub(crate) fn allow_polling(&self) -> bool { - *self != Self::OverCommit - } pub(crate) fn allow_overcommit(&self) -> bool { - *self == Self::OverCommit || *self == Self::RequestAndOverCommit + *self == Self::RequestAndOverCommit } pub(crate) fn allow_blocking_for_gc(&self) -> bool { *self == Self::RequestGC From f0b992cf4710b00a84c280f2a11679d39e35e872 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 1 Sep 2025 18:07:22 +0800 Subject: [PATCH 4/5] Rename RequestAndOverCommit to OverCommit --- src/util/alloc/allocator.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/alloc/allocator.rs b/src/util/alloc/allocator.rs index 8c5831e5e7..e8e8f42e02 100644 --- a/src/util/alloc/allocator.rs +++ b/src/util/alloc/allocator.rs @@ -41,7 +41,7 @@ pub enum OnAllocationFail { /// Request the GC. But instead of blocking for GC, the allocating thread continues to /// allocate, overcommitting the memory. GC will be scheduled asynchronously by the GC worker /// threads, and the current mutator may stop at a safepoint as soon as possible. - RequestAndOverCommit, + OverCommit, } impl OnAllocationFail { @@ -49,7 +49,7 @@ impl OnAllocationFail { *self == Self::RequestGC } pub(crate) fn allow_overcommit(&self) -> bool { - *self == Self::RequestAndOverCommit + *self == Self::OverCommit } pub(crate) fn allow_blocking_for_gc(&self) -> bool { *self == Self::RequestGC From 59ff447a11d8f65658ce547ec8ff1858dfd28be0 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 9 Oct 2025 16:57:38 +0800 Subject: [PATCH 5/5] Replace OnAllocationFail with boolean options. --- src/policy/lockfreeimmortalspace.rs | 2 +- src/policy/space.rs | 10 ++-- src/util/alloc/allocator.rs | 75 +++++++++++++++-------------- src/util/alloc/mod.rs | 1 - 4 files changed, 46 insertions(+), 42 deletions(-) diff --git a/src/policy/lockfreeimmortalspace.rs b/src/policy/lockfreeimmortalspace.rs index f62520a1c1..1a34d179e1 100644 --- a/src/policy/lockfreeimmortalspace.rs +++ b/src/policy/lockfreeimmortalspace.rs @@ -151,7 +151,7 @@ impl Space for LockFreeImmortalSpace { }) .expect("update cursor failed"); if start + bytes > self.limit { - if alloc_options.on_fail.allow_oom_call() { + if alloc_options.allow_oom_call() { panic!("OutOfMemory"); } else { return Address::ZERO; diff --git a/src/policy/space.rs b/src/policy/space.rs index 0f15bb7acd..d47db220c7 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -85,7 +85,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { alloc_options: AllocationOptions, ) -> bool { if self.will_oom_on_acquire(size) { - if alloc_options.on_fail.allow_oom_call() { + if alloc_options.allow_oom_call() { VM::VMCollection::out_of_memory( tls, crate::util::alloc::AllocationError::HeapOutOfMemory, @@ -111,18 +111,18 @@ pub trait Space: 'static + SFT + Sync + Downcast { // Should we poll to attempt to GC? // - If tls is collector, we shall not poll. // - If gc is disabled, we shall not poll. - let should_poll = VM::VMActivePlan::is_mutator(tls) - && VM::VMCollection::is_collection_enabled(); + let should_poll = + VM::VMActivePlan::is_mutator(tls) && VM::VMCollection::is_collection_enabled(); // Can we continue to allocate even if GC is triggered? - let allow_overcommit = alloc_options.on_fail.allow_overcommit(); + let allow_overcommit = alloc_options.allow_overcommit; // Can we block for GC if polling triggers GC? // - If the MMTk instance is not initialized, there is no GC workers, and we cannot block for GC. // - If the on_fail option does not allow blocking, we do not block for GC, either. let allow_blocking_for_gc = should_poll && self.common().global_state.is_initialized() - && alloc_options.on_fail.allow_blocking_for_gc(); + && alloc_options.at_safepoint; trace!("Reserving pages"); let pr = self.get_page_resource(); diff --git a/src/util/alloc/allocator.rs b/src/util/alloc/allocator.rs index e8e8f42e02..54298ea779 100644 --- a/src/util/alloc/allocator.rs +++ b/src/util/alloc/allocator.rs @@ -28,47 +28,54 @@ pub enum AllocationError { MmapOutOfMemory, } -/// Behavior when an allocation fails, and a GC is expected. -#[repr(u8)] -#[derive(Copy, Clone, Default, PartialEq, bytemuck::NoUninit, Debug)] -pub enum OnAllocationFail { - /// Request the GC and block until GC finishes. This is the default behavior. - #[default] - RequestGC, - /// Request the GC. But instead of blocking for GC, the allocation request returns with a - /// failure value. - ReturnFailure, - /// Request the GC. But instead of blocking for GC, the allocating thread continues to - /// allocate, overcommitting the memory. GC will be scheduled asynchronously by the GC worker - /// threads, and the current mutator may stop at a safepoint as soon as possible. - OverCommit, -} - -impl OnAllocationFail { - pub(crate) fn allow_oom_call(&self) -> bool { - *self == Self::RequestGC - } - pub(crate) fn allow_overcommit(&self) -> bool { - *self == Self::OverCommit - } - pub(crate) fn allow_blocking_for_gc(&self) -> bool { - *self == Self::RequestGC - } -} - /// Allow specifying different behaviors with [`Allocator::alloc_with_options`]. #[repr(C)] -#[derive(Copy, Clone, Default, PartialEq, bytemuck::NoUninit, Debug)] +#[derive(Copy, Clone, PartialEq, bytemuck::NoUninit, Debug)] pub struct AllocationOptions { - /// When the allocation fails and a GC is originally expected, on_fail - /// allows a different behavior to avoid the GC. - pub on_fail: OnAllocationFail, + /// Whether polling is allowed. + /// + /// If true, the allocation attempt will give the GC trigger a chance to schedule a collection + /// if the GC trigger considers it needed. + /// + /// If false, the allocation attempt will not notify the GC trigger, and GC will not be + /// scheduled. + pub allow_polling: bool, + /// Whether the allocation is at a safepoint. + /// + /// If true, the allocation attempt will block for GC if GC is triggered. It will also call + /// [`Collection::out_of_memory`] when out of memory. + /// + /// If false, the allocation attempt will never block for GC, and it will never call + /// [`Collection::out_of_memory`]. Instead it returns an null address when GC is triggerted. + pub at_safepoint: bool, + /// Whether over-committing is allowed at this allocation site. Only meaningful if + /// `allow_polling == false`. + /// + /// If true, the allocation attempt will still try to allocate even if the GC trigger has + /// triggered a GC. + pub allow_overcommit: bool, +} + +/// The default value for `AllocationOptions` has the same semantics as calling [`Allocator::alloc`] +/// directly. +impl Default for AllocationOptions { + fn default() -> Self { + Self { + allow_polling: true, + at_safepoint: true, + allow_overcommit: false, + } + } } impl AllocationOptions { pub(crate) fn is_default(&self) -> bool { *self == AllocationOptions::default() } + + pub(crate) fn allow_oom_call(&self) -> bool { + self.at_safepoint + } } pub fn align_allocation_no_fill( @@ -380,9 +387,7 @@ pub trait Allocator: Downcast { return result; } - if result.is_zero() - && self.get_context().get_alloc_options().on_fail == OnAllocationFail::ReturnFailure - { + if result.is_zero() && !self.get_context().get_alloc_options().allow_oom_call() { return result; } diff --git a/src/util/alloc/mod.rs b/src/util/alloc/mod.rs index d3148dceed..2f905a913d 100644 --- a/src/util/alloc/mod.rs +++ b/src/util/alloc/mod.rs @@ -6,7 +6,6 @@ pub use allocator::fill_alignment_gap; pub use allocator::AllocationError; pub use allocator::AllocationOptions; pub use allocator::Allocator; -pub use allocator::OnAllocationFail; /// A list of all the allocators, embedded in Mutator pub(crate) mod allocators;