Fallible notify using (#790)

On Windows, the most common way to out-of-memory is to fail to commit a
page.  The current design of the Pal assumes that notify_using always
succeeds, thus we have to raise an error if it fails.  This changes the
specification of notify_using to return a bool, indicating whether the
notify succeeded or not.

This allows Windows to fail the notification, and then the surrounding
code can handle the failure appropriately, such as by throwing an
exception or returning the nullptr for the allocation.

Co-authored-by: Copilot <[email protected]>

Author: mjp41

docs/PORTING.md

@@ -38,16 +38,18 @@ The state for a particular region of memory is set with
 static void notify_not_using(void* p, size_t size) noexcept;
 
 template<ZeroMem zero_mem>
-static void notify_using(void* p, size_t size) noexcept;
+static bool notify_using(void* p, size_t size) noexcept;
 
-static void notify_using_readonly(void* p, size_t size) noexcept;
+static bool notify_using_readonly(void* p, size_t size) noexcept;
 ```
 These functions notify the system that the range of memory from `p` to `p` + 
 `size` is in the relevant state.
 
 If the template parameter is set to `YesZero` then `notify_using` must ensure
 the range is full of zeros.
 
+The function should return `true` if the memory is now in the requested state, and `false` if it failed to make the memory useable.
+
 ```c++
 template<bool page_aligned = false>
 static void zero(void* p, size_t size) noexcept;

src/snmalloc/backend_helpers/authmap.h

@@ -13,8 +13,10 @@ namespace snmalloc
   {
     static SNMALLOC_FAST_PATH void init() {}
 
-    static SNMALLOC_FAST_PATH void register_range(capptr::Arena<void>, size_t)
-    {}
+    static SNMALLOC_FAST_PATH bool register_range(capptr::Arena<void>, size_t)
+    {
+      return true;
+    }
 
     template<bool potentially_out_of_range = false>
     static SNMALLOC_FAST_PATH capptr::Arena<void> amplify(capptr::Alloc<void> c)
@@ -44,7 +46,7 @@ namespace snmalloc
       concreteAuthmap.template init</* randomize_location */ false>();
     }
 
-    static SNMALLOC_FAST_PATH void
+    static SNMALLOC_FAST_PATH bool
     register_range(capptr::Arena<void> base, size_t size)
     {
       concreteAuthmap.register_range(address_cast(base), size);
@@ -55,6 +57,7 @@ namespace snmalloc
       {
         concreteAuthmap.set(a, base);
       }
+      return true;
     }
 
     template<bool potentially_out_of_range = false>

src/snmalloc/backend_helpers/commitrange.h

@@ -34,7 +34,17 @@ namespace snmalloc
           PAL::page_size);
         auto range = parent.alloc_range(size);
         if (range != nullptr)
-          PAL::template notify_using<NoZero>(range.unsafe_ptr(), size);
+        {
+          auto result =
+            PAL::template notify_using<NoZero>(range.unsafe_ptr(), size);
+          if (!result)
+          {
+            // If notify_using fails, we deallocate the range and return
+            // nullptr.
+            parent.dealloc_range(range, size);
+            return CapPtr<void, ChunkBounds>(nullptr);
+          }
+        }
         return range;
       }
 

src/snmalloc/backend_helpers/pagemap.h

@@ -96,15 +96,22 @@ namespace snmalloc
      * Mark the MetaEntry at the bottom of the range as a boundary, preventing
      * consolidation with a lower range, unless CONSOLIDATE_PAL_ALLOCS.
      */
-    static void register_range(capptr::Arena<void> p, size_t sz)
+    static bool register_range(capptr::Arena<void> p, size_t sz)
     {
-      concretePagemap.register_range(address_cast(p), sz);
+      auto result = concretePagemap.register_range(address_cast(p), sz);
+
+      if (!result)
+      {
+        return false;
+      }
+
       if constexpr (!CONSOLIDATE_PAL_ALLOCS)
       {
         // Mark start of allocation in pagemap.
         auto& entry = get_metaentry_mut(address_cast(p));
         entry.set_boundary();
       }
+      return true;
     }
 
     /**

src/snmalloc/backend_helpers/pagemapregisterrange.h

@@ -30,7 +30,13 @@ namespace snmalloc
 
         if (base != nullptr)
         {
-          Pagemap::register_range(base, size);
+          auto result = Pagemap::register_range(base, size);
+          if (!result)
+          {
+            // If register_range fails, typically there is no recovery from this
+            // so just return nullptr.
+            return CapPtr<void, ChunkBounds>(nullptr);
+          }
         }
 
         return base;

src/snmalloc/ds/pagemap.h

@@ -55,7 +55,7 @@ namespace snmalloc
     /**
      * Ensure this range of pagemap is accessible
      */
-    void register_range(address_t p, size_t length)
+    bool register_range(address_t p, size_t length)
     {
       SNMALLOC_ASSERT(is_initialised());
 
@@ -76,11 +76,18 @@ namespace snmalloc
       auto page_start = pointer_align_down<OS_PAGE_SIZE, char>(first);
       auto page_end = pointer_align_up<OS_PAGE_SIZE, char>(last);
       size_t using_size = pointer_diff(page_start, page_end);
-      PAL::template notify_using<NoZero>(page_start, using_size);
+      auto result = PAL::template notify_using<NoZero>(page_start, using_size);
+      if (!result)
+      {
+        // If notify_using fails, fails this call.
+        return false;
+      }
       if constexpr (pal_supports<CoreDump, PAL>)
       {
         PAL::notify_do_dump(page_start, using_size);
       }
+
+      return true;
     }
 
     constexpr FlatPagemap() = default;
@@ -234,22 +241,35 @@ namespace snmalloc
           // Only commit readonly memory for this range, if the platform
           // supports lazy commit.  Otherwise, this would be a lot of memory to
           // have mapped.
-          PAL::notify_using_readonly(
+          auto result = PAL::notify_using_readonly(
             start_page, pointer_diff(start_page, end_page));
+          if (!result)
+          {
+            PAL::error("Failed to initialise snmalloc.");
+          }
         }
       }
       else
       {
         if constexpr (pal_supports<LazyCommit, PAL>)
         {
-          PAL::notify_using_readonly(new_body_untyped, REQUIRED_SIZE);
+          auto result =
+            PAL::notify_using_readonly(new_body_untyped, REQUIRED_SIZE);
+          if (!result)
+          {
+            PAL::error("Failed to initialise snmalloc.");
+          }
         }
         new_body = static_cast<T*>(new_body_untyped);
       }
       // Ensure bottom page is committed
       // ASSUME: new memory is zeroed.
-      PAL::template notify_using<NoZero>(
+      auto result = PAL::template notify_using<NoZero>(
         pointer_align_down<OS_PAGE_SIZE>(new_body), OS_PAGE_SIZE);
+      if (!result)
+      {
+        PAL::error("Failed to initialise snmalloc.");
+      }
 
       // Set up zero page
       new_body[0] = body[0];
@@ -364,7 +384,8 @@ namespace snmalloc
 
     /**
      * Return the starting address corresponding to a given entry within the
-     * Pagemap. Also checks that the reference actually points to a valid entry.
+     * Pagemap. Also checks that the reference actually points to a valid
+     * entry.
      */
     [[nodiscard]] address_t get_address(const T& t) const
     {

src/snmalloc/mem/backend_concept.h

@@ -61,7 +61,7 @@ namespace snmalloc
   concept IsPagemapWithRegister = requires(capptr::Arena<void> p, size_t sz) {
                                     {
                                       Pagemap::register_range(p, sz)
-                                      } -> ConceptSame<void>;
+                                      } -> ConceptSame<bool>;
                                   };
 
   /**

src/snmalloc/pal/pal_apple.h

@@ -189,7 +189,7 @@ namespace snmalloc
      *
      */
     template<ZeroMem zero_mem>
-    static void notify_using(void* p, size_t size) noexcept
+    static bool notify_using(void* p, size_t size) noexcept
     {
       KeepErrno e;
       SNMALLOC_ASSERT(
@@ -207,7 +207,7 @@ namespace snmalloc
 
         if (SNMALLOC_LIKELY(r != MAP_FAILED))
         {
-          return;
+          return true;
         }
       }
 
@@ -232,6 +232,7 @@ namespace snmalloc
       {
         ::bzero(p, size);
       }
+      return true;
     }
 
     // Apple's `mmap` doesn't support user-specified alignment and only

src/snmalloc/pal/pal_concept.h

@@ -53,10 +53,10 @@ namespace snmalloc
 
                            {
                              PAL::template notify_using<NoZero>(vp, sz)
-                             } noexcept -> ConceptSame<void>;
+                             } noexcept -> ConceptSame<bool>;
                            {
                              PAL::template notify_using<YesZero>(vp, sz)
-                             } noexcept -> ConceptSame<void>;
+                             } noexcept -> ConceptSame<bool>;
 
                            {
                              PAL::template zero<false>(vp, sz)

src/snmalloc/pal/pal_freebsd_kernel.h

@@ -51,14 +51,15 @@ namespace snmalloc
 
     /// Notify platform that we will be using these pages
     template<ZeroMem zero_mem>
-    static void notify_using(void* p, size_t size)
+    static bool notify_using(void* p, size_t size)
     {
       vm_offset_t addr = get_vm_offset(p);
       int flags = M_WAITOK | ((zero_mem == YesZero) ? M_ZERO : 0);
       if (kmem_back(kernel_object, addr, size, flags) != KERN_SUCCESS)
       {
-        error("Out of memory");
+        return false;
       }
+      return true;
     }
 
     /// OS specific function for zeroing memory