inline helpers.rs

Author: echohumm

CHANGELOG.md

@@ -38,6 +38,10 @@ _no versions before 0.13.2 have a changelog as I started the changelog in that v
 
 ## Version 0.15.0
 
+### Commit 6 (2025-8-05)
+
+- Inline alloc_ext.rs, resize_in_place.rs, and helpers.rs
+
 ### Commit 5 (2025-8-05)
 
 - Inline jemalloc.rs and mimalloc.rs

src/external_alloc/jemalloc.rs

@@ -155,6 +155,7 @@ impl Alloc for Jemalloc {
     ///
     /// - `ptr` must point to a block previously allocated with this allocator.
     /// - `old_layout` must describe exactly that block.
+    #[inline]
     unsafe fn realloc(
         &self,
         ptr: NonNull<u8>,

src/external_alloc/mod.rs

@@ -11,7 +11,6 @@ pub(crate) const REALLOC_DIFF_ALIGN: &str =
     "unsupported operation: attempted to reallocate with a different alignment";
 
 #[cfg(any(feature = "jemalloc", feature = "mimalloc"))]
-#[cfg_attr(miri, track_caller)]
 pub(crate) unsafe fn resize<F: Fn() -> *mut libc::c_void>(
     ralloc: F,
     ptr: core::ptr::NonNull<u8>,
@@ -99,8 +98,6 @@ pub mod ffi {
         // /// # Safety
         // ///
         // /// `ptr` must have been allocated by jemalloc and must not have been freed yet.
-
-        #[cfg_attr(miri, track_caller)]
         pub(crate) unsafe fn raw_ralloc(
             ptr: *mut c_void,
             old_layout: Layout,

src/features/alloc_ext.rs

@@ -24,6 +24,7 @@ pub trait AllocExt: Alloc {
     /// - [`AllocError::AllocFailed`] if allocation fails.
     /// - [`AllocError::ZeroSizedLayout`] if `T::SZ == 0`.
     #[track_caller]
+    #[inline]
     fn alloc_init<T, I: Fn(NonNull<T>)>(&self, init: I) -> Result<NonNull<T>, AllocError> {
         let guard = AllocGuard::new(self.alloc(T::LAYOUT)?.cast::<T>(), self);
         init(*guard);
@@ -37,6 +38,7 @@ pub trait AllocExt: Alloc {
     /// - [`AllocError::AllocFailed`] if allocation fails.
     /// - [`AllocError::ZeroSizedLayout`] if `T::SZ == 0`.
     #[track_caller]
+    #[inline]
     fn alloc_default<T: Default>(&self) -> Result<NonNull<T>, AllocError> {
         self.alloc_write(T::default())
     }
@@ -48,6 +50,7 @@ pub trait AllocExt: Alloc {
     /// - [`AllocError::AllocFailed`] if allocation fails.
     /// - [`AllocError::ZeroSizedLayout`] if `T::SZ == 0`.
     #[cfg_attr(miri, track_caller)]
+    #[inline]
     fn alloc_write<T>(&self, data: T) -> Result<NonNull<T>, AllocError> {
         alloc_then::<NonNull<T>, Self, T, _>(self, T::LAYOUT, data, |p, data| unsafe {
             let ptr: NonNull<T> = p.cast();
@@ -64,6 +67,7 @@ pub trait AllocExt: Alloc {
     /// - [`AllocError::AllocFailed`] if allocation fails.
     /// - [`AllocError::ZeroSizedLayout`] if `T::SZ == 0`.
     #[track_caller]
+    #[inline]
     fn alloc_clone_to<T: Clone>(&self, data: &T) -> Result<NonNull<T>, AllocError> {
         alloc_then::<NonNull<T>, Self, &T, _>(self, T::LAYOUT, data, |p, data| {
             let mut guard = AllocGuard::new(p.cast(), self);
@@ -172,6 +176,7 @@ pub trait AllocExt: Alloc {
     /// - `ptr` must point to a block of memory allocated using this allocator, be valid for reads
     ///   and writes, aligned, and a valid `T`.
     #[cfg_attr(miri, track_caller)]
+    #[inline]
     unsafe fn drop_and_dealloc<T: ?Sized>(&self, ptr: NonNull<T>) {
         ptr::drop_in_place(ptr.as_ptr());
         self.dealloc(ptr.cast::<u8>(), ptr.layout());
@@ -184,6 +189,7 @@ pub trait AllocExt: Alloc {
     /// - `ptr` must point to a block of memory allocated using this allocator.
     /// - `layout` must describe exactly the same block.
     #[cfg_attr(miri, track_caller)]
+    #[inline]
     unsafe fn zero_and_dealloc(&self, ptr: NonNull<u8>, layout: Layout) {
         ptr::write_bytes(ptr.as_ptr(), 0, layout.size());
         self.dealloc(ptr, layout);
@@ -195,6 +201,7 @@ pub trait AllocExt: Alloc {
     ///
     /// - `ptr` must point to a block of memory allocated using this allocator.
     #[cfg_attr(miri, track_caller)]
+    #[inline]
     unsafe fn dealloc_typed<T: ?Sized>(&self, ptr: NonNull<T>) {
         self.dealloc(ptr.cast::<u8>(), ptr.layout());
     }
@@ -205,6 +212,7 @@ pub trait AllocExt: Alloc {
     ///
     /// - `ptr` must point to a block of memory allocated using this allocator.
     #[cfg_attr(miri, track_caller)]
+    #[inline]
     unsafe fn zero_and_dealloc_typed<T: ?Sized>(&self, ptr: NonNull<T>) {
         ptr::write_bytes(ptr.as_ptr().cast::<u8>(), 0, ptr.size());
         self.dealloc_typed(ptr);
@@ -217,6 +225,7 @@ pub trait AllocExt: Alloc {
     /// - `ptr` must point to a block of memory allocated using this allocator, be valid for reads
     ///   and writes, aligned, and a valid `T`.
     #[cfg_attr(miri, track_caller)]
+    #[inline]
     unsafe fn drop_zero_and_dealloc<T: ?Sized>(&self, ptr: NonNull<T>) {
         ptr::drop_in_place(ptr.as_ptr());
         self.zero_and_dealloc_typed(ptr);

src/features/resize_in_place.rs

@@ -375,11 +375,16 @@ impl ResizeInPlace for crate::external_alloc::mimalloc::MiMalloc {
         }
     }
 
-    /// Shrinking in-place is not supported by mimalloc. This is a no-op.
+    /// Shrinking in-place is not supported by mimalloc.
+    ///
+    /// This is a no-op and always returns an error.
     ///
     /// # Errors
     ///
     /// - [`AllocError::Other`]`("unsupported operation: attempted to shrink in place")`.
+    // it just returns an error, no reason not to inline it.
+    #[allow(clippy::inline_always)]
+    #[inline(always)]
     unsafe fn shrink_in_place(
         &self,
         _: NonNull<u8>,

src/helpers.rs

@@ -1,14 +1,10 @@
 use crate::{
+    error::AllocError,
     type_props::{
-        varsized_nonnull_from_raw_parts,
-        varsized_pointer_from_raw_parts,
-        VarSized,
-        PtrProps,
-        SizedProps,
-        USIZE_MAX_NO_HIGH_BIT
+        varsized_nonnull_from_raw_parts, varsized_pointer_from_raw_parts, PtrProps, SizedProps,
+        VarSized, USIZE_MAX_NO_HIGH_BIT,
     },
-    error::AllocError,
-    Alloc
+    Alloc,
 };
 use core::{
     alloc::Layout,
@@ -96,12 +92,6 @@ pub(crate) fn zsl_check<Ret, F: Fn(Layout) -> Result<Ret, AllocError>>(
     }
 }
 
-/// Checks equality between two [`NonNull`] pointers.
-#[must_use]
-pub fn nonnull_eq<T: ?Sized>(a: NonNull<T>, b: NonNull<T>) -> bool {
-    peq(a.as_ptr() as *const T, b.as_ptr() as *const T)
-}
-
 /// Aligns the given value up to a non-zero alignment.
 #[must_use]
 pub const fn align_up(n: usize, align: NonZeroUsize) -> usize {
@@ -131,6 +121,7 @@ pub const fn dangling_nonnull_for(layout: Layout) -> NonNull<u8> {
 ///
 /// The caller must ensure the `alignment` is a valid power of two.
 #[must_use]
+#[inline]
 pub const unsafe fn dangling_nonnull(align: usize) -> NonNull<u8> {
     transmute::<NonZeroUsize, NonNull<u8>>(NonZeroUsize::new_unchecked(align))
 }
@@ -200,6 +191,7 @@ impl<'a, T: ?Sized, A: Alloc + ?Sized> AllocGuard<'a, T, A> {
     const_if! {
         "extra_const",
         "Creates a new guard from a pointer and a reference to an allocator.",
+        #[inline]
         pub const fn new(ptr: NonNull<T>, alloc: &'a A) -> AllocGuard<'a, T, A> {
             AllocGuard { ptr, alloc }
         }
@@ -209,6 +201,7 @@ impl<'a, T: ?Sized, A: Alloc + ?Sized> AllocGuard<'a, T, A> {
         "extra_extra_const",
         "Initializes the value by writing to the contained pointer.",
         #[cfg_attr(miri, track_caller)]
+        #[inline]
         pub const fn init(&mut self, elem: T)
         where
             T: Sized
@@ -224,6 +217,7 @@ impl<'a, T: ?Sized, A: Alloc + ?Sized> AllocGuard<'a, T, A> {
         "Releases ownership of the allocation, preventing deallocation, and returns the raw \
         pointer.",
         #[must_use]
+        #[inline]
         pub const fn release(self) -> NonNull<T> {
             let ptr = self.ptr;
             forget(self);
@@ -244,6 +238,7 @@ impl<T: ?Sized, A: Alloc + ?Sized> Drop for AllocGuard<'_, T, A> {
 impl<T: ?Sized, A: Alloc + ?Sized> Deref for AllocGuard<'_, T, A> {
     type Target = NonNull<T>;
 
+    #[inline]
     fn deref(&self) -> &NonNull<T> {
         &self.ptr
     }
@@ -299,6 +294,7 @@ impl<'a, T, A: Alloc + ?Sized> SliceAllocGuard<'a, T, A> {
     const_if! {
         "extra_const",
         "Creates a new slice guard for `full` elements at `ptr` in the given allocator.",
+        #[inline]
         pub const fn new(ptr: NonNull<T>, alloc: &'a A, full: usize)
         -> SliceAllocGuard<'a, T, A> {
             SliceAllocGuard {
@@ -315,6 +311,7 @@ impl<'a, T, A: Alloc + ?Sized> SliceAllocGuard<'a, T, A> {
         "Release ownership of the slice without deallocating memory, returning a `NonNull<T>` \
         pointer to the slice.",
         #[must_use]
+        #[inline]
         pub const fn release(self) -> NonNull<[T]> {
             let ret = self.get_init_part();
             forget(self);
@@ -327,6 +324,7 @@ impl<'a, T, A: Alloc + ?Sized> SliceAllocGuard<'a, T, A> {
         "Release ownership of the slice without deallocating memory, returning a `NonNull<T>` \
         pointer to the slice's first element.",
         #[must_use]
+        #[inline]
         pub const fn release_first(self) -> NonNull<T> {
             let ret = self.ptr;
             forget(self);
@@ -370,6 +368,7 @@ impl<'a, T, A: Alloc + ?Sized> SliceAllocGuard<'a, T, A> {
         "extra_extra_const",
         "Sets the initialized element count.\n\n# Safety\n\nThe caller must ensure the new \
         count is correct.",
+        #[inline]
         pub const unsafe fn set_init(&mut self, init: usize) {
             self.init = init;
         }
@@ -379,6 +378,7 @@ impl<'a, T, A: Alloc + ?Sized> SliceAllocGuard<'a, T, A> {
         "extra_extra_const",
         "Initializes the next element of the slice with `elem`.\n\n# Errors\n\nReturns \
         `Err(elem)` if the slice is at capacity.",
+       #[inline]
         pub const fn init(&mut self, elem: T) -> Result<(), T> {
             if self.init == self.full {
                 return Err(elem);
@@ -394,6 +394,7 @@ impl<'a, T, A: Alloc + ?Sized> SliceAllocGuard<'a, T, A> {
         "extra_extra_const",
         "Initializes the next element of the slice with `elem`.\n\n# Safety\n\nThe caller must \
         ensure that the slice is not at capacity. (`initialized() < full()`)",
+        #[inline]
         pub const unsafe fn init_unchecked(&mut self, elem: T) {
             ptr::write(self.ptr.as_ptr().add(self.init), elem);
             self.init += 1;
@@ -426,6 +427,7 @@ impl<'a, T, A: Alloc + ?Sized> SliceAllocGuard<'a, T, A> {
         "extra_const",
         "Returns how many elements have been initialized.",
         #[must_use]
+        #[inline]
         pub const fn initialized(&self) -> usize {
             self.init
         }
@@ -435,6 +437,7 @@ impl<'a, T, A: Alloc + ?Sized> SliceAllocGuard<'a, T, A> {
         "extra_const",
         "Returns the total number of elements in the slice.",
         #[must_use]
+        #[inline]
         pub const fn full(&self) -> usize {
             self.full
         }
@@ -444,6 +447,7 @@ impl<'a, T, A: Alloc + ?Sized> SliceAllocGuard<'a, T, A> {
         "extra_const",
         "Returns `true` if every element in the slice has been initialized.",
         #[must_use]
+        #[inline]
         pub const fn is_full(&self) -> bool {
             self.init == self.full
         }
@@ -453,6 +457,7 @@ impl<'a, T, A: Alloc + ?Sized> SliceAllocGuard<'a, T, A> {
         "extra_const",
         "Returns `true` if no elements have been initialized.",
         #[must_use]
+        #[inline]
         pub const fn is_empty(&self) -> bool {
             self.init == 0
         }
@@ -508,6 +513,7 @@ impl<T, A: Alloc + ?Sized> Drop for SliceAllocGuard<'_, T, A> {
 impl<T, A: Alloc + ?Sized> Deref for SliceAllocGuard<'_, T, A> {
     type Target = NonNull<T>;
 
+    #[inline]
     fn deref(&self) -> &NonNull<T> {
         &self.ptr
     }

src/unstable_util.rs

@@ -1,4 +1,6 @@
-use crate::{error::ArithOp, AllocError};
+use crate::{
+    error::ArithOp, helpers::align_up_unchecked, type_props::USIZE_MAX_NO_HIGH_BIT, AllocError,
+};
 use alloc::alloc::Layout;
 
 #[cfg(feature = "metadata")]
@@ -24,7 +26,7 @@ pub const fn pad_layout_for(layout: Layout, align: usize) -> usize {
     }
 
     let sz = layout.size();
-    size_rounded_up_to_align(sz, align) - sz
+    unsafe { align_up_unchecked(sz, align) - sz }
 }
 
 /// Creates a layout by rounding the size of this layout up to a multiple of the layout's alignment.
@@ -33,11 +35,12 @@ pub const fn pad_layout_for(layout: Layout, align: usize) -> usize {
 #[must_use]
 #[inline]
 pub const fn pad_layout_to_align(layout: Layout, align: usize) -> Layout {
+    if !align.is_power_of_two() {
+        return unsafe { Layout::from_size_align_unchecked(USIZE_MAX_NO_HIGH_BIT, 1) };
+    }
+
     unsafe {
-        Layout::from_size_align_unchecked(
-            size_rounded_up_to_align(layout.size(), align),
-            layout.align(),
-        )
+        Layout::from_size_align_unchecked(align_up_unchecked(layout.size(), align), layout.align())
     }
 }
 
@@ -90,10 +93,3 @@ pub const fn repeat_layout_packed(layout: Layout, count: usize) -> Result<Layout
         ))
     }
 }
-
-// TODO: unchecked ops
-#[inline]
-const fn size_rounded_up_to_align(sz: usize, align: usize) -> usize {
-    let sub1 = align - 1;
-    (sz + sub1) & !sub1
-}