From 4f9cd6e1ec60f32d79d58a95a0ed879cd0d396bc Mon Sep 17 00:00:00 2001 From: Daniel Thornburgh Date: Mon, 24 Nov 2025 10:51:01 -0800 Subject: [PATCH] [libc][malloc] Ensure a minimum block alignment of 4 Most platforms inherently have a size_t alignment of 4, but this isn't true on every platform LLVM has some degree of backend support for. Accordingly, it's simple enough to just set the min alignment of each Block's usable space to 4 and lose the static_assert. --- libc/fuzzing/__support/freelist_heap_fuzz.cpp | 8 +-- libc/src/__support/block.h | 70 +++++++++---------- libc/src/__support/freelist_heap.h | 6 +- libc/src/__support/freestore.h | 9 ++- libc/test/src/__support/block_test.cpp | 38 +++++----- .../test/src/__support/freelist_heap_test.cpp | 4 +- libc/test/src/__support/freestore_test.cpp | 4 +- 7 files changed, 69 insertions(+), 70 deletions(-) diff --git a/libc/fuzzing/__support/freelist_heap_fuzz.cpp b/libc/fuzzing/__support/freelist_heap_fuzz.cpp index 0b400cb156491..b342b21895a08 100644 --- a/libc/fuzzing/__support/freelist_heap_fuzz.cpp +++ b/libc/fuzzing/__support/freelist_heap_fuzz.cpp @@ -147,7 +147,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) { // Perform allocation. void *ptr = nullptr; - size_t alignment = alignof(max_align_t); + size_t alignment = Block::MIN_ALIGN; switch (alloc_type) { case AllocType::MALLOC: ptr = heap.allocate(alloc_size); @@ -172,7 +172,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) { alloc_size - alloc.size); alloc.ptr = ptr; alloc.size = alloc_size; - alloc.alignment = alignof(max_align_t); + alloc.alignment = Block::MIN_ALIGN; } break; } @@ -194,8 +194,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) { if (ptr) { // aligned_allocate should automatically apply a minimum alignment. - if (alignment < alignof(max_align_t)) - alignment = alignof(max_align_t); + if (alignment < Block::MIN_ALIGN) + alignment = Block::MIN_ALIGN; // Check alignment. if (reinterpret_cast(ptr) % alignment) __builtin_trap(); diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h index b0d6576093244..d45af1079a5bc 100644 --- a/libc/src/__support/block.h +++ b/libc/src/__support/block.h @@ -43,8 +43,8 @@ using cpp::optional; /// The blocks store their offsets to the previous and next blocks. The latter /// is also the block's size. /// -/// All blocks have their usable space aligned to some multiple of max_align_t. -/// This also implies that block outer sizes are aligned to max_align_t. +/// All blocks have their usable space aligned to some multiple of MIN_ALIGN. +/// This also implies that block outer sizes are aligned to MIN_ALIGN. /// /// As an example, the diagram below represents two contiguous `Block`s. The /// indices indicate byte offsets: @@ -97,13 +97,17 @@ class Block { static constexpr size_t SIZE_MASK = ~(PREV_FREE_MASK | LAST_MASK); public: + // To ensure block sizes have two lower unused bits, ensure usable space is + // always aligned to at least 4 bytes. (The distances between usable spaces, + // the outer size, is then always also 4-aligned.) + static constexpr size_t MIN_ALIGN = cpp::max(size_t{4}, alignof(max_align_t)); // No copy or move. Block(const Block &other) = delete; Block &operator=(const Block &other) = delete; /// Initializes a given memory region into a first block and a sentinel last /// block. Returns the first block, which has its usable space aligned to - /// max_align_t. + /// MIN_ALIGN. static optional init(ByteSpan region); /// @returns A pointer to a `Block`, given a pointer to the start of the @@ -160,17 +164,17 @@ class Block { /// @returns A pointer to the usable space inside this block. /// - /// Aligned to some multiple of max_align_t. + /// Aligned to some multiple of MIN_ALIGN. LIBC_INLINE cpp::byte *usable_space() { auto *s = reinterpret_cast(this) + sizeof(Block); - LIBC_ASSERT(reinterpret_cast(s) % alignof(max_align_t) == 0 && - "usable space must be aligned to a multiple of max_align_t"); + LIBC_ASSERT(reinterpret_cast(s) % MIN_ALIGN == 0 && + "usable space must be aligned to MIN_ALIGN"); return s; } LIBC_INLINE const cpp::byte *usable_space() const { const auto *s = reinterpret_cast(this) + sizeof(Block); - LIBC_ASSERT(reinterpret_cast(s) % alignof(max_align_t) == 0 && - "usable space must be aligned to a multiple of max_align_t"); + LIBC_ASSERT(reinterpret_cast(s) % MIN_ALIGN == 0 && + "usable space must be aligned to MIN_ALIGN"); return s; } @@ -185,9 +189,9 @@ class Block { /// `new_inner_size`. The remaining space will be returned as a new block, /// with usable space aligned to `usable_space_alignment`. Note that the prev_ /// field of the next block counts as part of the inner size of the block. - /// `usable_space_alignment` must be a multiple of max_align_t. + /// `usable_space_alignment` must be a multiple of MIN_ALIGN. optional split(size_t new_inner_size, - size_t usable_space_alignment = alignof(max_align_t)); + size_t usable_space_alignment = MIN_ALIGN); /// Merges this block with the one that comes after it. bool merge_next(); @@ -228,13 +232,11 @@ class Block { LIBC_INLINE Block(size_t outer_size, bool is_last) : next_(outer_size) { // Last blocks are not usable, so they need not have sizes aligned to - // max_align_t. Their lower bits must still be free, so they must be aligned - // to Block. - LIBC_ASSERT( - outer_size % (is_last ? alignof(Block) : alignof(max_align_t)) == 0 && - "block sizes must be aligned"); - LIBC_ASSERT(is_usable_space_aligned(alignof(max_align_t)) && - "usable space must be aligned to a multiple of max_align_t"); + // MIN_ALIGN. + LIBC_ASSERT(outer_size % (is_last ? alignof(Block) : MIN_ALIGN) == 0 && + "block sizes must be aligned"); + LIBC_ASSERT(is_usable_space_aligned(MIN_ALIGN) && + "usable space must be aligned to a multiple of MIN_ALIGN"); if (is_last) next_ |= LAST_MASK; } @@ -249,11 +251,10 @@ class Block { // Returns 0 if there is no such size. LIBC_INLINE static size_t min_size_for_allocation(size_t alignment, size_t size) { - LIBC_ASSERT(alignment >= alignof(max_align_t) && - alignment % alignof(max_align_t) == 0 && - "alignment must be multiple of max_align_t"); + LIBC_ASSERT(alignment >= MIN_ALIGN && alignment % MIN_ALIGN == 0 && + "alignment must be multiple of MIN_ALIGN"); - if (alignment == alignof(max_align_t)) + if (alignment == MIN_ALIGN) return size; // We must create a new block inside this one (splitting). This requires a @@ -274,7 +275,7 @@ class Block { // So the maximum distance would be G - L. As a special case, if L is 1 // (unaligned), the max distance is G - 1. // - // This block's usable space is aligned to max_align_t >= Block. With zero + // This block's usable space is aligned to MIN_ALIGN >= Block. With zero // padding, the next block's usable space is sizeof(Block) past it, which is // a point aligned to Block. Thus the max padding needed is alignment - // alignof(Block). @@ -309,13 +310,15 @@ class Block { static BlockInfo allocate(Block *block, size_t alignment, size_t size); // These two functions may wrap around. - LIBC_INLINE static uintptr_t next_possible_block_start( - uintptr_t ptr, size_t usable_space_alignment = alignof(max_align_t)) { + LIBC_INLINE static uintptr_t + next_possible_block_start(uintptr_t ptr, + size_t usable_space_alignment = MIN_ALIGN) { return align_up(ptr + sizeof(Block), usable_space_alignment) - sizeof(Block); } - LIBC_INLINE static uintptr_t prev_possible_block_start( - uintptr_t ptr, size_t usable_space_alignment = alignof(max_align_t)) { + LIBC_INLINE static uintptr_t + prev_possible_block_start(uintptr_t ptr, + size_t usable_space_alignment = MIN_ALIGN) { return align_down(ptr, usable_space_alignment) - sizeof(Block); } @@ -360,9 +363,6 @@ class Block { static constexpr size_t PREV_FIELD_SIZE = sizeof(prev_); }; -static_assert(alignof(Block) >= 4, - "at least 2 bits must be available in block sizes for flags"); - LIBC_INLINE optional Block::init(ByteSpan region) { if (!region.data()) @@ -394,8 +394,8 @@ optional Block::init(ByteSpan region) { LIBC_INLINE Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size) { - LIBC_ASSERT(alignment % alignof(max_align_t) == 0 && - "alignment must be a multiple of max_align_t"); + LIBC_ASSERT(alignment % MIN_ALIGN == 0 && + "alignment must be a multiple of MIN_ALIGN"); BlockInfo info{block, /*prev=*/nullptr, /*next=*/nullptr}; @@ -430,8 +430,8 @@ Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size) { LIBC_INLINE optional Block::split(size_t new_inner_size, size_t usable_space_alignment) { - LIBC_ASSERT(usable_space_alignment % alignof(max_align_t) == 0 && - "alignment must be a multiple of max_align_t"); + LIBC_ASSERT(usable_space_alignment % MIN_ALIGN == 0 && + "alignment must be a multiple of MIN_ALIGN"); if (used()) return {}; @@ -445,8 +445,8 @@ optional Block::split(size_t new_inner_size, if (next_block_start < start) return {}; size_t new_outer_size = next_block_start - start; - LIBC_ASSERT(new_outer_size % alignof(max_align_t) == 0 && - "new size must be aligned to max_align_t"); + LIBC_ASSERT(new_outer_size % MIN_ALIGN == 0 && + "new size must be aligned to MIN_ALIGN"); if (outer_size() < new_outer_size || outer_size() - new_outer_size < sizeof(Block)) diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h index d58685194aeb8..2e0e371f96e8c 100644 --- a/libc/src/__support/freelist_heap.h +++ b/libc/src/__support/freelist_heap.h @@ -108,7 +108,7 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) { } LIBC_INLINE void *FreeListHeap::allocate(size_t size) { - return allocate_impl(alignof(max_align_t), size); + return allocate_impl(Block::MIN_ALIGN, size); } LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment, @@ -121,8 +121,8 @@ LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment, if (size % alignment != 0) return nullptr; - // The minimum alignment supported by Block is max_align_t. - alignment = cpp::max(alignment, alignof(max_align_t)); + // The minimum alignment supported by Block is MIN_ALIGN. + alignment = cpp::max(alignment, Block::MIN_ALIGN); return allocate_impl(alignment, size); } diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h index 09f2479debb36..2dcb4b10b93d5 100644 --- a/libc/src/__support/freestore.h +++ b/libc/src/__support/freestore.h @@ -41,11 +41,11 @@ class FreeStore { private: static constexpr size_t MIN_OUTER_SIZE = - align_up(sizeof(Block) + sizeof(FreeList::Node), alignof(max_align_t)); + align_up(sizeof(Block) + sizeof(FreeList::Node), Block::MIN_ALIGN); static constexpr size_t MIN_LARGE_OUTER_SIZE = - align_up(sizeof(Block) + sizeof(FreeTrie::Node), alignof(max_align_t)); + align_up(sizeof(Block) + sizeof(FreeTrie::Node), Block::MIN_ALIGN); static constexpr size_t NUM_SMALL_SIZES = - (MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / alignof(max_align_t); + (MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / Block::MIN_ALIGN; LIBC_INLINE static bool too_small(Block *block) { return block->outer_size() < MIN_OUTER_SIZE; @@ -98,8 +98,7 @@ LIBC_INLINE Block *FreeStore::remove_best_fit(size_t size) { LIBC_INLINE FreeList &FreeStore::small_list(Block *block) { LIBC_ASSERT(is_small(block) && "only legal for small blocks"); - return small_lists[(block->outer_size() - MIN_OUTER_SIZE) / - alignof(max_align_t)]; + return small_lists[(block->outer_size() - MIN_OUTER_SIZE) / Block::MIN_ALIGN]; } LIBC_INLINE FreeList *FreeStore::find_best_small_fit(size_t size) { diff --git a/libc/test/src/__support/block_test.cpp b/libc/test/src/__support/block_test.cpp index 904ac5c66994d..3029cde834a5d 100644 --- a/libc/test/src/__support/block_test.cpp +++ b/libc/test/src/__support/block_test.cpp @@ -22,14 +22,14 @@ using LIBC_NAMESPACE::cpp::span; TEST(LlvmLibcBlockTest, CanCreateSingleAlignedBlock) { constexpr size_t kN = 1024; - alignas(max_align_t) array bytes; + alignas(Block::MIN_ALIGN) array bytes; auto result = Block::init(bytes); ASSERT_TRUE(result.has_value()); Block *block = *result; EXPECT_EQ(reinterpret_cast(block) % alignof(Block), size_t{0}); - EXPECT_TRUE(block->is_usable_space_aligned(alignof(max_align_t))); + EXPECT_TRUE(block->is_usable_space_aligned(Block::MIN_ALIGN)); Block *last = block->next(); ASSERT_NE(last, static_cast(nullptr)); @@ -52,7 +52,7 @@ TEST(LlvmLibcBlockTest, CanCreateUnalignedSingleBlock) { constexpr size_t kN = 1024; // Force alignment, so we can un-force it below - alignas(max_align_t) array bytes; + alignas(Block::MIN_ALIGN) array bytes; span aligned(bytes); auto result = Block::init(aligned.subspan(1)); @@ -60,7 +60,7 @@ TEST(LlvmLibcBlockTest, CanCreateUnalignedSingleBlock) { Block *block = *result; EXPECT_EQ(reinterpret_cast(block) % alignof(Block), size_t{0}); - EXPECT_TRUE(block->is_usable_space_aligned(alignof(max_align_t))); + EXPECT_TRUE(block->is_usable_space_aligned(Block::MIN_ALIGN)); Block *last = block->next(); ASSERT_NE(last, static_cast(nullptr)); @@ -98,7 +98,7 @@ TEST(LlvmLibcBlockTest, CanSplitBlock) { EXPECT_EQ(block2->outer_size(), orig_size - block1->outer_size()); EXPECT_FALSE(block2->used()); EXPECT_EQ(reinterpret_cast(block2) % alignof(Block), size_t{0}); - EXPECT_TRUE(block2->is_usable_space_aligned(alignof(max_align_t))); + EXPECT_TRUE(block2->is_usable_space_aligned(Block::MIN_ALIGN)); EXPECT_EQ(block1->next(), block2); EXPECT_EQ(block2->prev_free(), block1); @@ -124,7 +124,7 @@ TEST(LlvmLibcBlockTest, CanSplitBlockUnaligned) { EXPECT_EQ(block2->outer_size(), orig_size - block1->outer_size()); EXPECT_FALSE(block2->used()); EXPECT_EQ(reinterpret_cast(block2) % alignof(Block), size_t{0}); - EXPECT_TRUE(block2->is_usable_space_aligned(alignof(max_align_t))); + EXPECT_TRUE(block2->is_usable_space_aligned(Block::MIN_ALIGN)); EXPECT_EQ(block1->next(), block2); EXPECT_EQ(block2->prev_free(), block1); @@ -211,7 +211,7 @@ TEST(LlvmLibcBlockTest, CanMakeMinimalSizeFirstBlock) { result = block->split(0); ASSERT_TRUE(result.has_value()); - EXPECT_LE(block->outer_size(), sizeof(Block) + alignof(max_align_t)); + EXPECT_LE(block->outer_size(), sizeof(Block) + Block::MIN_ALIGN); } TEST(LlvmLibcBlockTest, CanMakeMinimalSizeSecondBlock) { @@ -228,7 +228,7 @@ TEST(LlvmLibcBlockTest, CanMakeMinimalSizeSecondBlock) { reinterpret_cast(block1->usable_space()) + Block::PREV_FIELD_SIZE); ASSERT_TRUE(result.has_value()); - EXPECT_LE((*result)->outer_size(), sizeof(Block) + alignof(max_align_t)); + EXPECT_LE((*result)->outer_size(), sizeof(Block) + Block::MIN_ALIGN); } TEST(LlvmLibcBlockTest, CanMarkBlockUsed) { @@ -361,18 +361,18 @@ TEST(LlvmLibcBlockTest, Allocate) { if (i > block->inner_size()) continue; - auto info = Block::allocate(block, alignof(max_align_t), i); + auto info = Block::allocate(block, Block::MIN_ALIGN, i); EXPECT_NE(info.block, static_cast(nullptr)); } // Ensure we can allocate a byte at every guaranteeable alignment. - for (size_t i = 1; i < kN / alignof(max_align_t); ++i) { + for (size_t i = 1; i < kN / Block::MIN_ALIGN; ++i) { array bytes; auto result = Block::init(bytes); ASSERT_TRUE(result.has_value()); Block *block = *result; - size_t alignment = i * alignof(max_align_t); + size_t alignment = i * Block::MIN_ALIGN; if (Block::min_size_for_allocation(alignment, 1) > block->inner_size()) continue; @@ -393,14 +393,14 @@ TEST(LlvmLibcBlockTest, AllocateAlreadyAligned) { constexpr size_t SIZE = Block::PREV_FIELD_SIZE + 1; auto [aligned_block, prev, next] = - Block::allocate(block, alignof(max_align_t), SIZE); + Block::allocate(block, Block::MIN_ALIGN, SIZE); // Since this is already aligned, there should be no previous block. EXPECT_EQ(prev, static_cast(nullptr)); // Ensure we the block is aligned and large enough. EXPECT_NE(aligned_block, static_cast(nullptr)); - EXPECT_TRUE(aligned_block->is_usable_space_aligned(alignof(max_align_t))); + EXPECT_TRUE(aligned_block->is_usable_space_aligned(Block::MIN_ALIGN)); EXPECT_GE(aligned_block->inner_size(), SIZE); // Check the next block. @@ -422,9 +422,9 @@ TEST(LlvmLibcBlockTest, AllocateNeedsAlignment) { // Now pick an alignment such that the usable space is not already aligned to // it. We want to explicitly test that the block will split into one before // it. - size_t alignment = alignof(max_align_t); + size_t alignment = Block::MIN_ALIGN; while (block->is_usable_space_aligned(alignment)) - alignment += alignof(max_align_t); + alignment += Block::MIN_ALIGN; auto [aligned_block, prev, next] = Block::allocate(block, alignment, 10); @@ -464,9 +464,9 @@ TEST(LlvmLibcBlockTest, PreviousBlockMergedIfNotFirst) { // Now pick an alignment such that the usable space is not already aligned to // it. We want to explicitly test that the block will split into one before // it. - size_t alignment = alignof(max_align_t); + size_t alignment = Block::MIN_ALIGN; while (newblock->is_usable_space_aligned(alignment)) - alignment += alignof(max_align_t); + alignment += Block::MIN_ALIGN; // Ensure we can allocate in the new block. auto [aligned_block, prev, next] = Block::allocate(newblock, alignment, 1); @@ -501,9 +501,9 @@ TEST(LlvmLibcBlockTest, CanRemergeBlockAllocations) { // Now pick an alignment such that the usable space is not already aligned to // it. We want to explicitly test that the block will split into one before // it. - size_t alignment = alignof(max_align_t); + size_t alignment = Block::MIN_ALIGN; while (block->is_usable_space_aligned(alignment)) - alignment += alignof(max_align_t); + alignment += Block::MIN_ALIGN; auto [aligned_block, prev, next] = Block::allocate(block, alignment, 1); diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp index ea7310d1d0756..9d3a6b612555f 100644 --- a/libc/test/src/__support/freelist_heap_test.cpp +++ b/libc/test/src/__support/freelist_heap_test.cpp @@ -123,12 +123,12 @@ TEST_FOR_EACH_ALLOCATOR(ReturnedPointersAreAligned, 2048) { void *ptr1 = allocator.allocate(1); uintptr_t ptr1_start = reinterpret_cast(ptr1); - EXPECT_EQ(ptr1_start % alignof(max_align_t), static_cast(0)); + EXPECT_EQ(ptr1_start % Block::MIN_ALIGN, static_cast(0)); void *ptr2 = allocator.allocate(1); uintptr_t ptr2_start = reinterpret_cast(ptr2); - EXPECT_EQ(ptr2_start % alignof(max_align_t), static_cast(0)); + EXPECT_EQ(ptr2_start % Block::MIN_ALIGN, static_cast(0)); } TEST_FOR_EACH_ALLOCATOR(CanRealloc, 2048) { diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp index 39292b6a1211b..7017d6b9ebe93 100644 --- a/libc/test/src/__support/freestore_test.cpp +++ b/libc/test/src/__support/freestore_test.cpp @@ -51,8 +51,8 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) { ASSERT_TRUE(maybeBlock.has_value()); Block *largest_small = *maybeBlock; - maybeBlock = largest_small->split( - sizeof(FreeTrie::Node) + Block::PREV_FIELD_SIZE - alignof(max_align_t)); + maybeBlock = largest_small->split(sizeof(FreeTrie::Node) + + Block::PREV_FIELD_SIZE - Block::MIN_ALIGN); ASSERT_TRUE(maybeBlock.has_value()); if (largest_small->inner_size() == smallest->inner_size()) largest_small = smallest;