diff --git a/libc/fuzzing/__support/freelist_heap_fuzz.cpp b/libc/fuzzing/__support/freelist_heap_fuzz.cpp index 0b400cb156491..b342b21895a08 100644 --- a/libc/fuzzing/__support/freelist_heap_fuzz.cpp +++ b/libc/fuzzing/__support/freelist_heap_fuzz.cpp @@ -147,7 +147,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) { // Perform allocation. void *ptr = nullptr; - size_t alignment = alignof(max_align_t); + size_t alignment = Block::MIN_ALIGN; switch (alloc_type) { case AllocType::MALLOC: ptr = heap.allocate(alloc_size); @@ -172,7 +172,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) { alloc_size - alloc.size); alloc.ptr = ptr; alloc.size = alloc_size; - alloc.alignment = alignof(max_align_t); + alloc.alignment = Block::MIN_ALIGN; } break; } @@ -194,8 +194,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t remainder) { if (ptr) { // aligned_allocate should automatically apply a minimum alignment. - if (alignment < alignof(max_align_t)) - alignment = alignof(max_align_t); + if (alignment < Block::MIN_ALIGN) + alignment = Block::MIN_ALIGN; // Check alignment. if (reinterpret_cast(ptr) % alignment) __builtin_trap(); diff --git a/libc/src/__support/block.h b/libc/src/__support/block.h index b0d6576093244..d45af1079a5bc 100644 --- a/libc/src/__support/block.h +++ b/libc/src/__support/block.h @@ -43,8 +43,8 @@ using cpp::optional; /// The blocks store their offsets to the previous and next blocks. The latter /// is also the block's size. /// -/// All blocks have their usable space aligned to some multiple of max_align_t. -/// This also implies that block outer sizes are aligned to max_align_t. +/// All blocks have their usable space aligned to some multiple of MIN_ALIGN. +/// This also implies that block outer sizes are aligned to MIN_ALIGN. /// /// As an example, the diagram below represents two contiguous `Block`s. The /// indices indicate byte offsets: @@ -97,13 +97,17 @@ class Block { static constexpr size_t SIZE_MASK = ~(PREV_FREE_MASK | LAST_MASK); public: + // To ensure block sizes have two lower unused bits, ensure usable space is + // always aligned to at least 4 bytes. (The distances between usable spaces, + // the outer size, is then always also 4-aligned.) + static constexpr size_t MIN_ALIGN = cpp::max(size_t{4}, alignof(max_align_t)); // No copy or move. Block(const Block &other) = delete; Block &operator=(const Block &other) = delete; /// Initializes a given memory region into a first block and a sentinel last /// block. Returns the first block, which has its usable space aligned to - /// max_align_t. + /// MIN_ALIGN. static optional init(ByteSpan region); /// @returns A pointer to a `Block`, given a pointer to the start of the @@ -160,17 +164,17 @@ class Block { /// @returns A pointer to the usable space inside this block. /// - /// Aligned to some multiple of max_align_t. + /// Aligned to some multiple of MIN_ALIGN. LIBC_INLINE cpp::byte *usable_space() { auto *s = reinterpret_cast(this) + sizeof(Block); - LIBC_ASSERT(reinterpret_cast(s) % alignof(max_align_t) == 0 && - "usable space must be aligned to a multiple of max_align_t"); + LIBC_ASSERT(reinterpret_cast(s) % MIN_ALIGN == 0 && + "usable space must be aligned to MIN_ALIGN"); return s; } LIBC_INLINE const cpp::byte *usable_space() const { const auto *s = reinterpret_cast(this) + sizeof(Block); - LIBC_ASSERT(reinterpret_cast(s) % alignof(max_align_t) == 0 && - "usable space must be aligned to a multiple of max_align_t"); + LIBC_ASSERT(reinterpret_cast(s) % MIN_ALIGN == 0 && + "usable space must be aligned to MIN_ALIGN"); return s; } @@ -185,9 +189,9 @@ class Block { /// `new_inner_size`. The remaining space will be returned as a new block, /// with usable space aligned to `usable_space_alignment`. Note that the prev_ /// field of the next block counts as part of the inner size of the block. - /// `usable_space_alignment` must be a multiple of max_align_t. + /// `usable_space_alignment` must be a multiple of MIN_ALIGN. optional split(size_t new_inner_size, - size_t usable_space_alignment = alignof(max_align_t)); + size_t usable_space_alignment = MIN_ALIGN); /// Merges this block with the one that comes after it. bool merge_next(); @@ -228,13 +232,11 @@ class Block { LIBC_INLINE Block(size_t outer_size, bool is_last) : next_(outer_size) { // Last blocks are not usable, so they need not have sizes aligned to - // max_align_t. Their lower bits must still be free, so they must be aligned - // to Block. - LIBC_ASSERT( - outer_size % (is_last ? alignof(Block) : alignof(max_align_t)) == 0 && - "block sizes must be aligned"); - LIBC_ASSERT(is_usable_space_aligned(alignof(max_align_t)) && - "usable space must be aligned to a multiple of max_align_t"); + // MIN_ALIGN. + LIBC_ASSERT(outer_size % (is_last ? alignof(Block) : MIN_ALIGN) == 0 && + "block sizes must be aligned"); + LIBC_ASSERT(is_usable_space_aligned(MIN_ALIGN) && + "usable space must be aligned to a multiple of MIN_ALIGN"); if (is_last) next_ |= LAST_MASK; } @@ -249,11 +251,10 @@ class Block { // Returns 0 if there is no such size. LIBC_INLINE static size_t min_size_for_allocation(size_t alignment, size_t size) { - LIBC_ASSERT(alignment >= alignof(max_align_t) && - alignment % alignof(max_align_t) == 0 && - "alignment must be multiple of max_align_t"); + LIBC_ASSERT(alignment >= MIN_ALIGN && alignment % MIN_ALIGN == 0 && + "alignment must be multiple of MIN_ALIGN"); - if (alignment == alignof(max_align_t)) + if (alignment == MIN_ALIGN) return size; // We must create a new block inside this one (splitting). This requires a @@ -274,7 +275,7 @@ class Block { // So the maximum distance would be G - L. As a special case, if L is 1 // (unaligned), the max distance is G - 1. // - // This block's usable space is aligned to max_align_t >= Block. With zero + // This block's usable space is aligned to MIN_ALIGN >= Block. With zero // padding, the next block's usable space is sizeof(Block) past it, which is // a point aligned to Block. Thus the max padding needed is alignment - // alignof(Block). @@ -309,13 +310,15 @@ class Block { static BlockInfo allocate(Block *block, size_t alignment, size_t size); // These two functions may wrap around. - LIBC_INLINE static uintptr_t next_possible_block_start( - uintptr_t ptr, size_t usable_space_alignment = alignof(max_align_t)) { + LIBC_INLINE static uintptr_t + next_possible_block_start(uintptr_t ptr, + size_t usable_space_alignment = MIN_ALIGN) { return align_up(ptr + sizeof(Block), usable_space_alignment) - sizeof(Block); } - LIBC_INLINE static uintptr_t prev_possible_block_start( - uintptr_t ptr, size_t usable_space_alignment = alignof(max_align_t)) { + LIBC_INLINE static uintptr_t + prev_possible_block_start(uintptr_t ptr, + size_t usable_space_alignment = MIN_ALIGN) { return align_down(ptr, usable_space_alignment) - sizeof(Block); } @@ -360,9 +363,6 @@ class Block { static constexpr size_t PREV_FIELD_SIZE = sizeof(prev_); }; -static_assert(alignof(Block) >= 4, - "at least 2 bits must be available in block sizes for flags"); - LIBC_INLINE optional Block::init(ByteSpan region) { if (!region.data()) @@ -394,8 +394,8 @@ optional Block::init(ByteSpan region) { LIBC_INLINE Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size) { - LIBC_ASSERT(alignment % alignof(max_align_t) == 0 && - "alignment must be a multiple of max_align_t"); + LIBC_ASSERT(alignment % MIN_ALIGN == 0 && + "alignment must be a multiple of MIN_ALIGN"); BlockInfo info{block, /*prev=*/nullptr, /*next=*/nullptr}; @@ -430,8 +430,8 @@ Block::BlockInfo Block::allocate(Block *block, size_t alignment, size_t size) { LIBC_INLINE optional Block::split(size_t new_inner_size, size_t usable_space_alignment) { - LIBC_ASSERT(usable_space_alignment % alignof(max_align_t) == 0 && - "alignment must be a multiple of max_align_t"); + LIBC_ASSERT(usable_space_alignment % MIN_ALIGN == 0 && + "alignment must be a multiple of MIN_ALIGN"); if (used()) return {}; @@ -445,8 +445,8 @@ optional Block::split(size_t new_inner_size, if (next_block_start < start) return {}; size_t new_outer_size = next_block_start - start; - LIBC_ASSERT(new_outer_size % alignof(max_align_t) == 0 && - "new size must be aligned to max_align_t"); + LIBC_ASSERT(new_outer_size % MIN_ALIGN == 0 && + "new size must be aligned to MIN_ALIGN"); if (outer_size() < new_outer_size || outer_size() - new_outer_size < sizeof(Block)) diff --git a/libc/src/__support/freelist_heap.h b/libc/src/__support/freelist_heap.h index d58685194aeb8..2e0e371f96e8c 100644 --- a/libc/src/__support/freelist_heap.h +++ b/libc/src/__support/freelist_heap.h @@ -108,7 +108,7 @@ LIBC_INLINE void *FreeListHeap::allocate_impl(size_t alignment, size_t size) { } LIBC_INLINE void *FreeListHeap::allocate(size_t size) { - return allocate_impl(alignof(max_align_t), size); + return allocate_impl(Block::MIN_ALIGN, size); } LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment, @@ -121,8 +121,8 @@ LIBC_INLINE void *FreeListHeap::aligned_allocate(size_t alignment, if (size % alignment != 0) return nullptr; - // The minimum alignment supported by Block is max_align_t. - alignment = cpp::max(alignment, alignof(max_align_t)); + // The minimum alignment supported by Block is MIN_ALIGN. + alignment = cpp::max(alignment, Block::MIN_ALIGN); return allocate_impl(alignment, size); } diff --git a/libc/src/__support/freestore.h b/libc/src/__support/freestore.h index 09f2479debb36..2dcb4b10b93d5 100644 --- a/libc/src/__support/freestore.h +++ b/libc/src/__support/freestore.h @@ -41,11 +41,11 @@ class FreeStore { private: static constexpr size_t MIN_OUTER_SIZE = - align_up(sizeof(Block) + sizeof(FreeList::Node), alignof(max_align_t)); + align_up(sizeof(Block) + sizeof(FreeList::Node), Block::MIN_ALIGN); static constexpr size_t MIN_LARGE_OUTER_SIZE = - align_up(sizeof(Block) + sizeof(FreeTrie::Node), alignof(max_align_t)); + align_up(sizeof(Block) + sizeof(FreeTrie::Node), Block::MIN_ALIGN); static constexpr size_t NUM_SMALL_SIZES = - (MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / alignof(max_align_t); + (MIN_LARGE_OUTER_SIZE - MIN_OUTER_SIZE) / Block::MIN_ALIGN; LIBC_INLINE static bool too_small(Block *block) { return block->outer_size() < MIN_OUTER_SIZE; @@ -98,8 +98,7 @@ LIBC_INLINE Block *FreeStore::remove_best_fit(size_t size) { LIBC_INLINE FreeList &FreeStore::small_list(Block *block) { LIBC_ASSERT(is_small(block) && "only legal for small blocks"); - return small_lists[(block->outer_size() - MIN_OUTER_SIZE) / - alignof(max_align_t)]; + return small_lists[(block->outer_size() - MIN_OUTER_SIZE) / Block::MIN_ALIGN]; } LIBC_INLINE FreeList *FreeStore::find_best_small_fit(size_t size) { diff --git a/libc/test/src/__support/block_test.cpp b/libc/test/src/__support/block_test.cpp index 904ac5c66994d..3029cde834a5d 100644 --- a/libc/test/src/__support/block_test.cpp +++ b/libc/test/src/__support/block_test.cpp @@ -22,14 +22,14 @@ using LIBC_NAMESPACE::cpp::span; TEST(LlvmLibcBlockTest, CanCreateSingleAlignedBlock) { constexpr size_t kN = 1024; - alignas(max_align_t) array bytes; + alignas(Block::MIN_ALIGN) array bytes; auto result = Block::init(bytes); ASSERT_TRUE(result.has_value()); Block *block = *result; EXPECT_EQ(reinterpret_cast(block) % alignof(Block), size_t{0}); - EXPECT_TRUE(block->is_usable_space_aligned(alignof(max_align_t))); + EXPECT_TRUE(block->is_usable_space_aligned(Block::MIN_ALIGN)); Block *last = block->next(); ASSERT_NE(last, static_cast(nullptr)); @@ -52,7 +52,7 @@ TEST(LlvmLibcBlockTest, CanCreateUnalignedSingleBlock) { constexpr size_t kN = 1024; // Force alignment, so we can un-force it below - alignas(max_align_t) array bytes; + alignas(Block::MIN_ALIGN) array bytes; span aligned(bytes); auto result = Block::init(aligned.subspan(1)); @@ -60,7 +60,7 @@ TEST(LlvmLibcBlockTest, CanCreateUnalignedSingleBlock) { Block *block = *result; EXPECT_EQ(reinterpret_cast(block) % alignof(Block), size_t{0}); - EXPECT_TRUE(block->is_usable_space_aligned(alignof(max_align_t))); + EXPECT_TRUE(block->is_usable_space_aligned(Block::MIN_ALIGN)); Block *last = block->next(); ASSERT_NE(last, static_cast(nullptr)); @@ -98,7 +98,7 @@ TEST(LlvmLibcBlockTest, CanSplitBlock) { EXPECT_EQ(block2->outer_size(), orig_size - block1->outer_size()); EXPECT_FALSE(block2->used()); EXPECT_EQ(reinterpret_cast(block2) % alignof(Block), size_t{0}); - EXPECT_TRUE(block2->is_usable_space_aligned(alignof(max_align_t))); + EXPECT_TRUE(block2->is_usable_space_aligned(Block::MIN_ALIGN)); EXPECT_EQ(block1->next(), block2); EXPECT_EQ(block2->prev_free(), block1); @@ -124,7 +124,7 @@ TEST(LlvmLibcBlockTest, CanSplitBlockUnaligned) { EXPECT_EQ(block2->outer_size(), orig_size - block1->outer_size()); EXPECT_FALSE(block2->used()); EXPECT_EQ(reinterpret_cast(block2) % alignof(Block), size_t{0}); - EXPECT_TRUE(block2->is_usable_space_aligned(alignof(max_align_t))); + EXPECT_TRUE(block2->is_usable_space_aligned(Block::MIN_ALIGN)); EXPECT_EQ(block1->next(), block2); EXPECT_EQ(block2->prev_free(), block1); @@ -211,7 +211,7 @@ TEST(LlvmLibcBlockTest, CanMakeMinimalSizeFirstBlock) { result = block->split(0); ASSERT_TRUE(result.has_value()); - EXPECT_LE(block->outer_size(), sizeof(Block) + alignof(max_align_t)); + EXPECT_LE(block->outer_size(), sizeof(Block) + Block::MIN_ALIGN); } TEST(LlvmLibcBlockTest, CanMakeMinimalSizeSecondBlock) { @@ -228,7 +228,7 @@ TEST(LlvmLibcBlockTest, CanMakeMinimalSizeSecondBlock) { reinterpret_cast(block1->usable_space()) + Block::PREV_FIELD_SIZE); ASSERT_TRUE(result.has_value()); - EXPECT_LE((*result)->outer_size(), sizeof(Block) + alignof(max_align_t)); + EXPECT_LE((*result)->outer_size(), sizeof(Block) + Block::MIN_ALIGN); } TEST(LlvmLibcBlockTest, CanMarkBlockUsed) { @@ -361,18 +361,18 @@ TEST(LlvmLibcBlockTest, Allocate) { if (i > block->inner_size()) continue; - auto info = Block::allocate(block, alignof(max_align_t), i); + auto info = Block::allocate(block, Block::MIN_ALIGN, i); EXPECT_NE(info.block, static_cast(nullptr)); } // Ensure we can allocate a byte at every guaranteeable alignment. - for (size_t i = 1; i < kN / alignof(max_align_t); ++i) { + for (size_t i = 1; i < kN / Block::MIN_ALIGN; ++i) { array bytes; auto result = Block::init(bytes); ASSERT_TRUE(result.has_value()); Block *block = *result; - size_t alignment = i * alignof(max_align_t); + size_t alignment = i * Block::MIN_ALIGN; if (Block::min_size_for_allocation(alignment, 1) > block->inner_size()) continue; @@ -393,14 +393,14 @@ TEST(LlvmLibcBlockTest, AllocateAlreadyAligned) { constexpr size_t SIZE = Block::PREV_FIELD_SIZE + 1; auto [aligned_block, prev, next] = - Block::allocate(block, alignof(max_align_t), SIZE); + Block::allocate(block, Block::MIN_ALIGN, SIZE); // Since this is already aligned, there should be no previous block. EXPECT_EQ(prev, static_cast(nullptr)); // Ensure we the block is aligned and large enough. EXPECT_NE(aligned_block, static_cast(nullptr)); - EXPECT_TRUE(aligned_block->is_usable_space_aligned(alignof(max_align_t))); + EXPECT_TRUE(aligned_block->is_usable_space_aligned(Block::MIN_ALIGN)); EXPECT_GE(aligned_block->inner_size(), SIZE); // Check the next block. @@ -422,9 +422,9 @@ TEST(LlvmLibcBlockTest, AllocateNeedsAlignment) { // Now pick an alignment such that the usable space is not already aligned to // it. We want to explicitly test that the block will split into one before // it. - size_t alignment = alignof(max_align_t); + size_t alignment = Block::MIN_ALIGN; while (block->is_usable_space_aligned(alignment)) - alignment += alignof(max_align_t); + alignment += Block::MIN_ALIGN; auto [aligned_block, prev, next] = Block::allocate(block, alignment, 10); @@ -464,9 +464,9 @@ TEST(LlvmLibcBlockTest, PreviousBlockMergedIfNotFirst) { // Now pick an alignment such that the usable space is not already aligned to // it. We want to explicitly test that the block will split into one before // it. - size_t alignment = alignof(max_align_t); + size_t alignment = Block::MIN_ALIGN; while (newblock->is_usable_space_aligned(alignment)) - alignment += alignof(max_align_t); + alignment += Block::MIN_ALIGN; // Ensure we can allocate in the new block. auto [aligned_block, prev, next] = Block::allocate(newblock, alignment, 1); @@ -501,9 +501,9 @@ TEST(LlvmLibcBlockTest, CanRemergeBlockAllocations) { // Now pick an alignment such that the usable space is not already aligned to // it. We want to explicitly test that the block will split into one before // it. - size_t alignment = alignof(max_align_t); + size_t alignment = Block::MIN_ALIGN; while (block->is_usable_space_aligned(alignment)) - alignment += alignof(max_align_t); + alignment += Block::MIN_ALIGN; auto [aligned_block, prev, next] = Block::allocate(block, alignment, 1); diff --git a/libc/test/src/__support/freelist_heap_test.cpp b/libc/test/src/__support/freelist_heap_test.cpp index ea7310d1d0756..9d3a6b612555f 100644 --- a/libc/test/src/__support/freelist_heap_test.cpp +++ b/libc/test/src/__support/freelist_heap_test.cpp @@ -123,12 +123,12 @@ TEST_FOR_EACH_ALLOCATOR(ReturnedPointersAreAligned, 2048) { void *ptr1 = allocator.allocate(1); uintptr_t ptr1_start = reinterpret_cast(ptr1); - EXPECT_EQ(ptr1_start % alignof(max_align_t), static_cast(0)); + EXPECT_EQ(ptr1_start % Block::MIN_ALIGN, static_cast(0)); void *ptr2 = allocator.allocate(1); uintptr_t ptr2_start = reinterpret_cast(ptr2); - EXPECT_EQ(ptr2_start % alignof(max_align_t), static_cast(0)); + EXPECT_EQ(ptr2_start % Block::MIN_ALIGN, static_cast(0)); } TEST_FOR_EACH_ALLOCATOR(CanRealloc, 2048) { diff --git a/libc/test/src/__support/freestore_test.cpp b/libc/test/src/__support/freestore_test.cpp index 39292b6a1211b..7017d6b9ebe93 100644 --- a/libc/test/src/__support/freestore_test.cpp +++ b/libc/test/src/__support/freestore_test.cpp @@ -51,8 +51,8 @@ TEST(LlvmLibcFreeStore, RemoveBestFit) { ASSERT_TRUE(maybeBlock.has_value()); Block *largest_small = *maybeBlock; - maybeBlock = largest_small->split( - sizeof(FreeTrie::Node) + Block::PREV_FIELD_SIZE - alignof(max_align_t)); + maybeBlock = largest_small->split(sizeof(FreeTrie::Node) + + Block::PREV_FIELD_SIZE - Block::MIN_ALIGN); ASSERT_TRUE(maybeBlock.has_value()); if (largest_small->inner_size() == smallest->inner_size()) largest_small = smallest;