33 *
44 * A simple arena allocator that can be freed all at once.
55 *
6- * This allocator maintains a linked list of pages, which come in two flavours:
7- * 1. Small allocation pages, which are the same size as the system page size.
8- * 2. Large allocation pages, which are the exact size requested, for sizes greater than the small page size.
9- *
10- * Small allocations always fit into the unused space at the end of the "head" page. If there isn't enough room, a new
11- * page is allocated, and the small allocation is placed at its start. This approach wastes that unused slack at the
12- * end of the previous page, but it means that allocations are instant and never scan the linked list to find a gap.
13- *
146 * This allocator doesn't support freeing individual allocations. Only the whole arena can be freed at once at the end.
157 */
168
2012#include <stdlib.h>
2113#include <string.h> // for memset()
2214#include <stdint.h>
15+ #include <inttypes.h>
2316
2417#ifdef _WIN32
2518 #include <windows.h>
2619#else
2720 #include <unistd.h>
2821 #include <sys/types.h>
22+ #include <sys/mman.h>
2923#endif
3024
31- typedef struct rbs_allocator_page {
32- uint32_t payload_size ;
33-
34- // The offset of the next available byte.
35- uint32_t offset ;
36-
37- // The previously allocated page, or NULL if this is the first page.
38- struct rbs_allocator_page * next ;
25+ #if defined(__APPLE__ ) || defined(__FreeBSD__ ) || defined(__OpenBSD__ ) || defined(__sun )
26+ #define MAP_ANONYMOUS MAP_ANON
27+ #endif
3928
40- // The variably-sized payload of the page.
41- char payload [];
42- } rbs_allocator_page_t ;
4329
44- // This allocator's normal pages have the same size as the system memory pages, consisting of a fixed-size header
45- // (sizeof(rbs_allocator_page_t)) followed `default_page_payload_size` bytes of payload.
46- // TODO: When we have real-world usage data, we can tune this to use a smaller number of larger pages.
47- static size_t default_page_payload_size = 0 ;
48- static uint32_t large_page_flag = UINT32_MAX ;
30+ struct rbs_allocator {
31+ uintptr_t heap_ptr ;
32+ uintptr_t size ;
33+ };
4934
50- static size_t get_system_page_size () {
35+ static size_t get_system_page_size (void ) {
5136#ifdef _WIN32
5237 SYSTEM_INFO si ;
5338 GetSystemInfo (& si );
@@ -59,72 +44,74 @@ static size_t get_system_page_size() {
5944#endif
6045}
6146
62- void rbs__init_arena_allocator (void ) {
63- const size_t system_page_size = get_system_page_size ();
64-
65- // The size of a struct that ends with a flexible array member is the size of the struct without the
66- // flexible array member. https://en.wikipedia.org/wiki/Flexible_array_member#Effect_on_struct_size_and_padding
67- const size_t page_header_size = sizeof (rbs_allocator_page_t );
68- default_page_payload_size = system_page_size - page_header_size ;
47+ static void * map_memory (size_t size ) {
48+ #ifdef _WIN32
49+ LPVOID result = VirtualAlloc (NULL , size , MEM_RESERVE | MEM_COMMIT , PAGE_READWRITE );
50+ rbs_assert (result != NULL , "VirtualAlloc failed" );
51+ #else
52+ void * result = mmap (NULL , size , PROT_READ | PROT_WRITE ,
53+ MAP_PRIVATE | MAP_ANONYMOUS , -1 , 0 );
54+ rbs_assert (result != MAP_FAILED , "mmap failed" );
55+ #endif
56+ return result ;
6957}
7058
71- // Returns the number of bytes needed to pad the given pointer up to the nearest multiple of the given `alignment`.
72- static size_t needed_padding (void * ptr , size_t alignment ) {
73- const uintptr_t addr = (uintptr_t ) ptr ;
74- const uintptr_t aligned_addr = (addr + (alignment - 1 )) & - alignment ;
75- return (aligned_addr - addr );
59+ static void destroy_memory (void * memory , size_t size ) {
60+ #ifdef _WIN32
61+ VirtualFree (memory , 0 , MEM_RELEASE );
62+ #else
63+ munmap (memory , size );
64+ #endif
7665}
7766
78- static rbs_allocator_page_t * rbs_allocator_page_new (size_t payload_size ) {
79- const size_t page_header_size = sizeof (rbs_allocator_page_t );
80-
81- rbs_allocator_page_t * page = calloc (1 , page_header_size + payload_size );
82- * page = (rbs_allocator_page_t ) {
83- .payload_size = (uint32_t ) payload_size ,
84- .offset = page_header_size ,
85- .next = NULL ,
86- };
87- return page ;
67+ static void guard_page (void * memory , size_t page_size ) {
68+ #ifdef _WIN32
69+ DWORD old_protect_ ;
70+ BOOL result = VirtualProtect (memory , page_size , PAGE_NOACCESS , & old_protect_ );
71+ rbs_assert (result != 0 , "VirtualProtect failed" );
72+ #else
73+ int result = mprotect (memory , page_size , PROT_NONE );
74+ rbs_assert (result == 0 , "mprotect failed" );
75+ #endif
8876}
8977
90- static rbs_allocator_page_t * rbs_allocator_page_new_default (void ) {
91- return rbs_allocator_page_new (default_page_payload_size );
78+ static size_t rbs_allocator_default_mem (void ) {
79+ size_t kib = 1024 ;
80+ size_t mib = kib * 1024 ;
81+ size_t gib = mib * 1024 ;
82+ return 4 * gib ;
9283}
9384
94- static rbs_allocator_page_t * rbs_allocator_page_new_large (size_t payload_size ) {
95- rbs_allocator_page_t * page = rbs_allocator_page_new (payload_size );
96-
97- page -> offset = large_page_flag ;
98-
99- return page ;
85+ static inline bool is_power_of_two (uintptr_t value ) {
86+ return value > 0 && (value & (value - 1 )) == 0 ;
10087}
10188
102- // Attempts to allocate `size` bytes from `page`, returning NULL if there is insufficient space.
103- static void * rbs_allocator_page_attempt_alloc (rbs_allocator_page_t * page , size_t size , size_t alignment ) {
104- const size_t alignment_padding = needed_padding (page -> payload + page -> offset , alignment );
105-
106- const size_t remaining_size = page -> payload_size - page -> offset ;
107- const size_t needed_size = alignment_padding + size ;
108- if (remaining_size < needed_size ) return NULL ; // Not enough space in this page.
109-
110- void * ptr = page -> payload + page -> offset + alignment_padding ;
111- page -> offset += needed_size ;
112- return ptr ;
89+ // Align `val' to nearest multiple of `alignment'.
90+ static uintptr_t align (uintptr_t size , uintptr_t alignment ) {
91+ rbs_assert (is_power_of_two (alignment ), "alignment is not a power of two" );
92+ return (size + alignment - 1 ) & ~(alignment - 1 );
11393}
11494
115- void rbs_allocator_init (rbs_allocator_t * allocator ) {
116- * allocator = (rbs_allocator_t ) {
117- .page = rbs_allocator_page_new_default (),
95+ rbs_allocator_t * rbs_allocator_init (void ) {
96+ size_t size = rbs_allocator_default_mem ();
97+ size_t page_size = get_system_page_size ();
98+ size = align (size , page_size );
99+ void * mem = map_memory (size + page_size );
100+ // Guard page; remove range checks in alloc fast path and hard fail if we
101+ // consume all memory
102+ void * last_page = (char * ) mem + size ;
103+ guard_page (last_page , page_size );
104+ uintptr_t start = (uintptr_t ) mem ;
105+ rbs_allocator_t header = (rbs_allocator_t ) {
106+ .heap_ptr = start + sizeof header ,
107+ .size = size + page_size ,
118108 };
109+ memcpy (mem , & header , sizeof header );
110+ return (rbs_allocator_t * ) mem ;
119111}
120112
121113void rbs_allocator_free (rbs_allocator_t * allocator ) {
122- rbs_allocator_page_t * page = allocator -> page ;
123- while (page ) {
124- rbs_allocator_page_t * next = page -> next ;
125- free (page );
126- page = next ;
127- }
114+ destroy_memory ((void * ) allocator , allocator -> size );
128115}
129116
130117// Allocates `new_size` bytes from `allocator`, aligned to an `alignment`-byte boundary.
@@ -139,52 +126,20 @@ void *rbs_allocator_realloc_impl(rbs_allocator_t *allocator, void *ptr, size_t o
139126// Allocates `size` bytes from `allocator`, aligned to an `alignment`-byte boundary.
140127void * rbs_allocator_malloc_impl (rbs_allocator_t * allocator , size_t size , size_t alignment ) {
141128 rbs_assert (size % alignment == 0 , "size must be a multiple of the alignment. size: %zu, alignment: %zu" , size , alignment );
142-
143- if (default_page_payload_size < size ) { // Big allocation, give it its own page.
144- // How much we need to pad the new page's payload in order to get an aligned pointer
145- // hack?
146- const size_t alignment_padding = needed_padding ((void * ) (sizeof (rbs_allocator_page_t ) + size ), alignment );
147-
148- rbs_allocator_page_t * new_page = rbs_allocator_page_new_large (alignment_padding + size );
149-
150- // This simple allocator can only put small allocations into the head page.
151- // Naively prepending this large allocation page to the head of the allocator before the previous head page
152- // would waste the remaining space in the head page.
153- // So instead, we'll splice in the large page *after* the head page.
154- //
155- // +-------+ +-----------+ +-----------+
156- // | arena | | head page | | new_page |
157- // |-------| |-----------+ |-----------+
158- // | *page |--->| size | +--->| size | +---> ... previous tail
159- // +-------+ | offset | | | offset | |
160- // | *next ----+---+ | *next ----+---+
161- // | ... | | ... |
162- // +-----------+ +-----------+
163- //
164- new_page -> next = allocator -> page -> next ;
165- allocator -> page -> next = new_page ;
166-
167- return new_page -> payload + alignment_padding ;
168- }
169-
170- void * p = rbs_allocator_page_attempt_alloc (allocator -> page , size , alignment );
171- if (p != NULL ) return p ;
172-
173- // Not enough space. Allocate a new page and prepend it to the allocator's linked list.
174- rbs_allocator_page_t * new_page = rbs_allocator_page_new_default ();
175- new_page -> next = allocator -> page ;
176- allocator -> page = new_page ;
177-
178- p = rbs_allocator_page_attempt_alloc (new_page , size , alignment );
179- rbs_assert (p != NULL , "Failed to allocate a new allocator page" );
180- return p ;
129+ uintptr_t aligned = align (allocator -> heap_ptr , alignment );
130+ allocator -> heap_ptr = aligned + size ;
131+ return (void * ) aligned ;
181132}
182133
183134// Note: This will eagerly fill with zeroes, unlike `calloc()` which can map a page in a page to be zeroed lazily.
184135// It's assumed that callers to this function will immediately write to the allocated memory, anyway.
185136void * rbs_allocator_calloc_impl (rbs_allocator_t * allocator , size_t count , size_t size , size_t alignment ) {
186137 void * p = rbs_allocator_malloc_many_impl (allocator , count , size , alignment );
138+ #if defined(__linux__ )
139+ // mmap with MAP_ANONYMOUS gives zero-filled pages.
140+ #else
187141 memset (p , 0 , count * size );
142+ #endif
188143 return p ;
189144}
190145
0 commit comments