replace chunks bool array, with bit fields

Signed-off-by: Łukasz Plewa <[email protected]>

Author: bratpiorka

src/pool/pool_disjoint.c

@@ -74,7 +74,13 @@ static slab_t *create_slab(bucket_t *bucket) {
     umf_result_t res = UMF_RESULT_SUCCESS;
     umf_memory_provider_handle_t provider = bucket->pool->provider;
 
-    slab_t *slab = umf_ba_global_alloc(sizeof(*slab));
+    size_t num_chunks_total =
+        utils_max(bucket_slab_min_size(bucket) / bucket->size, 1);
+    size_t chunks_size_in_64_increments =
+        num_chunks_total / 8 + (num_chunks_total % 8 != 0);
+
+    slab_t *slab = umf_ba_global_alloc(
+        sizeof(*slab) + chunks_size_in_64_increments * sizeof(slab->chunks[0]));
     if (slab == NULL) {
         LOG_ERR("allocation of new slab failed!");
         return NULL;
@@ -87,15 +93,10 @@ static slab_t *create_slab(bucket_t *bucket) {
     slab->iter.val = slab;
     slab->iter.prev = slab->iter.next = NULL;
 
-    slab->num_chunks_total =
-        utils_max(bucket_slab_min_size(bucket) / bucket->size, 1);
-    slab->chunks =
-        umf_ba_global_alloc(sizeof(*slab->chunks) * slab->num_chunks_total);
-    if (slab->chunks == NULL) {
-        LOG_ERR("allocation of slab chunks failed!");
-        goto free_slab;
-    }
-    memset(slab->chunks, 0, sizeof(*slab->chunks) * slab->num_chunks_total);
+    slab->num_chunks_total = num_chunks_total;
+
+    memset(slab->chunks, 0,
+           chunks_size_in_64_increments * sizeof(slab->chunks[0]));
 
     // if slab_min_size is not a multiple of bucket size, we would have some
     // padding at the end of the slab
@@ -107,7 +108,7 @@ static slab_t *create_slab(bucket_t *bucket) {
     res = umfMemoryProviderAlloc(provider, slab->slab_size, 0, &slab->mem_ptr);
     if (res != UMF_RESULT_SUCCESS) {
         LOG_ERR("allocation of slab data failed!");
-        goto free_slab_chunks;
+        goto free_slab;
     }
 
     // raw allocation is not available for user so mark it as inaccessible
@@ -116,9 +117,6 @@ static slab_t *create_slab(bucket_t *bucket) {
     LOG_DEBUG("bucket: %p, slab_size: %zu", (void *)bucket, slab->slab_size);
     return slab;
 
-free_slab_chunks:
-    umf_ba_global_free(slab->chunks);
-
 free_slab:
     umf_ba_global_free(slab);
     return NULL;
@@ -135,25 +133,32 @@ static void destroy_slab(slab_t *slab) {
         LOG_ERR("deallocation of slab data failed!");
     }
 
-    umf_ba_global_free(slab->chunks);
     umf_ba_global_free(slab);
 }
 
-// return the index of the first available chunk, SIZE_MAX otherwise
 static size_t slab_find_first_available_chunk_idx(const slab_t *slab) {
-    // use the first free chunk index as a hint for the search
-    for (bool *chunk = slab->chunks + slab->first_free_chunk_idx;
-         chunk != slab->chunks + slab->num_chunks_total; chunk++) {
-
-        // false means not used
-        if (*chunk == false) {
-            size_t idx = chunk - slab->chunks;
-            LOG_DEBUG("idx: %zu", idx);
-            return idx;
+    // Calculate the number of 64-bit words needed.
+    size_t num_words = (slab->num_chunks_total + 63) / 64;
+    for (size_t i = 0; i < num_words; i++) {
+        // Invert the word: free bits (0 in the allocated mask) become 1.
+        uint64_t word = ~(slab->chunks[i]);
+
+        // For the final word, clear out bits that exceed num_chunks_total.
+        if (i == num_words - 1) {
+            size_t bits_in_last_word = slab->num_chunks_total - (i * 64);
+            if (bits_in_last_word < 64) {
+                word &= (((uint64_t)1 << bits_in_last_word) - 1);
+            }
+        }
+        if (word != 0) {
+            size_t bit_index = utils_get_rightmost_set_bit_pos(word);
+            size_t free_chunk = i * 64 + bit_index;
+            if (free_chunk < slab->num_chunks_total) {
+                return free_chunk;
+            }
         }
     }
-
-    LOG_DEBUG("idx: SIZE_MAX");
+    // No free chunk was found.
     return SIZE_MAX;
 }
 
@@ -166,7 +171,7 @@ static void *slab_get_chunk(slab_t *slab) {
         (void *)((uintptr_t)slab->mem_ptr + chunk_idx * slab->bucket->size);
 
     // mark chunk as used
-    slab->chunks[chunk_idx] = true;
+    slab_set_chunk_bit(slab, chunk_idx, true);
     slab->num_chunks_allocated += 1;
 
     // use the found index as the next hint
@@ -194,8 +199,8 @@ static void slab_free_chunk(slab_t *slab, void *ptr) {
     size_t chunk_idx = ptr_diff / slab->bucket->size;
 
     // Make sure that the chunk was allocated
-    assert(slab->chunks[chunk_idx] && "double free detected");
-    slab->chunks[chunk_idx] = false;
+    assert(slab_read_chunk_bit(slab, chunk_idx) && "double free detected");
+    slab_set_chunk_bit(slab, chunk_idx, false);
     slab->num_chunks_allocated -= 1;
 
     if (chunk_idx < slab->first_free_chunk_idx) {
@@ -467,7 +472,7 @@ static size_t size_to_idx(disjoint_pool_t *pool, size_t size) {
     }
 
     // get the position of the leftmost set bit
-    size_t position = getLeftmostSetBitPos(size);
+    size_t position = utils_get_leftmost_set_bit_pos(size);
 
     bool is_power_of_2 = 0 == (size & (size - 1));
     bool larger_than_halfway_between_powers_of_2 =
@@ -623,7 +628,8 @@ umf_result_t disjoint_pool_initialize(umf_memory_provider_handle_t provider,
     Size1 = utils_max(Size1, UMF_DISJOINT_POOL_MIN_BUCKET_DEFAULT_SIZE);
 
     // Calculate the exponent for min_bucket_size used for finding buckets.
-    disjoint_pool->min_bucket_size_exp = (size_t)log2Utils(Size1);
+    disjoint_pool->min_bucket_size_exp =
+        (size_t)utils_get_leftmost_set_bit_pos(Size1);
     disjoint_pool->default_shared_limits =
         umfDisjointPoolSharedLimitsCreate(SIZE_MAX);
 

src/pool/pool_disjoint_internal.h

@@ -81,9 +81,6 @@ typedef struct slab_t {
     void *mem_ptr;
     size_t slab_size;
 
-    // Represents the current state of each chunk: if the bit is set, the
-    // chunk is allocated; otherwise, the chunk is free for allocation
-    bool *chunks;
     size_t num_chunks_total;
 
     // Total number of allocated chunks at the moment.
@@ -98,6 +95,10 @@ typedef struct slab_t {
     // Store iterator to the corresponding node in avail/unavail list
     // to achieve O(1) removal
     slab_list_item_t iter;
+
+    // Represents the current state of each chunk: if the bit is set, the
+    // chunk is allocated; otherwise, the chunk is free for allocation
+    uint64_t chunks[];
 } slab_t;
 
 typedef struct umf_disjoint_pool_shared_limits_t {
@@ -158,4 +159,24 @@ typedef struct disjoint_pool_t {
     size_t provider_min_page_size;
 } disjoint_pool_t;
 
+static inline void slab_set_chunk_bit(slab_t *slab, size_t index, bool value) {
+    assert(index < slab->num_chunks_total && "Index out of range");
+
+    size_t word_index = index / 64;
+    unsigned bit_index = index % 64;
+    if (value) {
+        slab->chunks[word_index] |= (1ULL << bit_index);
+    } else {
+        slab->chunks[word_index] &= ~(1ULL << bit_index);
+    }
+}
+
+static inline int slab_read_chunk_bit(const slab_t *slab, size_t index) {
+    assert(index < slab->num_chunks_total && "Index out of range");
+
+    size_t word_index = index / 64;
+    unsigned bit_index = index % 64;
+    return (slab->chunks[word_index] >> bit_index) & 1;
+}
+
 #endif // UMF_POOL_DISJOINT_INTERNAL_H

test/pools/disjoint_pool.cpp

@@ -113,8 +113,8 @@ TEST_F(test, internals) {
     EXPECT_GE(slab->num_chunks_total, slab->slab_size / bucket->size);
 
     // check allocation in slab
-    EXPECT_EQ(slab->chunks[0], true);
-    EXPECT_EQ(slab->chunks[1], false);
+    EXPECT_EQ(slab_read_chunk_bit(slab, 0), true);
+    EXPECT_EQ(slab_read_chunk_bit(slab, 1), false);
     EXPECT_EQ(slab->first_free_chunk_idx, 1);
 
     // TODO: