make shared limits C structure

Author: bratpiorka

src/pool/pool_disjoint.c

@@ -51,6 +51,23 @@ extern "C" {
 #endif
 }
 
+typedef struct umf_disjoint_pool_shared_limits_t {
+    size_t max_size;
+    _Atomic(size_t) total_size;
+} umf_disjoint_pool_shared_limits_t;
+
+umf_disjoint_pool_shared_limits_t *shared_limits_create(size_t max_size) {
+    umf_disjoint_pool_shared_limits_t *ptr =
+        umf_ba_global_alloc(sizeof(umf_disjoint_pool_shared_limits_t));
+    ptr->max_size = max_size;
+    ptr->total_size = 0;
+    return ptr;
+}
+
+void shared_limits_destroy(umf_disjoint_pool_shared_limits_t *shared_limits) {
+    umf_ba_global_free(shared_limits);
+}
+
 size_t bucket_get_size(bucket_t *bucket);
 
 void slab_reg(slab_t *slab);
@@ -212,7 +229,8 @@ void slab_unreg(slab_t *slab) {
     slab_unreg_by_addr(end_addr, slab);
 }
 
-bucket_t *create_bucket(size_t Sz, void *AllocCtx) {
+bucket_t *create_bucket(size_t Sz, void *AllocCtx,
+                        umf_disjoint_pool_shared_limits_t *shared_limits) {
     bucket_t *bucket = (bucket_t *)umf_ba_global_alloc(sizeof(bucket_t));
 
     bucket->Size = Sz;
@@ -228,6 +246,9 @@ bucket_t *create_bucket(size_t Sz, void *AllocCtx) {
     bucket->allocCount = 0;
     bucket->maxSlabsInUse = 0;
 
+    bucket->shared_limits = shared_limits;
+    assert(shared_limits);
+
     utils_mutex_init(&bucket->bucket_lock);
 
     return bucket;
@@ -453,6 +474,56 @@ void bucket_update_stats(bucket_t *bucket, int in_use, int in_pool) {
         in_pool * bucket_slab_alloc_size(bucket);
 }
 
+// If a slab was available in the pool then note that the current pooled
+// size has reduced by the size of a slab in this bucket.
+void bucket_decrement_pool(bucket_t *bucket, bool *from_pool) {
+    *from_pool = true;
+    bucket_update_stats(bucket, 1, -1);
+    utils_fetch_and_add64(&bucket->shared_limits->total_size,
+                          -bucket_slab_alloc_size(bucket));
+}
+
+bool bucket_can_pool(bucket_t *bucket, bool *to_pool) {
+    size_t NewFreeSlabsInBucket;
+    // Check if this bucket is used in chunked form or as full slabs.
+    bool chunkedBucket =
+        bucket_get_size(bucket) <= bucket_chunk_cut_off(bucket);
+    if (chunkedBucket) {
+        NewFreeSlabsInBucket = bucket->chunkedSlabsInPool + 1;
+    } else {
+        // TODO optimize
+        size_t avail_num = 0;
+        slab_list_item_t *it = NULL;
+        DL_FOREACH(bucket->AvailableSlabs, it) { avail_num++; }
+        NewFreeSlabsInBucket = avail_num + 1;
+    }
+    if (bucket_capacity(bucket) >= NewFreeSlabsInBucket) {
+        size_t pool_size = bucket->shared_limits->total_size;
+        while (true) {
+            size_t new_pool_size = pool_size + bucket_slab_alloc_size(bucket);
+
+            if (bucket->shared_limits->max_size < new_pool_size) {
+                break;
+            }
+
+            if (utils_compare_exchange(&bucket->shared_limits->total_size,
+                                       &pool_size, &new_pool_size)) {
+                if (chunkedBucket) {
+                    ++bucket->chunkedSlabsInPool;
+                }
+
+                bucket_update_stats(bucket, -1, 1);
+                *to_pool = true;
+                return true;
+            }
+        }
+    }
+
+    bucket_update_stats(bucket, -1, 0);
+    *to_pool = false;
+    return false;
+}
+
 #ifdef __cplusplus
 }
 #endif

src/pool/pool_disjoint.cpp

@@ -35,6 +35,8 @@
 #include "utils_math.h"
 #include "utils_sanitizers.h"
 
+#include "utils_concurrency.h"
+
 // TODO remove
 #ifdef __cplusplus
 extern "C" {
@@ -49,11 +51,6 @@ struct slab_t;
 #endif
 // end TODO remove
 
-typedef struct umf_disjoint_pool_shared_limits_t {
-    size_t MaxSize;
-    std::atomic<size_t> TotalSize;
-} umf_disjoint_pool_shared_limits_t;
-
 class DisjointPool {
   public:
     class AllocImpl;
@@ -78,12 +75,12 @@ class DisjointPool {
 
 umf_disjoint_pool_shared_limits_t *
 umfDisjointPoolSharedLimitsCreate(size_t MaxSize) {
-    return new umf_disjoint_pool_shared_limits_t{MaxSize, 0};
+    return shared_limits_create(MaxSize);
 }
 
 void umfDisjointPoolSharedLimitsDestroy(
     umf_disjoint_pool_shared_limits_t *limits) {
-    delete limits;
+    shared_limits_destroy(limits);
 }
 
 // Allocations are a minimum of 4KB/64KB/2MB even when a smaller size is
@@ -125,8 +122,7 @@ class DisjointPool::AllocImpl {
     // Configuration for this instance
     umf_disjoint_pool_params_t params;
 
-    umf_disjoint_pool_shared_limits_t DefaultSharedLimits = {
-        (std::numeric_limits<size_t>::max)(), 0};
+    umf_disjoint_pool_shared_limits_t *DefaultSharedLimits;
 
     // Used in algorithm for finding buckets
     std::size_t MinBucketSizeExp;
@@ -150,13 +146,14 @@ class DisjointPool::AllocImpl {
         Size1 = std::max(Size1, UMF_DISJOINT_POOL_MIN_BUCKET_DEFAULT_SIZE);
         // Calculate the exponent for MinBucketSize used for finding buckets.
         MinBucketSizeExp = (size_t)log2Utils(Size1);
+        DefaultSharedLimits = shared_limits_create(SIZE_MAX);
         auto Size2 = Size1 + Size1 / 2;
         for (; Size2 < CutOff; Size1 *= 2, Size2 *= 2) {
             // TODO copy allocimpl
-            Buckets.push_back(create_bucket(Size1, this));
-            Buckets.push_back(create_bucket(Size2, this));
+            Buckets.push_back(create_bucket(Size1, this, this->getLimits()));
+            Buckets.push_back(create_bucket(Size2, this, this->getLimits()));
         }
-        Buckets.push_back(create_bucket(CutOff, this));
+        Buckets.push_back(create_bucket(CutOff, this, this->getLimits()));
 
         auto ret = umfMemoryProviderGetMinPageSize(hProvider, nullptr,
                                                    &ProviderMinPageSize);
@@ -166,6 +163,8 @@ class DisjointPool::AllocImpl {
     }
 
     ~AllocImpl() {
+        // TODO
+        // destroy DefaultSharedLimits
 
         for (auto it = Buckets.begin(); it != Buckets.end(); it++) {
             destroy_bucket(*it);
@@ -196,7 +195,7 @@ class DisjointPool::AllocImpl {
         if (params.SharedLimits) {
             return params.SharedLimits;
         } else {
-            return &DefaultSharedLimits;
+            return DefaultSharedLimits;
         }
     };
 
@@ -254,14 +253,6 @@ std::ostream &operator<<(std::ostream &Os, slab_t &Slab) {
 }
 */
 
-// If a slab was available in the pool then note that the current pooled
-// size has reduced by the size of a slab in this bucket.
-void bucket_decrement_pool(bucket_t *bucket, bool *FromPool) {
-    *FromPool = true;
-    bucket_update_stats(bucket, 1, -1);
-    bucket_get_limits(bucket)->TotalSize -= bucket_slab_alloc_size(bucket);
-}
-
 /*
 void Bucket::printStats(bool &TitlePrinted, const std::string &Label) {
     if (allocCount) {
@@ -553,7 +544,7 @@ size_t DisjointPool::malloc_usable_size(void *) {
 umf_result_t DisjointPool::free(void *ptr) {
     bool ToPool;
     umf_result_t ret = impl->deallocate(ptr, ToPool);
-
+    /*
     if (ret == UMF_RESULT_SUCCESS) {
 
         if (impl->getParams().PoolTrace > 2) {
@@ -565,7 +556,7 @@ umf_result_t DisjointPool::free(void *ptr) {
                       << ", Current pool size for " << MT << " "
                       << impl->getParams().CurPoolSize << "\n";
         }
-    }
+    }*/
     return ret;
 }
 
@@ -577,10 +568,11 @@ DisjointPool::DisjointPool() {}
 
 // Define destructor for use with unique_ptr
 DisjointPool::~DisjointPool() {
-    bool TitlePrinted = false;
-    size_t HighBucketSize;
-    size_t HighPeakSlabsInUse;
+    /*
     if (impl->getParams().PoolTrace > 1) {
+        bool TitlePrinted = false;
+        size_t HighBucketSize;
+        size_t HighPeakSlabsInUse;
         auto name = impl->getParams().Name;
         //try { // cannot throw in destructor
         impl->printStats(TitlePrinted, HighBucketSize, HighPeakSlabsInUse,
@@ -596,6 +588,7 @@ DisjointPool::~DisjointPool() {
         //} catch (...) { // ignore exceptions
         // }
     }
+    */
 }
 
 static umf_memory_pool_ops_t UMF_DISJOINT_POOL_OPS =
@@ -666,47 +659,6 @@ void slab_unreg_by_addr(void *addr, slab_t *slab) {
     assert(false && "Slab is not found");
 }
 
-bool bucket_can_pool(bucket_t *bucket, bool *ToPool) {
-    size_t NewFreeSlabsInBucket;
-    // Check if this bucket is used in chunked form or as full slabs.
-    bool chunkedBucket =
-        bucket_get_size(bucket) <= bucket_chunk_cut_off(bucket);
-    if (chunkedBucket) {
-        NewFreeSlabsInBucket = bucket->chunkedSlabsInPool + 1;
-    } else {
-        // TODO optimize
-        size_t avail_num = 0;
-        slab_list_item_t *it = NULL;
-        DL_FOREACH(bucket->AvailableSlabs, it) { avail_num++; }
-        NewFreeSlabsInBucket = avail_num + 1;
-    }
-    if (bucket_capacity(bucket) >= NewFreeSlabsInBucket) {
-        size_t PoolSize = bucket_get_limits(bucket)->TotalSize;
-        while (true) {
-            size_t NewPoolSize = PoolSize + bucket_slab_alloc_size(bucket);
-
-            if (bucket_get_limits(bucket)->MaxSize < NewPoolSize) {
-                break;
-            }
-
-            if (bucket_get_limits(bucket)->TotalSize.compare_exchange_strong(
-                    PoolSize, NewPoolSize)) {
-                if (chunkedBucket) {
-                    ++bucket->chunkedSlabsInPool;
-                }
-
-                bucket_update_stats(bucket, -1, 1);
-                *ToPool = true;
-                return true;
-            }
-        }
-    }
-
-    bucket_update_stats(bucket, -1, 0);
-    *ToPool = false;
-    return false;
-}
-
 #ifdef __cplusplus
 }
 #endif

src/pool/pool_disjoint_temp.h

@@ -6,6 +6,9 @@
 void annotate_memory_inaccessible(void *ptr, size_t size);
 void annotate_memory_undefined(void *ptr, size_t size);
 
+umf_disjoint_pool_shared_limits_t *shared_limits_create(size_t max_size);
+void shared_limits_destroy(umf_disjoint_pool_shared_limits_t *shared_limits);
+
 typedef struct slab_list_item_t slab_list_item_t;
 
 typedef struct bucket_t {
@@ -24,6 +27,8 @@ typedef struct bucket_t {
     // routines, slab map and etc.
     void *OwnAllocCtx;
 
+    umf_disjoint_pool_shared_limits_t *shared_limits;
+
     // For buckets used in chunked mode, a counter of slabs in the pool.
     // For allocations that use an entire slab each, the entries in the Available
     // list are entries in the pool.Each slab is available for a new
@@ -109,7 +114,8 @@ void slab_reg_by_addr(void *addr, slab_t *slab);
 void slab_unreg(slab_t *slab);
 void slab_unreg_by_addr(void *addr, slab_t *slab);
 
-bucket_t *create_bucket(size_t sz, void *alloc_ctx);
+bucket_t *create_bucket(size_t sz, void *alloc_ctx,
+                        umf_disjoint_pool_shared_limits_t *shared_limits);
 void destroy_bucket(bucket_t *bucket);
 
 void bucket_update_stats(bucket_t *bucket, int in_use, int in_pool);

src/utils/utils_concurrency.h

@@ -61,11 +61,13 @@ int utils_mutex_unlock(utils_mutex_t *mutex);
 void utils_init_once(UTIL_ONCE_FLAG *flag, void (*onceCb)(void));
 
 #if defined(_WIN32)
+
 static __inline unsigned char utils_lssb_index(long long value) {
     unsigned long ret;
     _BitScanForward64(&ret, value);
     return (unsigned char)ret;
 }
+
 static __inline unsigned char utils_mssb_index(long long value) {
     unsigned long ret;
     _BitScanReverse64(&ret, value);
@@ -81,15 +83,21 @@ static __inline unsigned char utils_mssb_index(long long value) {
 
 #define utils_atomic_store_release(object, desired)                            \
     InterlockedExchange64((LONG64 volatile *)object, (LONG64)desired)
+
 #define utils_atomic_increment(object)                                         \
     InterlockedIncrement64((LONG64 volatile *)object)
+
 #define utils_atomic_decrement(object)                                         \
     InterlockedDecrement64((LONG64 volatile *)object)
+
 #define utils_fetch_and_add64(ptr, value)                                      \
     InterlockedExchangeAdd64((LONG64 *)(ptr), value)
-#else
+
+#else // !defined(_WIN32)
+
 #define utils_lssb_index(x) ((unsigned char)__builtin_ctzll(x))
 #define utils_mssb_index(x) ((unsigned char)(63 - __builtin_clzll(x)))
+
 #define utils_atomic_load_acquire(object, dest)                                \
     do {                                                                       \
         utils_annotate_acquire((void *)object);                                \
@@ -103,12 +111,19 @@ static __inline unsigned char utils_mssb_index(long long value) {
     } while (0)
 
 #define utils_atomic_increment(object)                                         \
-    __atomic_add_fetch(object, 1, __ATOMIC_ACQ_REL)
+    __atomic_add_fetch(object, 1, memory_order_acq_rel)
+
 #define utils_atomic_decrement(object)                                         \
-    __atomic_sub_fetch(object, 1, __ATOMIC_ACQ_REL)
-#define utils_fetch_and_add64 __sync_fetch_and_add
+    __atomic_sub_fetch(object, 1, memory_order_acq_rel)
 
-#endif
+#define utils_fetch_and_add64(object, value)                                   \
+    __atomic_fetch_add(object, value, memory_order_acq_rel)
+
+#define utils_compare_exchange(object, expected, desired)                      \
+    __atomic_compare_exchange(object, expected, desired, 0 /* strong */,       \
+                              memory_order_acq_rel, memory_order_relaxed)
+
+#endif // !defined(_WIN32)
 
 #ifdef __cplusplus
 }