fix atomic shared limits update in D. P.

Author: bratpiorka

cmake/helpers.cmake

@@ -270,7 +270,10 @@ function(add_umf_target_compile_options name)
                     /W4
                     /Gy
                     /GS
-                    # disable warning 6326: Potential comparison of a constant
+                    # disable 4324 warning: structure was padded due to 
+                    # alignment specifier
+                    /wd4324
+                    # disable 6326 warning: Potential comparison of a constant
                     # with another constant
                     /wd6326
                     # disable 4200 warning: nonstandard extension used:

src/pool/pool_disjoint.c

@@ -433,18 +433,19 @@ static bool bucket_can_pool(bucket_t *bucket) {
 
     // we keep at most params.capacity slabs in the pool
     if (bucket_max_pooled_slabs(bucket) >= new_free_slabs_in_bucket) {
-        size_t pool_size = 0;
+        CACHE_ALIGNED size_t shared_size = 0;
         utils_atomic_load_acquire(&bucket->shared_limits->total_size,
-                                  &pool_size);
+                                  &shared_size);
         while (true) {
-            size_t new_pool_size = pool_size + bucket_slab_alloc_size(bucket);
-
-            if (bucket->shared_limits->max_size < new_pool_size) {
+            CACHE_ALIGNED size_t new_shared_size = 0;
+            new_shared_size = shared_size + bucket_slab_alloc_size(bucket);
+            if (bucket->shared_limits->max_size < new_shared_size) {
                 break;
             }
 
+            // update the total shared size if needed
             if (utils_compare_exchange(&bucket->shared_limits->total_size,
-                                       &pool_size, &new_pool_size)) {
+                                       &shared_size, &new_shared_size)) {
                 ++bucket->chunked_slabs_in_pool;
 
                 bucket_update_stats(bucket, -1, 1);
@@ -895,7 +896,8 @@ umf_memory_pool_ops_t *umfDisjointPoolOps(void) {
 
 umf_disjoint_pool_shared_limits_t *
 umfDisjointPoolSharedLimitsCreate(size_t max_size) {
-    umf_disjoint_pool_shared_limits_t *ptr = umf_ba_global_alloc(sizeof(*ptr));
+    umf_disjoint_pool_shared_limits_t *ptr =
+        umf_ba_global_aligned_alloc(sizeof(*ptr), 64);
     if (ptr == NULL) {
         LOG_ERR("cannot allocate memory for disjoint pool shared limits");
         return NULL;

src/pool/pool_disjoint_internal.h

@@ -101,8 +101,8 @@ typedef struct slab_t {
 } slab_t;
 
 typedef struct umf_disjoint_pool_shared_limits_t {
+    CACHE_ALIGNED size_t total_size; // requires atomic access
     size_t max_size;
-    size_t total_size; // requires atomic access
 } umf_disjoint_pool_shared_limits_t;
 
 typedef struct umf_disjoint_pool_params_t {

src/utils/utils_concurrency.h

@@ -31,6 +31,7 @@
 
 #endif /* _WIN32 */
 
+#include "utils_common.h"
 #include "utils_sanitizers.h"
 
 #ifdef __cplusplus
@@ -45,6 +46,12 @@ typedef struct utils_mutex_t {
 #endif
 } utils_mutex_t;
 
+#ifdef _WIN32
+#define CACHE_ALIGNED __declspec(align(64))
+#else
+#define CACHE_ALIGNED __attribute__((aligned(64)))
+#endif
+
 size_t utils_mutex_get_size(void);
 utils_mutex_t *utils_mutex_init(utils_mutex_t *ptr);
 void utils_mutex_destroy_not_free(utils_mutex_t *m);
@@ -110,9 +117,17 @@ static __inline unsigned char utils_mssb_index(long long value) {
 #define utils_fetch_and_add64(ptr, value)                                      \
     InterlockedExchangeAdd64((LONG64 *)(ptr), value)
 
-// NOTE: windows version have different order of args
-#define utils_compare_exchange(object, desired, expected)                      \
-    InterlockedCompareExchange64((LONG64 volatile *)object, *expected, *desired)
+// NOTE: windows InterlockedCompareExchange64 have different order of args than
+// __atomic_compare_exchange. Also, the windows version returns the initial
+// value of "object", so to match the linux version we return true if the
+// exchange was successful or not needed
+#define utils_compare_exchange(object, expected, desired)                      \
+    assert(IS_ALIGNED((uintptr_t)object, 64)),                                 \
+        assert(IS_ALIGNED((uintptr_t)expected, 64)),                           \
+        assert(IS_ALIGNED((uintptr_t)desired, 64)),                            \
+        InterlockedCompareExchange64((LONG64 volatile *)object, *desired,      \
+                                     *expected),                               \
+        *(LONG64 volatile *)object == *(LONG64 volatile *)desired
 
 #else // !defined(_WIN32)