a

Author: bratpiorka

benchmark/benchmark.cpp

@@ -96,7 +96,6 @@ UMF_BENCHMARK_TEMPLATE_DEFINE(multiple_malloc_free_benchmark,
                               pool_allocator<disjoint_pool<os_provider>>);
 UMF_BENCHMARK_REGISTER_F(multiple_malloc_free_benchmark, disjoint_pool_uniform)
     ->Apply(&default_multiple_alloc_uniform_size)
-    ->Apply(&singlethreaded)
     ->Apply(&multithreaded);
 
 #ifdef UMF_POOL_JEMALLOC_ENABLED

src/critnib/critnib.c

@@ -119,7 +119,6 @@ struct critnib_leaf {
 
 struct critnib {
     struct critnib_node *root;
-    uint64_t remove_count;
 
     /* pool of freed nodes: singly linked list, next at child[0] */
     struct critnib_node *deleted_node;
@@ -129,6 +128,8 @@ struct critnib {
     struct critnib_node *pending_del_nodes[DELETED_LIFE];
     struct critnib_leaf *pending_del_leaves[DELETED_LIFE];
 
+    uint64_t remove_count;
+
     struct utils_mutex_t mutex; /* writes/removes */
 };
 

src/pool/pool_disjoint.c

@@ -444,7 +444,6 @@ static bool bucket_can_pool(bucket_t *bucket) {
             &bucket->shared_limits->total_size, size_to_add);
 
         if (previous_size + size_to_add <= bucket->shared_limits->max_size) {
-
             ++bucket->chunked_slabs_in_pool;
             bucket_update_stats(bucket, -1, 1);
             return true;

src/utils/utils_concurrency.h

@@ -21,7 +21,7 @@
 #include "utils_windows_intrin.h"
 
 #pragma intrinsic(_BitScanForward64)
-#else
+#else /* !_WIN32 */
 #include <pthread.h>
 
 #ifndef __cplusplus
@@ -37,7 +37,7 @@ using std::memory_order_release;
 
 #endif /* __cplusplus */
 
-#endif /* _WIN32 */
+#endif /* !_WIN32 */
 
 #include "utils_common.h"
 #include "utils_sanitizers.h"
@@ -106,29 +106,25 @@ static __inline void utils_atomic_load_acquire_u64(uint64_t *ptr,
     // NOTE: Windows cl complains about direct accessing 'ptr' which is next
     // accessed using Interlocked* functions (warning 28112 - disabled)
     ASSERT_IS_ALIGNED((uintptr_t)ptr, 8);
+
+    // On Windows, there is no equivalent to __atomic_load, so we use cmpxchg
+    // with 0, 0 here. This will always return the value under the pointer
+    // without writing anything.
     LONG64 ret = InterlockedCompareExchange64((LONG64 volatile *)ptr, 0, 0);
-    utils_annotate_acquire(ptr);
+    //utils_annotate_acquire(ptr);
     *out = *(uint64_t *)&ret;
 }
 
 static __inline void utils_atomic_load_acquire_ptr(void **ptr, void **out) {
     ASSERT_IS_ALIGNED((uintptr_t)ptr, 8);
     uintptr_t ret = (uintptr_t)InterlockedCompareExchangePointer(ptr, 0, 0);
-    utils_annotate_acquire((void *)ptr);
+    //utils_annotate_acquire((void *)ptr);
     *(uintptr_t *)out = ret;
 }
 
-static __inline void utils_atomic_store_release_u64(uint64_t *ptr,
-                                                    uint64_t *val) {
-    ASSERT_IS_ALIGNED((uintptr_t)ptr, 8);
-    ASSERT_IS_ALIGNED((uintptr_t)val, 8);
-    utils_annotate_release(ptr);
-    InterlockedExchange64((LONG64 volatile *)ptr, *(LONG64 *)val);
-}
-
 static __inline void utils_atomic_store_release_ptr(void **ptr, void *val) {
     ASSERT_IS_ALIGNED((uintptr_t)ptr, 8);
-    utils_annotate_release(ptr);
+    //utils_annotate_release(ptr);
     InterlockedExchangePointer(ptr, val);
 }
 
@@ -146,14 +142,12 @@ static __inline uint64_t utils_atomic_decrement_u64(uint64_t *ptr) {
 
 static __inline uint64_t utils_fetch_and_add_u64(uint64_t *ptr, uint64_t val) {
     ASSERT_IS_ALIGNED((uintptr_t)ptr, 8);
-    ASSERT_IS_ALIGNED((uintptr_t)&val, 8);
     // return the value that had previously been in *ptr
     return InterlockedExchangeAdd64((LONG64 volatile *)(ptr), val);
 }
 
 static __inline uint64_t utils_fetch_and_sub_u64(uint64_t *ptr, uint64_t val) {
     ASSERT_IS_ALIGNED((uintptr_t)ptr, 8);
-    ASSERT_IS_ALIGNED((uintptr_t)&val, 8);
     // return the value that had previously been in *ptr
     // NOTE: on Windows there is no *Sub* version of InterlockedExchange
     return InterlockedExchangeAdd64((LONG64 volatile *)(ptr), -(LONG64)val);
@@ -193,27 +187,19 @@ static inline void utils_atomic_load_acquire_u64(uint64_t *ptr, uint64_t *out) {
     ASSERT_IS_ALIGNED((uintptr_t)ptr, 8);
     ASSERT_IS_ALIGNED((uintptr_t)out, 8);
     __atomic_load(ptr, out, memory_order_acquire);
-    utils_annotate_acquire(ptr);
+    //utils_annotate_acquire(ptr);
 }
 
 static inline void utils_atomic_load_acquire_ptr(void **ptr, void **out) {
     ASSERT_IS_ALIGNED((uintptr_t)ptr, 8);
     ASSERT_IS_ALIGNED((uintptr_t)out, 8);
-    *out = (void *)__atomic_load_n((uintptr_t *)ptr, memory_order_acquire);
-    utils_annotate_acquire((void *)ptr);
-}
-
-static inline void utils_atomic_store_release_u64(uint64_t *ptr,
-                                                  uint64_t *val) {
-    ASSERT_IS_ALIGNED((uintptr_t)ptr, 8);
-    ASSERT_IS_ALIGNED((uintptr_t)val, 8);
-    utils_annotate_release(ptr);
-    __atomic_store(ptr, val, memory_order_release);
+    __atomic_load((uintptr_t *)ptr, (uintptr_t *)out, memory_order_acquire);
+    //utils_annotate_acquire((void *)ptr);
 }
 
 static inline void utils_atomic_store_release_ptr(void **ptr, void *val) {
     ASSERT_IS_ALIGNED((uintptr_t)ptr, 8);
-    utils_annotate_release(ptr);
+    //utils_annotate_release(ptr);
     __atomic_store_n((uintptr_t *)ptr, (uintptr_t)val, memory_order_release);
 }
 

src/utils/utils_level_zero.cpp

@@ -5,12 +5,11 @@
  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  */
 
-#include "utils_level_zero.h"
-
 #include <memory>
 #include <stdlib.h>
 
 #include "utils_concurrency.h"
+#include "utils_level_zero.h"
 #include "utils_load_library.h"
 
 #include "ze_api.h"

test/providers/cuda_helpers.cpp

@@ -9,6 +9,7 @@
 #include <memory>
 #include <stdlib.h>
 
+#include "base.hpp"
 #include "cuda_helpers.h"
 #include "utils_concurrency.h"
 #include "utils_load_library.h"