diff --git a/include/umf/memory_pool.h b/include/umf/memory_pool.h index bdd7d4e7c..93329bc17 100644 --- a/include/umf/memory_pool.h +++ b/include/umf/memory_pool.h @@ -124,6 +124,9 @@ UMF_EXPORT size_t umfPoolMallocUsableSize(umf_memory_pool_handle_t hPool, /// UMF_EXPORT umf_result_t umfPoolFree(umf_memory_pool_handle_t hPool, void *ptr); +UMF_EXPORT umf_result_t umfPoolFreeSized(umf_memory_pool_handle_t hPool, + void *ptr, size_t size); + /// /// @brief Frees the memory space pointed by ptr if it belongs to UMF pool, does nothing otherwise. /// @param ptr pointer to the allocated memory diff --git a/include/umf/memory_pool_ops.h b/include/umf/memory_pool_ops.h index 9ddb34700..2fb2b96d0 100644 --- a/include/umf/memory_pool_ops.h +++ b/include/umf/memory_pool_ops.h @@ -99,7 +99,7 @@ typedef struct umf_memory_pool_ops_t { /// Whether any status other than UMF_RESULT_SUCCESS can be returned /// depends on the memory provider used by the \p pool. /// - umf_result_t (*free)(void *pool, void *); + umf_result_t (*free)(void *pool, void *, size_t size); /// /// @brief Retrieve \p umf_result_t representing the error of the last failed allocation diff --git a/include/umf/pools/pool_base.h b/include/umf/pools/pool_base.h new file mode 100644 index 000000000..32312d31c --- /dev/null +++ b/include/umf/pools/pool_base.h @@ -0,0 +1,25 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_BASE_MEMORY_POOL_H +#define UMF_BASE_MEMORY_POOL_H 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +umf_memory_pool_ops_t *umfBasePoolOps(void); + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_JEMALLOC_MEMORY_POOL_H */ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b943c2c55..e920a08d0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -33,6 +33,7 @@ set(UMF_SOURCES provider/provider_tracking.c critnib/critnib.c pool/pool_proxy.c + pool/pool_base.c ) set(UMF_SOURCES_LINUX diff --git a/src/base_alloc/base_alloc.c b/src/base_alloc/base_alloc.c index 0764f123a..386ac7e91 100644 --- a/src/base_alloc/base_alloc.c +++ b/src/base_alloc/base_alloc.c @@ -196,29 +196,6 @@ void *umf_ba_alloc(umf_ba_pool_t *pool) { return chunk; } -#ifndef NDEBUG -// Checks if given pointer belongs to the pool. Should be called -// under the lock -static int pool_contains_pointer(umf_ba_pool_t *pool, void *ptr) { - char *cptr = (char *)ptr; - if (cptr >= pool->data && - cptr < ((char *)(pool)) + pool->metadata.pool_size) { - return 1; - } - - umf_ba_next_pool_t *next_pool = pool->next_pool; - while (next_pool) { - if (cptr >= next_pool->data && - cptr < ((char *)(next_pool)) + pool->metadata.pool_size) { - return 1; - } - next_pool = next_pool->next_pool; - } - - return 0; -} -#endif - void umf_ba_free(umf_ba_pool_t *pool, void *ptr) { if (ptr == NULL) { return; @@ -227,7 +204,6 @@ void umf_ba_free(umf_ba_pool_t *pool, void *ptr) { umf_ba_chunk_t *chunk = (umf_ba_chunk_t *)ptr; util_mutex_lock(&pool->metadata.free_lock); - assert(pool_contains_pointer(pool, ptr)); chunk->next = pool->metadata.free_list; pool->metadata.free_list = chunk; #ifndef NDEBUG diff --git a/src/base_alloc/base_alloc_global.c b/src/base_alloc/base_alloc_global.c index 0489eaa68..645bedbb3 100644 --- a/src/base_alloc/base_alloc_global.c +++ b/src/base_alloc/base_alloc_global.c @@ -5,40 +5,44 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include +/* A MT-safe base allocator */ -#include "base_alloc.h" +#include -#define SIZE_BA_POOL_CHUNK 128 +#include -// global base allocator used by all providers and pools -static umf_ba_pool_t *BA_pool = NULL; +// this is where base pool will allocate it's own metadata +char base_pool_metadata[128]; -int umf_ba_create_global(void) { - assert(BA_pool == NULL); +static int base_pool_init; +static umf_memory_pool_ops_t *ops; +static void *pool; - BA_pool = umf_ba_create(SIZE_BA_POOL_CHUNK); - if (!BA_pool) { - return -1; - } +void umf_ba_create_global(void) { + ops = umfBasePoolOps(); - return 0; + // do not call umfPoolCreate to avoid inifite recursion + ops->initialize(NULL, NULL, &pool); } -void umf_ba_destroy_global(void) { - assert(BA_pool); - umf_ba_destroy(BA_pool); - BA_pool = NULL; -} +void umf_ba_destroy_global(void) { ops->finalize(pool); } -umf_ba_pool_t *umf_ba_get_pool(size_t size) { - // TODO: a specific class-size base allocator can be returned here - assert(BA_pool); - assert(size <= SIZE_BA_POOL_CHUNK); +void *umf_ba_global_alloc(size_t size) { + if (!base_pool_init) { + assert(size <= sizeof(base_pool_metadata)); + base_pool_init = 1; + return &base_pool_metadata; + } + + return ops->malloc(pool, size); +} - if (size > SIZE_BA_POOL_CHUNK) { - return NULL; +void umf_ba_global_free(void *ptr, size_t size) { + if (ptr == &base_pool_metadata) { + // nothing to do + return; } - return BA_pool; + umf_result_t ret = ops->free(pool, ptr, size); + assert(ret == UMF_RESULT_SUCCESS); } diff --git a/src/base_alloc/base_alloc_global.h b/src/base_alloc/base_alloc_global.h index a048cf009..e0439517b 100644 --- a/src/base_alloc/base_alloc_global.h +++ b/src/base_alloc/base_alloc_global.h @@ -14,9 +14,11 @@ extern "C" { #endif -int umf_ba_create_global(void); +void umf_ba_create_global(void); void umf_ba_destroy_global(void); -umf_ba_pool_t *umf_ba_get_pool(size_t size); + +void *umf_ba_global_alloc(size_t size); +void umf_ba_global_free(void *ptr, size_t size); #ifdef __cplusplus } diff --git a/src/memory_pool.c b/src/memory_pool.c index 95d641853..cab90d1e4 100644 --- a/src/memory_pool.c +++ b/src/memory_pool.c @@ -60,10 +60,16 @@ size_t umfPoolMallocUsableSize(umf_memory_pool_handle_t hPool, void *ptr) { umf_result_t umfPoolFree(umf_memory_pool_handle_t hPool, void *ptr) { UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); - return hPool->ops.free(hPool->pool_priv, ptr); + return hPool->ops.free(hPool->pool_priv, ptr, 0); } umf_result_t umfPoolGetLastAllocationError(umf_memory_pool_handle_t hPool) { UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); return hPool->ops.get_last_allocation_error(hPool->pool_priv); } + +umf_result_t umfPoolFreeSized(umf_memory_pool_handle_t hPool, void *ptr, + size_t size) { + UMF_CHECK((hPool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + return hPool->ops.free(hPool->pool_priv, ptr, size); +} \ No newline at end of file diff --git a/src/memory_pool_default.c b/src/memory_pool_default.c index ef7b52e05..38c7768ec 100644 --- a/src/memory_pool_default.c +++ b/src/memory_pool_default.c @@ -20,31 +20,26 @@ umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, umf_memory_provider_handle_t provider, void *params, umf_memory_pool_handle_t *hPool) { - if (!ops || !provider || !hPool) { + if (!ops || !hPool) { return UMF_RESULT_ERROR_INVALID_ARGUMENT; } umf_result_t ret = UMF_RESULT_SUCCESS; - umf_ba_pool_t *base_allocator = umf_ba_get_pool(sizeof(umf_memory_pool_t)); - if (!base_allocator) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - umf_memory_pool_handle_t pool = umf_ba_alloc(base_allocator); + umf_memory_pool_handle_t pool = + umf_ba_global_alloc(sizeof(umf_memory_pool_t)); if (!pool) { return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } assert(ops->version == UMF_VERSION_CURRENT); - pool->base_allocator = base_allocator; pool->provider = provider; pool->own_provider = false; pool->ops = *ops; ret = ops->initialize(pool->provider, params, &pool->pool_priv); if (ret != UMF_RESULT_SUCCESS) { - umf_ba_free(base_allocator, pool); + umf_ba_global_free(pool, sizeof(umf_memory_pool_t)); return ret; } @@ -61,7 +56,7 @@ void umfPoolDestroy(umf_memory_pool_handle_t hPool) { umfMemoryProviderDestroy(hProvider); } // TODO: this free keeps memory in base allocator, so it can lead to OOM in some scenarios (it should be optimized) - umf_ba_free(hPool->base_allocator, hPool); + umf_ba_global_free(hPool, sizeof(umf_memory_pool_t)); } umf_result_t umfFree(void *ptr) { diff --git a/src/memory_pool_internal.h b/src/memory_pool_internal.h index bbbae844e..22478b7df 100644 --- a/src/memory_pool_internal.h +++ b/src/memory_pool_internal.h @@ -31,9 +31,6 @@ typedef struct umf_memory_pool_t { umf_memory_provider_handle_t provider; // Tells whether memory provider is owned by the pool. bool own_provider; - - // saved pointer to the global base allocator - umf_ba_pool_t *base_allocator; } umf_memory_pool_t; umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, diff --git a/src/memory_pool_tracking.c b/src/memory_pool_tracking.c index 4c932df5f..f725c5245 100644 --- a/src/memory_pool_tracking.c +++ b/src/memory_pool_tracking.c @@ -21,17 +21,13 @@ umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, umf_memory_provider_handle_t provider, void *params, umf_memory_pool_handle_t *hPool) { - if (!ops || !provider || !hPool) { + if (!ops || !hPool) { return UMF_RESULT_ERROR_INVALID_ARGUMENT; } umf_result_t ret = UMF_RESULT_SUCCESS; - umf_ba_pool_t *base_allocator = umf_ba_get_pool(sizeof(umf_memory_pool_t)); - if (!base_allocator) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - umf_memory_pool_handle_t pool = umf_ba_alloc(base_allocator); + umf_memory_pool_handle_t pool = + umf_ba_global_alloc(sizeof(umf_memory_pool_t)); if (!pool) { return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } @@ -44,7 +40,6 @@ umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, goto err_provider_create; } - pool->base_allocator = base_allocator; pool->own_provider = false; pool->ops = *ops; @@ -59,7 +54,7 @@ umf_result_t umfPoolCreateInternal(const umf_memory_pool_ops_t *ops, err_pool_init: umfMemoryProviderDestroy(pool->provider); err_provider_create: - umf_ba_free(base_allocator, pool); + umf_ba_global_free(pool, sizeof(umf_memory_pool_t)); return ret; } @@ -74,7 +69,7 @@ void umfPoolDestroy(umf_memory_pool_handle_t hPool) { // Destroy tracking provider. umfMemoryProviderDestroy(hPool->provider); // TODO: this free keeps memory in base allocator, so it can lead to OOM in some scenarios (it should be optimized) - umf_ba_free(hPool->base_allocator, hPool); + umf_ba_global_free(hPool, sizeof(umf_memory_pool_t)); } umf_result_t umfFree(void *ptr) { diff --git a/src/memory_provider.c b/src/memory_provider.c index f5c7936d8..9cdbe6abb 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -22,9 +22,6 @@ typedef struct umf_memory_provider_t { umf_memory_provider_ops_t ops; void *provider_priv; - - // saved pointer to the global base allocator - umf_ba_pool_t *base_allocator; } umf_memory_provider_t; umf_result_t umfMemoryProviderCreate(const umf_memory_provider_ops_t *ops, @@ -35,13 +32,8 @@ umf_result_t umfMemoryProviderCreate(const umf_memory_provider_ops_t *ops, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - umf_ba_pool_t *base_allocator = - umf_ba_get_pool(sizeof(umf_memory_provider_t)); - if (!base_allocator) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - umf_memory_provider_handle_t provider = umf_ba_alloc(base_allocator); + umf_memory_provider_handle_t provider = + umf_ba_global_alloc(sizeof(umf_memory_provider_t)); if (!provider) { return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } @@ -49,12 +41,11 @@ umf_result_t umfMemoryProviderCreate(const umf_memory_provider_ops_t *ops, assert(ops->version == UMF_VERSION_CURRENT); provider->ops = *ops; - provider->base_allocator = base_allocator; void *provider_priv; umf_result_t ret = ops->initialize(params, &provider_priv); if (ret != UMF_RESULT_SUCCESS) { - umf_ba_free(base_allocator, provider); + umf_ba_global_free(provider, sizeof(umf_memory_provider_t)); return ret; } @@ -67,7 +58,7 @@ umf_result_t umfMemoryProviderCreate(const umf_memory_provider_ops_t *ops, void umfMemoryProviderDestroy(umf_memory_provider_handle_t hProvider) { hProvider->ops.finalize(hProvider->provider_priv); - umf_ba_free(hProvider->base_allocator, hProvider); + umf_ba_global_free(hProvider, sizeof(umf_memory_provider_t)); } static void diff --git a/src/memory_target.c b/src/memory_target.c index 0999d7013..3bae34092 100644 --- a/src/memory_target.c +++ b/src/memory_target.c @@ -24,27 +24,20 @@ umf_result_t umfMemoryTargetCreate(const umf_memory_target_ops_t *ops, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - umf_ba_pool_t *base_allocator = - umf_ba_get_pool(sizeof(umf_memory_target_t)); - if (!base_allocator) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - umf_memory_target_handle_t target = - (umf_memory_target_t *)umf_ba_alloc(base_allocator); + (umf_memory_target_t *)umf_ba_global_alloc(sizeof(umf_memory_target_t)); if (!target) { return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } assert(ops->version == UMF_VERSION_CURRENT); - target->base_allocator = base_allocator; target->ops = ops; void *target_priv; umf_result_t ret = ops->initialize(params, &target_priv); if (ret != UMF_RESULT_SUCCESS) { - umf_ba_free(base_allocator, target); + umf_ba_global_free(target, sizeof(umf_memory_target_t)); return ret; } @@ -58,5 +51,5 @@ umf_result_t umfMemoryTargetCreate(const umf_memory_target_ops_t *ops, void umfMemoryTargetDestroy(umf_memory_target_handle_t memoryTarget) { assert(memoryTarget); memoryTarget->ops->finalize(memoryTarget->priv); - umf_ba_free(memoryTarget->base_allocator, memoryTarget); + umf_ba_global_free(memoryTarget, sizeof(umf_memory_target_t)); } diff --git a/src/memory_target.h b/src/memory_target.h index fccd809fe..3183b29ec 100644 --- a/src/memory_target.h +++ b/src/memory_target.h @@ -24,9 +24,6 @@ typedef struct umf_memory_target_ops_t umf_memory_target_ops_t; typedef struct umf_memory_target_t { const umf_memory_target_ops_t *ops; void *priv; - - // saved pointer to the global base allocator - umf_ba_pool_t *base_allocator; } umf_memory_target_t; typedef umf_memory_target_t *umf_memory_target_handle_t; diff --git a/src/memory_targets/memory_target_numa.c b/src/memory_targets/memory_target_numa.c index 0b48a8cd0..d1c4a0067 100644 --- a/src/memory_targets/memory_target_numa.c +++ b/src/memory_targets/memory_target_numa.c @@ -22,9 +22,6 @@ struct numa_memory_target_t { size_t id; - - // saved pointer to the global base allocator - umf_ba_pool_t *base_allocator; }; static umf_result_t numa_initialize(void *params, void **memTarget) { @@ -35,27 +32,19 @@ static umf_result_t numa_initialize(void *params, void **memTarget) { struct umf_numa_memory_target_config_t *config = (struct umf_numa_memory_target_config_t *)params; - umf_ba_pool_t *base_allocator = - umf_ba_get_pool(sizeof(struct numa_memory_target_t)); - if (!base_allocator) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - struct numa_memory_target_t *numaTarget = umf_ba_alloc(base_allocator); + struct numa_memory_target_t *numaTarget = + umf_ba_global_alloc(sizeof(struct numa_memory_target_t)); if (!numaTarget) { return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - numaTarget->base_allocator = base_allocator; numaTarget->id = config->id; *memTarget = numaTarget; return UMF_RESULT_SUCCESS; } static void numa_finalize(void *memTarget) { - struct numa_memory_target_t *numaTarget = - (struct numa_memory_target_t *)memTarget; - umf_ba_free(numaTarget->base_allocator, memTarget); + umf_ba_global_free(memTarget, sizeof(struct numa_memory_target_t)); } // sets maxnode and allocates and initializes mask based on provided memory targets diff --git a/src/memspace.c b/src/memspace.c index fe4515893..60d8834e7 100644 --- a/src/memspace.c +++ b/src/memspace.c @@ -12,6 +12,7 @@ #include +#include "base_alloc_global.h" #include "memory_target.h" #include "memory_target_ops.h" #include "memspace_internal.h" @@ -105,5 +106,5 @@ void umfMemspaceDestroy(umf_memspace_handle_t memspace) { } umf_ba_linear_destroy(memspace->linear_allocator); - umf_ba_free(memspace->base_allocator, memspace); + umf_ba_global_free(memspace, sizeof(struct umf_memspace_t)); } diff --git a/src/memspace_internal.h b/src/memspace_internal.h index 834a68037..07fe1c18c 100644 --- a/src/memspace_internal.h +++ b/src/memspace_internal.h @@ -24,9 +24,6 @@ struct umf_memspace_t { size_t size; umf_memory_target_handle_t *nodes; - // saved pointer to the global base allocator - umf_ba_pool_t *base_allocator; - // own local linear base allocator umf_ba_linear_pool_t *linear_allocator; }; diff --git a/src/memspaces/memspace_numa.c b/src/memspaces/memspace_numa.c index 693ace2a6..89417a2f1 100644 --- a/src/memspaces/memspace_numa.c +++ b/src/memspaces/memspace_numa.c @@ -23,15 +23,9 @@ umfMemspaceCreateFromNumaArray(size_t *nodeIds, size_t numIds, } enum umf_result_t ret = UMF_RESULT_SUCCESS; - - umf_ba_pool_t *base_allocator = - umf_ba_get_pool(sizeof(struct umf_memspace_t)); - if (!base_allocator) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - umf_memspace_handle_t memspace = - (struct umf_memspace_t *)umf_ba_alloc(base_allocator); + (struct umf_memspace_t *)umf_ba_global_alloc( + sizeof(struct umf_memspace_t)); if (!memspace) { return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } @@ -43,7 +37,6 @@ umfMemspaceCreateFromNumaArray(size_t *nodeIds, size_t numIds, goto err_umf_ba_linear_create; } - memspace->base_allocator = base_allocator; memspace->linear_allocator = linear_allocator; memspace->size = numIds; @@ -75,6 +68,6 @@ umfMemspaceCreateFromNumaArray(size_t *nodeIds, size_t numIds, err_nodes_alloc: umf_ba_linear_destroy(linear_allocator); err_umf_ba_linear_create: - umf_ba_free(base_allocator, memspace); + umf_ba_global_free(memspace, sizeof(struct umf_memspace_t)); return ret; } diff --git a/src/pool/pool_base.c b/src/pool/pool_base.c new file mode 100644 index 000000000..adf4f0017 --- /dev/null +++ b/src/pool/pool_base.c @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +/* A MT-safe base allocator */ + +#include "base_alloc_global.h" +#include "base_alloc_internal.h" +#include "utils_common.h" +#include "utils_math.h" + +#include +#include + +#include +#include +#include + +// TODO: we can make this a param to the pool +// allocation classes need to be powers of 2 +#define ALLOCATION_CLASSES \ + { 16, 32, 64, 128 } +#define NUM_ALLOCATION_CLASSES 4 + +struct base_memory_pool { + umf_memory_provider_handle_t hProvider; + size_t ac_sizes[NUM_ALLOCATION_CLASSES]; + umf_ba_pool_t *ac[NUM_ALLOCATION_CLASSES]; + size_t smallest_ac_size_log2; +}; + +// returns index of the allocation class for a give size +static int size_to_idx(struct base_memory_pool *pool, size_t size) { + assert(size <= pool->ac_sizes[NUM_ALLOCATION_CLASSES - 1]); + + if (size <= pool->ac_sizes[0]) { + return 0; + } + + int isPowerOf2 = (0 == (size & (size - 1))); + int index = + (int)(log2Utils(size) + !isPowerOf2 - pool->smallest_ac_size_log2); + + assert(index >= 0); + assert(index < NUM_ALLOCATION_CLASSES); + + return index; +} + +static umf_result_t base_pool_initialize(umf_memory_provider_handle_t hProvider, + void *params, void **ppPool) { + (void)params; // unused + + struct base_memory_pool *pool = + umf_ba_global_alloc(sizeof(struct base_memory_pool)); + if (!pool) { + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + *pool = (struct base_memory_pool){.ac_sizes = ALLOCATION_CLASSES}; + + for (int i = 0; i < NUM_ALLOCATION_CLASSES; i++) { + // allocation classes need to be powers of 2 + assert(0 == (pool->ac_sizes[i] & (pool->ac_sizes[i] - 1))); + pool->ac[i] = umf_ba_create(pool->ac_sizes[i]); + if (!pool->ac[i]) { + fprintf(stderr, + "Cannot create base alloc allocation class for size: %zu\n", + pool->ac_sizes[i]); + } + } + + size_t smallestSize = pool->ac_sizes[0]; + pool->smallest_ac_size_log2 = log2Utils(smallestSize); + + pool->hProvider = hProvider; + *ppPool = (void *)pool; + + return UMF_RESULT_SUCCESS; +} + +static void base_pool_finalize(void *p) { + struct base_memory_pool *pool = (struct base_memory_pool *)p; + for (int i = 0; i < NUM_ALLOCATION_CLASSES; i++) { + if (pool->ac[i]) { + umf_ba_destroy(pool->ac[i]); + } + } + umf_ba_global_free(pool, sizeof(struct base_memory_pool)); +} + +static void *base_aligned_malloc(void *p, size_t size, size_t alignment) { + struct base_memory_pool *pool = (struct base_memory_pool *)p; + assert(pool); + + if (alignment > 8) { + return NULL; + } + + if (size > pool->ac_sizes[NUM_ALLOCATION_CLASSES - 1] || + !pool->ac[size_to_idx(pool, size)]) { + return ba_os_alloc(size); + } + + return umf_ba_alloc(pool->ac[size_to_idx(pool, size)]); +} + +static void *base_malloc(void *pool, size_t size) { + assert(pool); + return base_aligned_malloc(pool, size, 0); +} + +static void *base_calloc(void *pool, size_t num, size_t size) { + assert(pool); + void *ptr = base_malloc(pool, num * size); + + if (ptr) { + // TODO: memory might not be accessible on the host + memset(ptr, 0, num * size); + } + + return ptr; +} + +static void *base_realloc(void *pool, void *ptr, size_t size) { + assert(pool); + + (void)pool; + (void)ptr; + (void)size; + + // TODO: implement + return NULL; +} + +static umf_result_t base_free(void *p, void *ptr, size_t size) { + if (!ptr) { + return UMF_RESULT_SUCCESS; + } + + struct base_memory_pool *pool = (struct base_memory_pool *)p; + assert(pool); + + if (size == 0) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (size > pool->ac_sizes[NUM_ALLOCATION_CLASSES - 1] || + !pool->ac[size_to_idx(pool, size)]) { + ba_os_free(ptr, size); + return UMF_RESULT_SUCCESS; + } + + umf_ba_free(pool->ac[size_to_idx(pool, size)], ptr); + + return UMF_RESULT_SUCCESS; +} + +static size_t base_malloc_usable_size(void *pool, void *ptr) { + (void)pool; + (void)ptr; + return 0; +} + +static umf_result_t base_get_last_allocation_error(void *pool) { + (void)pool; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +static umf_memory_pool_ops_t UMF_BASE_POOL_OPS = { + .version = UMF_VERSION_CURRENT, + .initialize = base_pool_initialize, + .finalize = base_pool_finalize, + .malloc = base_malloc, + .calloc = base_calloc, + .realloc = base_realloc, + .aligned_malloc = base_aligned_malloc, + .malloc_usable_size = base_malloc_usable_size, + .free = base_free, + .get_last_allocation_error = base_get_last_allocation_error}; + +umf_memory_pool_ops_t *umfBasePoolOps(void) { return &UMF_BASE_POOL_OPS; } diff --git a/src/pool/pool_disjoint.cpp b/src/pool/pool_disjoint.cpp index f52cb1b2d..0edd54ed3 100644 --- a/src/pool/pool_disjoint.cpp +++ b/src/pool/pool_disjoint.cpp @@ -44,7 +44,7 @@ class DisjointPool { void *realloc(void *, size_t); void *aligned_malloc(size_t size, size_t alignment); size_t malloc_usable_size(void *); - umf_result_t free(void *ptr); + umf_result_t free(void *ptr, size_t size); umf_result_t get_last_allocation_error(); DisjointPool(); @@ -1022,7 +1022,7 @@ size_t DisjointPool::malloc_usable_size(void *) { return 0; } -umf_result_t DisjointPool::free(void *ptr) try { +umf_result_t DisjointPool::free(void *ptr, size_t) try { bool ToPool; impl->deallocate(ptr, ToPool); diff --git a/src/pool/pool_jemalloc.c b/src/pool/pool_jemalloc.c index 896a7147a..0946089af 100644 --- a/src/pool/pool_jemalloc.c +++ b/src/pool/pool_jemalloc.c @@ -24,9 +24,6 @@ typedef struct jemalloc_memory_pool_t { umf_memory_provider_handle_t provider; unsigned int arena_index; // index of jemalloc arena - - // saved pointer to the global base allocator - umf_ba_pool_t *base_allocator; } jemalloc_memory_pool_t; static __TLS umf_result_t TLS_last_allocation_error; @@ -287,8 +284,9 @@ static void *je_malloc(void *pool, size_t size) { return ptr; } -static umf_result_t je_free(void *pool, void *ptr) { +static umf_result_t je_free(void *pool, void *ptr, size_t size) { (void)pool; // unused + (void)size; assert(pool); if (ptr != NULL) { @@ -361,18 +359,12 @@ static umf_result_t je_initialize(umf_memory_provider_handle_t provider, size_t unsigned_size = sizeof(unsigned); int err; - umf_ba_pool_t *base_allocator = - umf_ba_get_pool(sizeof(jemalloc_memory_pool_t)); - if (!base_allocator) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - jemalloc_memory_pool_t *pool = umf_ba_alloc(base_allocator); + jemalloc_memory_pool_t *pool = + umf_ba_global_alloc(sizeof(jemalloc_memory_pool_t)); if (!pool) { return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - pool->base_allocator = base_allocator; pool->provider = provider; unsigned arena_index; @@ -403,7 +395,7 @@ static umf_result_t je_initialize(umf_memory_provider_handle_t provider, return UMF_RESULT_SUCCESS; err_free_pool: - umf_ba_free(base_allocator, pool); + umf_ba_global_free(pool, sizeof(jemalloc_memory_pool_t)); return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; } @@ -414,7 +406,7 @@ static void je_finalize(void *pool) { snprintf(cmd, sizeof(cmd), "arena.%u.destroy", je_pool->arena_index); mallctl(cmd, NULL, 0, NULL, 0); pool_by_arena_index[je_pool->arena_index] = NULL; - umf_ba_free(je_pool->base_allocator, je_pool); + umf_ba_global_free(je_pool, sizeof(jemalloc_memory_pool_t)); } static size_t je_malloc_usable_size(void *pool, void *ptr) { diff --git a/src/pool/pool_proxy.c b/src/pool/pool_proxy.c index 100af920f..c0429c012 100644 --- a/src/pool/pool_proxy.c +++ b/src/pool/pool_proxy.c @@ -89,7 +89,8 @@ static void *proxy_realloc(void *pool, void *ptr, size_t size) { return NULL; } -static umf_result_t proxy_free(void *pool, void *ptr) { +static umf_result_t proxy_free(void *pool, void *ptr, size_t size) { + (void)size; assert(pool); struct proxy_memory_pool *hPool = (struct proxy_memory_pool *)pool; diff --git a/src/pool/pool_scalable.c b/src/pool/pool_scalable.c index 2e1ecb78a..b0200c20e 100644 --- a/src/pool/pool_scalable.c +++ b/src/pool/pool_scalable.c @@ -55,9 +55,6 @@ struct tbb_callbacks { struct tbb_memory_pool { umf_memory_provider_handle_t mem_provider; void *tbb_pool; - - // saved pointer to the global base allocator - umf_ba_pool_t *base_allocator; }; static struct tbb_callbacks g_tbb_ops; @@ -150,19 +147,13 @@ static umf_result_t tbb_pool_initialize(umf_memory_provider_handle_t provider, return UMF_RESULT_ERROR_UNKNOWN; } - umf_ba_pool_t *base_allocator = - umf_ba_get_pool(sizeof(struct tbb_memory_pool)); - if (!base_allocator) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - - struct tbb_memory_pool *pool_data = umf_ba_alloc(base_allocator); + struct tbb_memory_pool *pool_data = + umf_ba_global_alloc(sizeof(struct tbb_memory_pool)); if (!pool_data) { fprintf(stderr, "cannot allocate memory for metadata\n"); return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - pool_data->base_allocator = base_allocator; pool_data->mem_provider = provider; int ret = g_tbb_ops.pool_create_v1((intptr_t)pool_data, &policy, &(pool_data->tbb_pool)); @@ -179,7 +170,7 @@ static void tbb_pool_finalize(void *pool) { pthread_once(&tbb_is_initialized, load_tbb_symbols); struct tbb_memory_pool *pool_data = (struct tbb_memory_pool *)pool; g_tbb_ops.pool_destroy(pool_data->tbb_pool); - umf_ba_free(pool_data->base_allocator, pool_data); + umf_ba_global_free(pool_data, sizeof(struct tbb_memory_pool)); } static void *tbb_malloc(void *pool, size_t size) { @@ -238,7 +229,8 @@ static void *tbb_aligned_malloc(void *pool, size_t size, size_t alignment) { return ptr; } -static umf_result_t tbb_free(void *pool, void *ptr) { +static umf_result_t tbb_free(void *pool, void *ptr, size_t size) { + (void)size; if (ptr == NULL) { return UMF_RESULT_SUCCESS; } diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index 6796a4294..8392a5e18 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -47,9 +47,6 @@ typedef struct os_memory_provider_t { int traces; // log level of debug traces hwloc_topology_t topo; - - // saved pointer to the global base allocator - umf_ba_pool_t *base_allocator; } os_memory_provider_t; #define TLS_MSG_BUF_LEN 1024 @@ -275,20 +272,14 @@ static umf_result_t os_initialize(void *params, void **provider) { return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - umf_ba_pool_t *base_allocator = - umf_ba_get_pool(sizeof(os_memory_provider_t)); - if (!base_allocator) { - return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; - } - os_memory_provider_t *os_provider = - (os_memory_provider_t *)umf_ba_alloc(base_allocator); + (os_memory_provider_t *)umf_ba_global_alloc( + sizeof(os_memory_provider_t)); if (!os_provider) { return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } memset(os_provider, 0, sizeof(*os_provider)); - os_provider->base_allocator = base_allocator; int r = hwloc_topology_init(&os_provider->topo); if (r) { @@ -333,7 +324,7 @@ static umf_result_t os_initialize(void *params, void **provider) { err_destroy_hwloc_topology: hwloc_topology_destroy(os_provider->topo); err_free_os_provider: - umf_ba_free(base_allocator, os_provider); + umf_ba_global_free(os_provider, sizeof(os_memory_provider_t)); return ret; } @@ -346,7 +337,7 @@ static void os_finalize(void *provider) { os_memory_provider_t *os_provider = provider; hwloc_bitmap_free(os_provider->nodeset); hwloc_topology_destroy(os_provider->topo); - umf_ba_free(os_provider->base_allocator, os_provider); + umf_ba_global_free(os_provider, sizeof(os_memory_provider_t)); } static umf_result_t os_get_min_page_size(void *provider, void *ptr, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 410623bf1..776969e63 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -125,3 +125,9 @@ add_umf_test(NAME base_alloc add_umf_test(NAME base_alloc_linear SRCS ${BA_SOURCES_FOR_TEST} test_base_alloc_linear.cpp LIBS umf_utils) +if(LINUX) + # TODO: fix this on windows + add_umf_test(NAME base_alloc_global + SRCS ${BA_SOURCES_FOR_TEST} pools/pool_base_alloc.cpp malloc_compliance_tests.cpp + LIBS umf_utils) +endif() diff --git a/test/common/pool.hpp b/test/common/pool.hpp index 488c27663..368243973 100644 --- a/test/common/pool.hpp +++ b/test/common/pool.hpp @@ -82,6 +82,22 @@ bool isCallocSupported(umf_memory_pool_handle_t hPool) { return supported; } +bool isAlignedAllocSupported(umf_memory_pool_handle_t hPool) { + static constexpr size_t allocSize = 8; + static constexpr size_t alignment = 8; + auto *ptr = umfPoolAlignedMalloc(hPool, allocSize, alignment); + + if (ptr) { + umfPoolFree(hPool, ptr); + return true; + } else if (umfPoolGetLastAllocationError(hPool) == + UMF_RESULT_ERROR_NOT_SUPPORTED) { + return false; + } else { + throw std::runtime_error("AlignedMalloc failed with unexpected error"); + } +} + typedef struct pool_base_t { umf_result_t initialize(umf_memory_provider_handle_t) noexcept { return UMF_RESULT_SUCCESS; @@ -91,7 +107,7 @@ typedef struct pool_base_t { void *realloc(void *, size_t) noexcept { return nullptr; } void *aligned_malloc(size_t, size_t) noexcept { return nullptr; } size_t malloc_usable_size(void *) noexcept { return 0; } - umf_result_t free(void *) noexcept { return UMF_RESULT_SUCCESS; } + umf_result_t free(void *, size_t) noexcept { return UMF_RESULT_SUCCESS; } umf_result_t get_last_allocation_error() noexcept { return UMF_RESULT_SUCCESS; } @@ -122,7 +138,7 @@ struct malloc_pool : public pool_base_t { return ::malloc_usable_size(ptr); #endif } - umf_result_t free(void *ptr) noexcept { + umf_result_t free(void *ptr, size_t) noexcept { ::free(ptr); return UMF_RESULT_SUCCESS; } diff --git a/test/common/pool_null.c b/test/common/pool_null.c index c34bcfc16..fec76f5e7 100644 --- a/test/common/pool_null.c +++ b/test/common/pool_null.c @@ -52,9 +52,10 @@ static size_t nullMallocUsableSize(void *pool, void *ptr) { return 0; } -static umf_result_t nullFree(void *pool, void *ptr) { +static umf_result_t nullFree(void *pool, void *ptr, size_t size) { (void)pool; (void)ptr; + (void)size; return UMF_RESULT_SUCCESS; } diff --git a/test/common/pool_trace.c b/test/common/pool_trace.c index 83221be90..7199cc6b8 100644 --- a/test/common/pool_trace.c +++ b/test/common/pool_trace.c @@ -73,7 +73,8 @@ static size_t traceMallocUsableSize(void *pool, void *ptr) { return umfPoolMallocUsableSize(trace_pool->params.hUpstreamPool, ptr); } -static umf_result_t traceFree(void *pool, void *ptr) { +static umf_result_t traceFree(void *pool, void *ptr, size_t size) { + (void)size; trace_pool_t *trace_pool = (trace_pool_t *)pool; trace_pool->params.trace("free"); diff --git a/test/common/provider.hpp b/test/common/provider.hpp index 2389d0bf6..20d547a81 100644 --- a/test/common/provider.hpp +++ b/test/common/provider.hpp @@ -63,6 +63,9 @@ typedef struct provider_base_t { const char *get_name() noexcept { return "base"; } } provider_base_t; +umf_memory_provider_ops_t BASE_PROVIDER_OPS = + umf::providerMakeCOps(); + struct provider_malloc : public provider_base_t { umf_result_t alloc(size_t size, size_t align, void **ptr) noexcept { if (!align) { diff --git a/test/memoryPoolAPI.cpp b/test/memoryPoolAPI.cpp index d8bc28577..3f23bc715 100644 --- a/test/memoryPoolAPI.cpp +++ b/test/memoryPoolAPI.cpp @@ -183,12 +183,6 @@ TEST_P(umfPoolWithCreateFlagsTest, umfPoolCreateFlagsNullPoolHandle) { umfMemoryProviderDestroy(provider); } -TEST_P(umfPoolWithCreateFlagsTest, umfPoolCreateFlagsInvalidProviders) { - umf_memory_pool_handle_t hPool; - auto ret = umfPoolCreate(&MALLOC_POOL_OPS, nullptr, nullptr, flags, &hPool); - ASSERT_EQ(ret, UMF_RESULT_ERROR_INVALID_ARGUMENT); -} - struct poolInitializeTest : umf_test::test, ::testing::WithParamInterface {}; diff --git a/test/poolFixtures.hpp b/test/poolFixtures.hpp index 152476646..522af1c6e 100644 --- a/test/poolFixtures.hpp +++ b/test/poolFixtures.hpp @@ -136,6 +136,9 @@ TEST_P(umfPoolTest, callocFree) { } void pow2AlignedAllocHelper(umf_memory_pool_handle_t pool) { + if (!umf_test::isAlignedAllocSupported(pool)) { + GTEST_SKIP(); + } static constexpr size_t maxAlignment = (1u << 22); static constexpr size_t numAllocs = 4; for (size_t alignment = 1; alignment <= maxAlignment; alignment <<= 1) { diff --git a/test/pools/pool_base_alloc.cpp b/test/pools/pool_base_alloc.cpp new file mode 100644 index 000000000..1500b3ecb --- /dev/null +++ b/test/pools/pool_base_alloc.cpp @@ -0,0 +1,67 @@ +// Copyright (C) 2023 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include + +#include "umf/pools/pool_base.h" + +#include "pool.hpp" +#include "poolFixtures.hpp" +#include "provider.hpp" + +struct base_alloc_pool : public umf_test::pool_base_t { + std::unordered_map sizes; + std::mutex m; + umf_memory_pool_handle_t pool; + + umf_result_t initialize(umf_memory_provider_handle_t) { + auto ops = umfBasePoolOps(); + return umfPoolCreate(ops, nullptr, nullptr, 0, &pool); + } + + void *malloc(size_t size) noexcept { + auto *ptr = umfPoolMalloc(pool, size); + std::unique_lock l(m); + sizes[ptr] = size; + return ptr; + } + void *calloc(size_t, size_t) noexcept { + umf::getPoolLastStatusRef() = + UMF_RESULT_ERROR_NOT_SUPPORTED; + return NULL; + } + void *realloc(void *, size_t) noexcept { + umf::getPoolLastStatusRef() = + UMF_RESULT_ERROR_NOT_SUPPORTED; + return NULL; + } + void *aligned_malloc(size_t, size_t) noexcept { + umf::getPoolLastStatusRef() = + UMF_RESULT_ERROR_NOT_SUPPORTED; + return NULL; + } + size_t malloc_usable_size(void *ptr) noexcept { + std::unique_lock l(m); + return sizes[ptr]; + } + umf_result_t free(void *ptr, size_t) noexcept { + size_t size; + { + std::unique_lock l(m); + size = sizes[ptr]; + } + + return umfPoolFreeSized(pool, ptr, size); + } + umf_result_t get_last_allocation_error() { + return umf::getPoolLastStatusRef(); + } +}; + +umf_memory_pool_ops_t BA_POOL_OPS = umf::poolMakeCOps(); + +INSTANTIATE_TEST_SUITE_P(baPool, umfPoolTest, + ::testing::Values(poolCreateExtParams{ + &BA_POOL_OPS, nullptr, + &umf_test::BASE_PROVIDER_OPS, nullptr}));