diff --git a/.github/workflows/.spellcheck-conf.toml b/.github/workflows/.spellcheck-conf.toml index 288af6a19..bb0f480d6 100644 --- a/.github/workflows/.spellcheck-conf.toml +++ b/.github/workflows/.spellcheck-conf.toml @@ -1,6 +1,6 @@ [default] # Don't correct the following words: -extend-ignore-words-re = ["ASSER", "Tne", "ba", "BA", "PN"] +extend-ignore-words-re = ["ASSER", "Tne", "ba", "BA", "PN", "usm"] [files] # completely exclude those files from consideration: diff --git a/docs/config/api.rst b/docs/config/api.rst index 879623428..609ffc147 100644 --- a/docs/config/api.rst +++ b/docs/config/api.rst @@ -170,6 +170,26 @@ Memtarget .. doxygenfile:: experimental/memtarget.h :sections: define enum typedef func +Memory Properties +========================================== + +Memory properties in UMF describe the characteristics and capabilities of +different memory regions or allocations. These properties can include +information such as memory type, allocation size, context and device used for +allocation, and other attributes that are relevant for memory management. + +The Memory Properties API allows users to retrieve and interpret these +attributes for memory managed by UMF, enabling advanced memory management +strategies and improved interoperability with heterogeneous systems. + +.. note:: + The memory properties APIs are experimental and may change in future releases. + +Memory Properties +------------------------------------------ +.. doxygenfile:: experimental/memory_properties.h + :sections: define enum typedef func var + Inter-Process Communication ========================================== diff --git a/docs/config/spelling_exceptions.txt b/docs/config/spelling_exceptions.txt index 54acc5e73..3385a2216 100644 --- a/docs/config/spelling_exceptions.txt +++ b/docs/config/spelling_exceptions.txt @@ -3,22 +3,24 @@ allocatable allocator allocators calloc -CXL copyable +CUcontext +CUdevice customizable +CXL daxX -deallocation deallocating +deallocation deallocations -Devdax dev +Devdax Globals +highPtr hMemtarget hPool hProvider -highPtr -io interprocess +io ipc jemalloc lowPtr @@ -48,8 +50,9 @@ partList pid poolable preallocated -providerIpcData +propertyId providential +providerIpcData ptr realloc Scalable @@ -58,11 +61,16 @@ stdout Tiering tiering topologies +uint +uintptr umf umfGetIPCHandle +umfGetMemoryPropertySize umfMemoryProviderAlloc umfMemoryProviderGetLastNativeError umfMemoryProviderOpenIPCHandle +umfMemspaceMemtargetAdd +umfMemspaceUserFilter umfOsMemoryProviderParamsDestroy umfPool umfPoolCalloc @@ -73,4 +81,6 @@ umfPoolRealloc umfMemspaceUserFilter umfMemspaceMemtargetAdd unfreed +usm zA +ze diff --git a/include/umf/base.h b/include/umf/base.h index f7cd9de63..096934b18 100644 --- a/include/umf/base.h +++ b/include/umf/base.h @@ -51,6 +51,33 @@ typedef enum umf_result_t { UMF_RESULT_ERROR_UNKNOWN = 0x7ffffffe ///< Unknown error } umf_result_t; +/// @brief Handle to the memory properties structure +typedef struct umf_memory_properties_t *umf_memory_properties_handle_t; + +/// @brief ID of the memory property +typedef enum umf_memory_property_id_t { + UMF_MEMORY_PROPERTY_INVALID = -1, ///< Invalid property + + // UMF specific + UMF_MEMORY_PROPERTY_PROVIDER_HANDLE = 0, ///< Handle to the memory provider + UMF_MEMORY_PROPERTY_POOL_HANDLE = 1, ///< Handle to the memory pool + + // generic pointer properties + UMF_MEMORY_PROPERTY_BASE_ADDRESS = 10, ///< Base address of the allocation + UMF_MEMORY_PROPERTY_BASE_SIZE = 11, ///< Base size of the allocation + UMF_MEMORY_PROPERTY_BUFFER_ID = 12, ///< Unique identifier for the buffer + + // GPU specific + UMF_MEMORY_PROPERTY_POINTER_TYPE = 20, ///< Type of the pointer + UMF_MEMORY_PROPERTY_CONTEXT = 21, ///< GPU context of the allocation + UMF_MEMORY_PROPERTY_DEVICE = + 22, ///< GPU device where the allocation resides + + /// @cond + UMF_MEMORY_PROPERTY_MAX_RESERVED = 0x1000, ///< Maximum reserved value + /// @endcond +} umf_memory_property_id_t; + /// @brief Type of the CTL query typedef enum umf_ctl_query_type { CTL_QUERY_READ, diff --git a/include/umf/experimental/memory_properties.h b/include/umf/experimental/memory_properties.h new file mode 100644 index 000000000..fe44ef592 --- /dev/null +++ b/include/umf/experimental/memory_properties.h @@ -0,0 +1,64 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_MEMORY_PROPERTIES_H +#define UMF_MEMORY_PROPERTIES_H 1 + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// @brief Get the memory properties handle for a given pointer +/// \details +/// The handle returned by this function is valid until the memory pointed +/// to by the pointer is freed. +/// @param ptr pointer to the allocated memory +/// @param props_handle [out] pointer to the memory properties handle +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure +umf_result_t +umfGetMemoryPropertiesHandle(const void *ptr, + umf_memory_properties_handle_t *props_handle); + +/// @brief Get the size of a specific memory property +/// \details +/// The size of the property should be used to allocate a buffer to hold the +/// value of the property. +/// @param props_handle handle to the memory properties +/// @param memory_property_id ID of the memory property to get the size of +/// @param size [out] pointer to the size of the property +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure +umf_result_t +umfGetMemoryPropertySize(umf_memory_properties_handle_t props_handle, + umf_memory_property_id_t memory_property_id, + size_t *size); + +/// @brief Get a specific memory property from the properties handle +/// \details +/// The type of the property value depends on the property ID. The size of +/// the property value buffer must be large enough to hold the +/// value of the property. The size of the property can be obtained by +/// calling umfGetMemoryPropertySize() with the same property ID. +/// @param props_handle handle to the memory properties +/// @param memory_property_id ID of the memory property to get +/// @param property_value [out] pointer to the value of the memory property +/// which will be filled +/// @param max_property_size size of the property value buffer +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure +umf_result_t umfGetMemoryProperty(umf_memory_properties_handle_t props_handle, + umf_memory_property_id_t memory_property_id, + void *property_value, + size_t max_property_size); + +#ifdef __cplusplus +} +#endif + +#endif /* UMF_MEMORY_PROPERTIES_H */ diff --git a/include/umf/memory_provider_ops.h b/include/umf/memory_provider_ops.h index 5ea5e0a87..a520ed889 100644 --- a/include/umf/memory_provider_ops.h +++ b/include/umf/memory_provider_ops.h @@ -21,7 +21,7 @@ extern "C" { /// @brief Version of the Memory Provider ops structure. /// NOTE: This is equal to the latest UMF version, in which the ops structure /// has been modified. -#define UMF_PROVIDER_OPS_VERSION_CURRENT UMF_MAKE_VERSION(1, 0) +#define UMF_PROVIDER_OPS_VERSION_CURRENT UMF_MAKE_VERSION(1, 1) /// /// @brief This structure comprises function pointers used by corresponding @@ -288,6 +288,40 @@ typedef struct umf_memory_provider_ops_t { const char *name, void *arg, size_t size, umf_ctl_query_type_t queryType, va_list args); + // The following operations were added in ops version 1.1 + + /// + /// @brief Retrieve provider-specific properties of the memory allocation. + /// \details + /// If provider supports allocation properties, + /// ext_get_allocation_properties and ext_get_allocation_properties_size, + /// must either be all set or all NULL. + /// @param provider pointer to the memory provider + /// @param ptr pointer to the allocated memory + /// @param memory_property_id ID of the memory property + /// @param property_value [out] pointer to the value of the memory property + /// which will be filled + /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure + /// + umf_result_t (*ext_get_allocation_properties)( + void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, void *property_value); + + /// @brief Retrieve size of the provider-specific properties of the memory + /// allocation. + /// \details + /// If provider supports allocation properties, + /// ext_get_allocation_properties and ext_get_allocation_properties_size, + /// must either be all set or all NULL. + /// @param provider pointer to the memory provider + /// @param memory_property_id ID of the memory property to get the size of + /// @param size [out] pointer to the size of the property + /// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure + /// + umf_result_t (*ext_get_allocation_properties_size)( + void *provider, umf_memory_property_id_t memory_property_id, + size_t *size); + } umf_memory_provider_ops_t; #ifdef __cplusplus diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 990a75dc2..294839309 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -42,6 +42,7 @@ set(UMF_SOURCES ipc.c ipc_cache.c memory_pool.c + memory_properties.c memory_provider.c memory_provider_get_last_failed.c memtarget.c diff --git a/src/ipc.c b/src/ipc.c index d4e5cc806..29ed5bac2 100644 --- a/src/ipc.c +++ b/src/ipc.c @@ -58,14 +58,19 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, } size_t ipcHandleSize = 0; - umf_alloc_info_t allocInfo; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); + umf_memory_properties_handle_t props = NULL; + umf_result_t ret = umfGetMemoryPropertiesHandle(ptr, &props); if (ret != UMF_RESULT_SUCCESS) { - LOG_ERR("cannot get alloc info for ptr = %p.", ptr); + LOG_ERR("cannot get alloc props for ptr = %p.", ptr); return ret; } - ret = umfPoolGetIPCHandleSize(allocInfo.pool, &ipcHandleSize); + if (props == NULL || props->pool == NULL) { + LOG_ERR("cannot get pool from alloc info for ptr = %p.", ptr); + return UMF_RESULT_ERROR_UNKNOWN; + } + + ret = umfPoolGetIPCHandleSize(props->pool, &ipcHandleSize); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("cannot get IPC handle size."); return ret; @@ -79,11 +84,14 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, // We cannot use umfPoolGetMemoryProvider function because it returns // upstream provider but we need tracking one - umf_memory_provider_handle_t provider = allocInfo.pool->provider; - assert(provider); + if (props->pool->provider == NULL) { + LOG_ERR("cannot get memory provider from pool"); + umf_ba_global_free(ipcData); + return UMF_RESULT_ERROR_UNKNOWN; + } + umf_memory_provider_handle_t provider = props->pool->provider; - ret = umfMemoryProviderGetIPCHandle(provider, allocInfo.base, - allocInfo.baseSize, + ret = umfMemoryProviderGetIPCHandle(provider, props->base, props->base_size, (void *)ipcData->providerIpcData); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("failed to get IPC handle."); @@ -92,10 +100,10 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, } // ipcData->handle_id is filled by tracking provider - ipcData->base = allocInfo.base; + ipcData->base = props->base; ipcData->pid = utils_getpid(); - ipcData->baseSize = allocInfo.baseSize; - ipcData->offset = (uintptr_t)ptr - (uintptr_t)allocInfo.base; + ipcData->baseSize = props->base_size; + ipcData->offset = (uintptr_t)ptr - (uintptr_t)props->base; *umfIPCHandle = ipcData; *size = ipcHandleSize; diff --git a/src/libumf.def b/src/libumf.def index 9496343a0..68163c6b5 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -6,7 +6,7 @@ LIBRARY UMF -VERSION 1.0 +VERSION 1.1 EXPORTS DllMain @@ -149,6 +149,9 @@ EXPORTS umfDevDaxMemoryProviderParamsSetName umfFileMemoryProviderParamsSetName umfFixedMemoryProviderParamsSetName + umfGetMemoryPropertiesHandle + umfGetMemoryProperty + umfGetMemoryPropertySize umfJemallocPoolParamsSetName umfLevelZeroMemoryProviderParamsSetName umfOsMemoryProviderParamsSetName diff --git a/src/libumf.map b/src/libumf.map index 4e482c649..98b913499 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -147,6 +147,9 @@ UMF_1.1 { umfDevDaxMemoryProviderParamsSetName; umfFileMemoryProviderParamsSetName; umfFixedMemoryProviderParamsSetName; + umfGetMemoryPropertiesHandle; + umfGetMemoryProperty; + umfGetMemoryPropertySize; umfJemallocPoolParamsSetName; umfLevelZeroMemoryProviderParamsSetName; umfOsMemoryProviderParamsSetName; diff --git a/src/memory_pool.c b/src/memory_pool.c index 692c0a1b3..80dd05342 100644 --- a/src/memory_pool.c +++ b/src/memory_pool.c @@ -653,9 +653,18 @@ umf_result_t umfFree(void *ptr) { } umf_result_t umfPoolByPtr(const void *ptr, umf_memory_pool_handle_t *pool) { - UMF_CHECK((pool != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); - *pool = umfMemoryTrackerGetPool(ptr); - return *pool ? UMF_RESULT_SUCCESS : UMF_RESULT_ERROR_INVALID_ARGUMENT; + UMF_CHECK(pool != NULL, UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK(ptr != NULL, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_memory_properties_handle_t props = NULL; + umf_result_t ret = umfGetMemoryPropertiesHandle(ptr, &props); + if (ret != UMF_RESULT_SUCCESS || props == NULL || props->pool == NULL) { + *pool = NULL; + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + *pool = props->pool; + return UMF_RESULT_SUCCESS; } umf_result_t umfPoolGetMemoryProvider(umf_memory_pool_handle_t hPool, diff --git a/src/memory_properties.c b/src/memory_properties.c new file mode 100644 index 000000000..11d4e5593 --- /dev/null +++ b/src/memory_properties.c @@ -0,0 +1,158 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include + +#include +#include +#include + +#include "memory_properties_internal.h" +#include "memory_provider_internal.h" +#include "provider/provider_tracking.h" + +umf_result_t +umfGetMemoryPropertiesHandle(const void *ptr, + umf_memory_properties_handle_t *props_handle) { + UMF_CHECK((props_handle != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + + tracker_alloc_info_t *info = NULL; + umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &info); + if (ret == UMF_RESULT_SUCCESS) { + *props_handle = &info->props; + return UMF_RESULT_SUCCESS; + } + + // try to get IPC info + umf_ipc_info_t ipc_info; + ret = umfMemoryTrackerGetIpcInfo(ptr, &ipc_info); + if (ret == UMF_RESULT_SUCCESS) { + *props_handle = ipc_info.props; + return UMF_RESULT_SUCCESS; + } + + LOG_ERR("Failed to get memory properties handle for ptr=%p", ptr); + return ret; +} + +umf_result_t +umfGetMemoryPropertySize(umf_memory_properties_handle_t props_handle, + umf_memory_property_id_t memory_property_id, + size_t *size) { + UMF_CHECK((size != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_INVALID: + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + case UMF_MEMORY_PROPERTY_PROVIDER_HANDLE: + *size = sizeof(umf_memory_provider_handle_t); + return UMF_RESULT_SUCCESS; + case UMF_MEMORY_PROPERTY_POOL_HANDLE: + *size = sizeof(umf_memory_pool_handle_t); + return UMF_RESULT_SUCCESS; + case UMF_MEMORY_PROPERTY_BASE_ADDRESS: + *size = sizeof(uintptr_t); + return UMF_RESULT_SUCCESS; + case UMF_MEMORY_PROPERTY_BASE_SIZE: + *size = sizeof(size_t); + return UMF_RESULT_SUCCESS; + case UMF_MEMORY_PROPERTY_BUFFER_ID: + *size = sizeof(uint64_t); + return UMF_RESULT_SUCCESS; + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + *size = sizeof(umf_usm_memory_type_t); + return UMF_RESULT_SUCCESS; + default: + break; + } + + // custom memory properties should be handled by the user provider + umf_memory_provider_t *provider = props_handle->provider; + if (provider->ops.ext_get_allocation_properties_size) { + return provider->ops.ext_get_allocation_properties_size( + provider->provider_priv, memory_property_id, size); + } + + LOG_ERR("Unknown memory property ID: %d", memory_property_id); + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +umf_result_t umfGetMemoryProperty(umf_memory_properties_handle_t props_handle, + umf_memory_property_id_t memory_property_id, + void *value, size_t max_property_size) { + UMF_CHECK((value != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((props_handle != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((max_property_size > 0), UMF_RESULT_ERROR_INVALID_ARGUMENT); + + umf_memory_provider_t *provider = props_handle->provider; + + size_t property_size = 0; + umf_result_t ret = umfGetMemoryPropertySize( + props_handle, memory_property_id, &property_size); + if (UNLIKELY(ret != UMF_RESULT_SUCCESS)) { + LOG_ERR("Failed to get memory property size for ID %d", + memory_property_id); + return ret; + } + + if (UNLIKELY(property_size > max_property_size)) { + LOG_ERR("Memory property size %zu exceeds max size %zu for ID %d", + property_size, max_property_size, memory_property_id); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_INVALID: + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + + case UMF_MEMORY_PROPERTY_POOL_HANDLE: + *(umf_memory_pool_handle_t *)value = props_handle->pool; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_PROVIDER_HANDLE: + *(umf_memory_provider_handle_t *)value = provider; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_BUFFER_ID: + *(uint64_t *)value = props_handle->id; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_BASE_ADDRESS: + *(uintptr_t *)value = (uintptr_t)props_handle->base; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_BASE_SIZE: + *(size_t *)value = props_handle->base_size; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + // NOTE: this property is "cached" in the props_handle but the value is + // determined by the memory provider and set during addition to the + // tracker. + *(umf_usm_memory_type_t *)value = props_handle->memory_type; + return UMF_RESULT_SUCCESS; + + // GPU Memory Provider specific properties - should be handled by the + // provider + case UMF_MEMORY_PROPERTY_CONTEXT: + case UMF_MEMORY_PROPERTY_DEVICE: + default: + break; + }; + + // custom memory properties should be handled by the user provider + if (provider->ops.ext_get_allocation_properties) { + return provider->ops.ext_get_allocation_properties( + provider->provider_priv, props_handle->base, memory_property_id, + value); + } + + LOG_ERR("Unknown memory property ID: %d", memory_property_id); + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} diff --git a/src/memory_properties_internal.h b/src/memory_properties_internal.h new file mode 100644 index 000000000..3f2d6ef3a --- /dev/null +++ b/src/memory_properties_internal.h @@ -0,0 +1,45 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_MEMORY_PROPERTIES_INTERNAL_H +#define UMF_MEMORY_PROPERTIES_INTERNAL_H 1 + +#include + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct umf_memory_properties_t { + umf_memory_pool_handle_t pool; + umf_memory_provider_handle_t provider; + uint64_t id; + void *base; + size_t base_size; + umf_usm_memory_type_t memory_type; +} umf_memory_properties_t; + +umf_result_t umfMemoryProviderGetAllocationProperties( + umf_memory_provider_handle_t hProvider, const void *ptr, + umf_memory_property_id_t propertyId, void *property_value); + +umf_result_t umfMemoryProviderGetAllocationPropertiesSize( + umf_memory_provider_handle_t hProvider, umf_memory_property_id_t propertyId, + size_t *size); + +#ifdef __cplusplus +} +#endif + +#endif // UMF_MEMORY_PROPERTIES_INTERNAL_H diff --git a/src/memory_provider.c b/src/memory_provider.c index eb7e88ead..324fa751b 100644 --- a/src/memory_provider.c +++ b/src/memory_provider.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -134,6 +135,25 @@ umfDefaultCtlHandle(void *provider, umf_ctl_query_source_t operationType, return UMF_RESULT_ERROR_NOT_SUPPORTED; } +static umf_result_t +umfDefaultGetAllocationProperties(void *provider, const void *ptr, + umf_memory_property_id_t propertyId, + void *propertyValue) { + (void)provider; + (void)ptr; + (void)propertyId; + (void)propertyValue; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +static umf_result_t umfDefaultGetAllocationPropertiesSize( + void *provider, umf_memory_property_id_t propertyId, size_t *size) { + (void)provider; + (void)propertyId; + (void)size; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { if (!ops->ext_purge_lazy) { ops->ext_purge_lazy = umfDefaultPurgeLazy; @@ -154,6 +174,15 @@ void assignOpsExtDefaults(umf_memory_provider_ops_t *ops) { if (!ops->ext_ctl) { ops->ext_ctl = umfDefaultCtlHandle; } + + if (!ops->ext_get_allocation_properties) { + ops->ext_get_allocation_properties = umfDefaultGetAllocationProperties; + } + + if (!ops->ext_get_allocation_properties_size) { + ops->ext_get_allocation_properties_size = + umfDefaultGetAllocationPropertiesSize; + } } void assignOpsIpcDefaults(umf_memory_provider_ops_t *ops) { @@ -214,6 +243,14 @@ static bool validateOps(const umf_memory_provider_ops_t *ops) { return false; } + if ((ops->ext_get_allocation_properties == NULL) != + (ops->ext_get_allocation_properties_size == NULL)) { + LOG_ERR("ext_get_allocation_properties and " + "ext_get_allocation_properties_size must be " + "both set or both NULL\n"); + return false; + } + return true; } @@ -221,14 +258,36 @@ umf_result_t umfMemoryProviderCreate(const umf_memory_provider_ops_t *ops, const void *params, umf_memory_provider_handle_t *hProvider) { libumfInit(); - if (!ops || !hProvider || !validateOps(ops)) { + if (!ops || !hProvider) { return UMF_RESULT_ERROR_INVALID_ARGUMENT; } + umf_memory_provider_ops_t compatible_ops; if (ops->version != UMF_PROVIDER_OPS_VERSION_CURRENT) { LOG_WARN("Memory Provider ops version \"%d\" is different than the " "current version \"%d\"", ops->version, UMF_PROVIDER_OPS_VERSION_CURRENT); + + // Create a new ops compatible structure with the current version + memset(&compatible_ops, 0, sizeof(compatible_ops)); + + if (UMF_MINOR_VERSION(ops->version) == 0) { + LOG_INFO("Detected 1.0 version of Memory Provider ops, " + "upgrading to current version"); + memcpy(&compatible_ops, ops, + offsetof(umf_memory_provider_ops_t, + ext_get_allocation_properties)); + } else { + LOG_ERR("Unsupported Memory Provider ops version: %d", + ops->version); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + ops = &compatible_ops; + } + + if (!validateOps(ops)) { + return UMF_RESULT_ERROR_INVALID_ARGUMENT; } umf_memory_provider_handle_t provider = @@ -507,3 +566,43 @@ umfMemoryProviderCloseIPCHandle(umf_memory_provider_handle_t hProvider, checkErrorAndSetLastProvider(res, hProvider); return res; } + +umf_result_t umfMemoryProviderGetAllocationProperties( + umf_memory_provider_handle_t hProvider, const void *ptr, + umf_memory_property_id_t propertyId, void *property_value) { + + UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((property_value != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((propertyId != UMF_MEMORY_PROPERTY_INVALID), + UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // NOTE: we do not check if the propertyId is below + // UMF_MEMORY_PROPERTY_MAX_RESERVED value, as the user could use a custom + // property ID that is above the reserved range + + umf_result_t res = hProvider->ops.ext_get_allocation_properties( + hProvider->provider_priv, ptr, propertyId, property_value); + + checkErrorAndSetLastProvider(res, hProvider); + return res; +} + +umf_result_t umfMemoryProviderGetAllocationPropertiesSize( + umf_memory_provider_handle_t hProvider, umf_memory_property_id_t propertyId, + size_t *size) { + + UMF_CHECK((hProvider != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((size != NULL), UMF_RESULT_ERROR_INVALID_ARGUMENT); + UMF_CHECK((propertyId != UMF_MEMORY_PROPERTY_INVALID), + UMF_RESULT_ERROR_INVALID_ARGUMENT); + + // NOTE: we do not check if the propertyId is below + // UMF_MEMORY_PROPERTY_MAX_RESERVED value, as the user could use a custom + // property ID that is above the reserved range + + umf_result_t res = hProvider->ops.ext_get_allocation_properties_size( + hProvider->provider_priv, propertyId, size); + + checkErrorAndSetLastProvider(res, hProvider); + return res; +} diff --git a/src/pool/pool_disjoint.c b/src/pool/pool_disjoint.c index 06bfd58c3..d1327899e 100644 --- a/src/pool/pool_disjoint.c +++ b/src/pool/pool_disjoint.c @@ -982,14 +982,19 @@ umf_result_t disjoint_pool_malloc_usable_size(void *pool, const void *ptr, critnib_release(disjoint_pool->known_slabs, ref_slab); } - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); - if (ret != UMF_RESULT_SUCCESS) { - *size = 0; - return ret; + umf_memory_properties_handle_t props = NULL; + umf_result_t umf_result = umfGetMemoryPropertiesHandle(ptr, &props); + if (umf_result != UMF_RESULT_SUCCESS) { + return umf_result; + } + + if (props == NULL) { + TLS_last_allocation_error = UMF_RESULT_ERROR_UNKNOWN; + LOG_ERR("failed to get allocation info from the memory tracker"); + return UMF_RESULT_ERROR_UNKNOWN; } - *size = allocInfo.baseSize; + *size = props->base_size; return UMF_RESULT_SUCCESS; } @@ -1025,15 +1030,21 @@ umf_result_t disjoint_pool_free(void *pool, void *ptr) { critnib_release(disjoint_pool->known_slabs, ref_slab); } - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); + umf_memory_properties_handle_t props = NULL; + umf_result_t ret = umfGetMemoryPropertiesHandle(ptr, &props); if (ret != UMF_RESULT_SUCCESS) { TLS_last_allocation_error = ret; LOG_ERR("failed to get allocation info from the memory tracker"); return ret; } - size_t size = allocInfo.baseSize; + if (props == NULL) { + TLS_last_allocation_error = UMF_RESULT_ERROR_UNKNOWN; + LOG_ERR("failed to get allocation info from the memory tracker"); + return UMF_RESULT_ERROR_UNKNOWN; + } + + size_t size = props->base_size; umf_memory_provider_handle_t provider = disjoint_pool->provider; ret = umfMemoryProviderFree(provider, ptr, size); if (ret != UMF_RESULT_SUCCESS) { diff --git a/src/pool/pool_proxy.c b/src/pool/pool_proxy.c index 4a4e89247..6e256c491 100644 --- a/src/pool/pool_proxy.c +++ b/src/pool/pool_proxy.c @@ -100,11 +100,22 @@ static umf_result_t proxy_free(void *pool, void *ptr) { struct proxy_memory_pool *hPool = (struct proxy_memory_pool *)pool; if (ptr) { - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t umf_result = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); - if (umf_result == UMF_RESULT_SUCCESS) { - size = allocInfo.baseSize; + umf_memory_properties_handle_t props = NULL; + umf_result_t umf_result = umfGetMemoryPropertiesHandle(ptr, &props); + + if (umf_result != UMF_RESULT_SUCCESS) { + TLS_last_allocation_error = umf_result; + LOG_ERR("failed to get allocation info from the memory tracker"); + return umf_result; + } + + if (props == NULL) { + TLS_last_allocation_error = UMF_RESULT_ERROR_UNKNOWN; + LOG_ERR("failed to get allocation info from the memory tracker"); + return UMF_RESULT_ERROR_UNKNOWN; } + + size = props->base_size; } return umfMemoryProviderFree(hPool->hProvider, ptr, size); diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index 1c15b49e4..b6c15d2d1 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -12,6 +12,7 @@ #include #include +#include "memory_provider_internal.h" #include "provider_ctl_stats_type.h" #include "provider_cuda_internal.h" #include "utils_load_library.h" @@ -746,6 +747,61 @@ static umf_result_t cu_ctl(void *provider, umf_ctl_query_source_t operationType, query_type, arg, size, args); } +static umf_result_t cu_memory_provider_get_allocation_properties( + void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, void *value) { + + // unused + (void)ptr; + + cu_memory_provider_t *cuda_provider = (cu_memory_provider_t *)provider; + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + *(umf_usm_memory_type_t *)value = cuda_provider->memory_type; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_CONTEXT: + *(CUcontext *)value = cuda_provider->context; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_DEVICE: + *(CUdevice *)value = cuda_provider->device; + return UMF_RESULT_SUCCESS; + + default: + break; + }; + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; +} + +static umf_result_t cu_memory_provider_get_allocation_properties_size( + void *provider, umf_memory_property_id_t memory_property_id, size_t *size) { + + // unused + (void)provider; + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + *size = sizeof(umf_usm_memory_type_t); + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_CONTEXT: + *size = sizeof(CUcontext); + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_DEVICE: + *size = sizeof(CUdevice); + return UMF_RESULT_SUCCESS; + + default: + break; + }; + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; +} + static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = cu_memory_provider_initialize, @@ -769,6 +825,10 @@ static umf_memory_provider_ops_t UMF_CUDA_MEMORY_PROVIDER_OPS = { .ext_open_ipc_handle = cu_memory_provider_open_ipc_handle, .ext_close_ipc_handle = cu_memory_provider_close_ipc_handle, .ext_ctl = cu_ctl, + .ext_get_allocation_properties = + cu_memory_provider_get_allocation_properties, + .ext_get_allocation_properties_size = + cu_memory_provider_get_allocation_properties_size, }; const umf_memory_provider_ops_t *umfCUDAMemoryProviderOps(void) { diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index 8703e8fdc..d5ab3e8e4 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -14,6 +14,7 @@ #include #include +#include "memory_provider_internal.h" #include "provider_ctl_stats_type.h" #include "provider_level_zero_internal.h" #include "utils_load_library.h" @@ -32,6 +33,7 @@ void fini_ze_global_state(void) { #include "base_alloc_global.h" #include "libumf.h" +#include "provider_level_zero_internal.h" #include "utils_assert.h" #include "utils_common.h" #include "utils_concurrency.h" @@ -871,6 +873,63 @@ static umf_result_t ze_ctl(void *hProvider, query_type, arg, size, args); } +static umf_result_t ze_memory_provider_get_allocation_properties( + void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, void *value) { + + // unused + (void)ptr; + + struct ze_memory_provider_t *ze_provider = + (struct ze_memory_provider_t *)provider; + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + *(umf_usm_memory_type_t *)value = + ze2umf_memory_type(ze_provider->memory_type); + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_CONTEXT: + *(ze_context_handle_t *)value = ze_provider->context; + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_DEVICE: + *(ze_device_handle_t *)value = ze_provider->device; + return UMF_RESULT_SUCCESS; + + default: + break; + } + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; +} + +static umf_result_t ze_memory_provider_get_allocation_properties_size( + void *provider, umf_memory_property_id_t memory_property_id, size_t *size) { + + // unused + (void)provider; + + switch (memory_property_id) { + case UMF_MEMORY_PROPERTY_POINTER_TYPE: + *size = sizeof(umf_usm_memory_type_t); + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_CONTEXT: + *size = sizeof(ze_context_handle_t); + return UMF_RESULT_SUCCESS; + + case UMF_MEMORY_PROPERTY_DEVICE: + *size = sizeof(ze_device_handle_t); + return UMF_RESULT_SUCCESS; + + default: + break; + } + + return UMF_RESULT_ERROR_INVALID_ARGUMENT; +} + static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = ze_memory_provider_initialize, @@ -891,6 +950,10 @@ static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .ext_open_ipc_handle = ze_memory_provider_open_ipc_handle, .ext_close_ipc_handle = ze_memory_provider_close_ipc_handle, .ext_ctl = ze_ctl, + .ext_get_allocation_properties = + ze_memory_provider_get_allocation_properties, + .ext_get_allocation_properties_size = + ze_memory_provider_get_allocation_properties_size, }; const umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void) { diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index abea227a3..7e003484f 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -8,11 +8,11 @@ #include #include #include - #include #include #include #include + #include #include #include @@ -1351,7 +1351,8 @@ static umf_result_t os_open_ipc_handle(void *provider, void *providerIpcData, os_ipc_data->visibility, fd, os_ipc_data->fd_offset); if (*ptr == NULL) { os_store_last_native_error(UMF_OS_RESULT_ERROR_ALLOC_FAILED, errno); - LOG_PERR("memory mapping failed"); + LOG_PERR("memory mapping failed: %zu bytes at fd=%d, offset=%zu", + os_ipc_data->size, fd, os_ipc_data->fd_offset); ret = UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; } diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index 386eef0ba..cf76d2be7 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -15,6 +15,7 @@ #include #include +#include #include #include "base_alloc_global.h" @@ -22,6 +23,8 @@ #include "ipc_cache.h" #include "ipc_internal.h" #include "memory_pool_internal.h" +#include "memory_properties_internal.h" +#include "memory_provider_internal.h" #include "provider_tracking.h" #include "utils_common.h" #include "utils_concurrency.h" @@ -32,6 +35,8 @@ uint64_t IPC_HANDLE_ID = 0; +uint64_t unique_alloc_id = 0; // requires atomic access + struct umf_memory_tracker_t { umf_ba_pool_t *alloc_info_allocator; // Multilevel maps are needed to support the case @@ -43,19 +48,8 @@ struct umf_memory_tracker_t { critnib *ipc_segments_map; }; -typedef struct tracker_alloc_info_t { - umf_memory_pool_handle_t pool; - size_t size; - // number of overlapping memory regions - // in the next level of map - // falling within the current range - size_t n_children; -#if !defined(NDEBUG) && defined(UMF_DEVELOPER_MODE) - uint64_t is_freed; -#endif -} tracker_alloc_info_t; - typedef struct tracker_ipc_info_t { + umf_memory_properties_t props; size_t size; umf_memory_provider_handle_t provider; ipc_opened_cache_value_t *ipc_cache_value; @@ -113,7 +107,8 @@ static tracker_alloc_info_t *get_most_nested_alloc_segment( continue; } - utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->props.base_size, + &rsize); utils_atomic_load_acquire_size_t(&rvalue->n_children, &n_children); if (found && ((uintptr_t)ptr < rkey + rsize) && n_children) { if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { @@ -192,8 +187,26 @@ umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - value->pool = pool; - value->size = size; + // get provider + umf_memory_provider_handle_t provider = NULL; + umfPoolGetMemoryProvider(pool, &provider); + + // get memory type + umf_usm_memory_type_t memory_type = UMF_MEMORY_TYPE_UNKNOWN; + if (provider && provider->ops.ext_get_allocation_properties) { + provider->ops.ext_get_allocation_properties( + provider->provider_priv, ptr, UMF_MEMORY_PROPERTY_POINTER_TYPE, + &memory_type); + } + + memset(&value->props, 0, sizeof(umf_memory_properties_t)); + value->props.id = utils_atomic_increment_u64(&unique_alloc_id); + value->props.base = (void *)ptr; + value->props.base_size = size; + value->props.pool = pool; + value->props.provider = provider; + value->props.memory_type = memory_type; + value->n_children = 0; #if !defined(NDEBUG) && defined(UMF_DEVELOPER_MODE) value->is_freed = 0; @@ -214,8 +227,8 @@ umfMemoryTrackerAddAtLevel(umf_memory_tracker_handle_t hTracker, int level, "child #%zu added to memory region: tracker=%p, level=%i, " "pool=%p, ptr=%p, size=%zu", n_children, (void *)hTracker, level - 1, - (void *)parent_value->pool, (void *)parent_key, - parent_value->size); + (void *)parent_value->props.pool, (void *)parent_key, + parent_value->props.base_size); assert(ref_parent_value); critnib_release(hTracker->alloc_segments_map[level - 1], ref_parent_value); @@ -286,7 +299,8 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, assert(is_freed != 0xDEADBEEF); #endif - utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->props.base_size, + &rsize); if ((uintptr_t)ptr < rkey + rsize) { if (level == MAX_LEVELS_OF_ALLOC_SEGMENT_MAP - 1) { @@ -300,8 +314,8 @@ static umf_result_t umfMemoryTrackerAdd(umf_memory_tracker_handle_t hTracker, "cannot insert to the tracker value (pool=%p, ptr=%p, " "size=%zu) " "that exceeds the parent value (pool=%p, ptr=%p, size=%zu)", - (void *)pool, ptr, size, (void *)rvalue->pool, (void *)rkey, - (size_t)rsize); + (void *)pool, ptr, size, (void *)rvalue->props.pool, + (void *)rkey, (size_t)rsize); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } parent_key = rkey; @@ -363,7 +377,8 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, LOG_DEBUG("memory region removed: tracker=%p, level=%i, pool=%p, ptr=%p, " "size=%zu", - (void *)hTracker, level, (void *)value->pool, ptr, value->size); + (void *)hTracker, level, (void *)value->props.pool, ptr, + value->props.base_size); // release the reference to the value got from critnib_remove() assert(ref_value); @@ -375,8 +390,9 @@ static umf_result_t umfMemoryTrackerRemove(umf_memory_tracker_handle_t hTracker, LOG_DEBUG( "child #%zu removed from memory region: tracker=%p, level=%i, " "pool=%p, ptr=%p, size=%zu", - n_children, (void *)hTracker, level - 1, (void *)parent_value->pool, - (void *)parent_key, parent_value->size); + n_children, (void *)hTracker, level - 1, + (void *)parent_value->props.pool, (void *)parent_key, + parent_value->props.base_size); assert(ref_parent_value); assert(level >= 1); @@ -405,10 +421,26 @@ umfMemoryTrackerAddIpcSegment(umf_memory_tracker_handle_t hTracker, return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } + // get memory type + umf_usm_memory_type_t memory_type = UMF_MEMORY_TYPE_UNKNOWN; + if (provider && provider->ops.ext_get_allocation_properties) { + provider->ops.ext_get_allocation_properties( + provider->provider_priv, ptr, UMF_MEMORY_PROPERTY_POINTER_TYPE, + &memory_type); + } + value->size = size; value->provider = provider; value->ipc_cache_value = cache_entry; + memset(&value->props, 0, sizeof(umf_memory_properties_t)); + value->props.id = utils_atomic_increment_u64(&unique_alloc_id); + value->props.base = (void *)ptr; + value->props.base_size = size; + value->props.pool = NULL; // unknown + value->props.provider = provider; + value->props.memory_type = memory_type; + int ret = critnib_insert(hTracker->ipc_segments_map, (uintptr_t)ptr, value, 0); if (ret == 0) { @@ -458,37 +490,26 @@ umfMemoryTrackerRemoveIpcSegment(umf_memory_tracker_handle_t hTracker, return UMF_RESULT_SUCCESS; } -umf_memory_pool_handle_t umfMemoryTrackerGetPool(const void *ptr) { - umf_alloc_info_t allocInfo = {NULL, 0, NULL}; - umf_result_t ret = umfMemoryTrackerGetAllocInfo(ptr, &allocInfo); - if (ret != UMF_RESULT_SUCCESS) { - return NULL; - } - - return allocInfo.pool; -} - umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, - umf_alloc_info_t *pAllocInfo) { - assert(pAllocInfo); + tracker_alloc_info_t **info) { + assert(info); - if (ptr == NULL) { + if (UNLIKELY(ptr == NULL)) { return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if (TRACKER == NULL) { + if (UNLIKELY(TRACKER == NULL)) { LOG_ERR("tracker does not exist"); return UMF_RESULT_ERROR_NOT_SUPPORTED; } - if (TRACKER->alloc_segments_map[0] == NULL) { + if (UNLIKELY(TRACKER->alloc_segments_map[0] == NULL)) { LOG_ERR("tracker's alloc_segments_map does not exist"); return UMF_RESULT_ERROR_NOT_SUPPORTED; } tracker_alloc_info_t *top_most_value = NULL; tracker_alloc_info_t *rvalue = NULL; - uintptr_t top_most_key = 0; uintptr_t rkey = 0; uint64_t rsize = 0; size_t n_children = 0; @@ -514,7 +535,6 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, critnib_release(TRACKER->alloc_segments_map[level], ref_value); } top_most_value = NULL; - top_most_key = 0; rkey = 0; rsize = 0; level = 0; @@ -525,10 +545,10 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, continue; } - utils_atomic_load_acquire_u64((uint64_t *)&rvalue->size, &rsize); + utils_atomic_load_acquire_u64((uint64_t *)&rvalue->props.base_size, + &rsize); utils_atomic_load_acquire_size_t(&rvalue->n_children, &n_children); if (found && (uintptr_t)ptr < rkey + rsize) { - top_most_key = rkey; top_most_value = rvalue; if (ref_top_most_value) { assert(level >= 1); @@ -555,9 +575,7 @@ umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - pAllocInfo->base = (void *)top_most_key; - pAllocInfo->baseSize = top_most_value->size; - pAllocInfo->pool = top_most_value->pool; + *info = top_most_value; assert(ref_top_most_value); critnib_release(TRACKER->alloc_segments_map[ref_level], ref_top_most_value); @@ -603,6 +621,8 @@ umf_result_t umfMemoryTrackerGetIpcInfo(const void *ptr, pIpcInfo->baseSize = rvalue->size; pIpcInfo->provider = rvalue->provider; + pIpcInfo->props = &rvalue->props; + if (ref_value) { critnib_release(TRACKER->ipc_segments_map, ref_value); } @@ -692,9 +712,9 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; } - if (value->size != totalSize) { + if (value->props.base_size != totalSize) { LOG_ERR("tracked size=%zu does not match requested size to split: %zu", - value->size, totalSize); + value->props.base_size, totalSize); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err; } @@ -732,7 +752,8 @@ static umf_result_t trackingAllocationSplit(void *hProvider, void *ptr, } // update the size of the first part - utils_atomic_store_release_u64((uint64_t *)&value->size, firstSize); + utils_atomic_store_release_u64((uint64_t *)&value->props.base_size, + firstSize); critnib_release(provider->hTracker->alloc_segments_map[level], ref_value); utils_mutex_unlock(&provider->hTracker->splitMergeMutex); @@ -805,12 +826,12 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_fatal; } - if (lowValue->pool != highValue->pool) { + if (lowValue->props.pool != highValue->props.pool) { LOG_FATAL("pool mismatch"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_fatal; } - if (lowValue->size + highValue->size != totalSize) { + if (lowValue->props.base_size + highValue->props.base_size != totalSize) { LOG_FATAL("lowValue->size + highValue->size != totalSize"); ret = UMF_RESULT_ERROR_INVALID_ARGUMENT; goto err_fatal; @@ -824,7 +845,8 @@ static umf_result_t trackingAllocationMerge(void *hProvider, void *lowPtr, } // we only need to update the size of the first part - utils_atomic_store_release_u64((uint64_t *)&lowValue->size, totalSize); + utils_atomic_store_release_u64((uint64_t *)&lowValue->props.base_size, + totalSize); size_t low_children = lowValue->n_children; size_t high_children = highValue->n_children; @@ -950,12 +972,13 @@ static void check_if_tracker_is_empty(umf_memory_tracker_handle_t hTracker, while (1 == critnib_find(hTracker->alloc_segments_map[i], last_key, FIND_G, &rkey, (void **)&rvalue, &ref_value)) { - if (rvalue && ((rvalue->pool == pool) || pool == NULL)) { + if (rvalue && ((rvalue->props.pool == pool) || pool == NULL)) { n_items++; LOG_DEBUG( "found abandoned allocation in the tracking provider: " "pool=%p, ptr=%p, size=%zu", - (void *)rvalue->pool, (void *)rkey, (size_t)rvalue->size); + (void *)rvalue->props.pool, (void *)rkey, + (size_t)rvalue->props.base_size); } if (ref_value) { @@ -1295,6 +1318,24 @@ static umf_result_t trackingCloseIpcHandle(void *provider, void *ptr, return umf_result; } +static umf_result_t +trackingGetAllocationProperties(void *provider, const void *ptr, + umf_memory_property_id_t memory_property_id, + void *value) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + return umfMemoryProviderGetAllocationProperties(p->hUpstream, ptr, + memory_property_id, value); +} + +static umf_result_t trackingGetAllocationPropertiesSize( + void *provider, umf_memory_property_id_t memory_property_id, size_t *size) { + umf_tracking_memory_provider_t *p = + (umf_tracking_memory_provider_t *)provider; + return umfMemoryProviderGetAllocationPropertiesSize( + p->hUpstream, memory_property_id, size); +} + umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = trackingInitialize, @@ -1313,7 +1354,11 @@ umf_memory_provider_ops_t UMF_TRACKING_MEMORY_PROVIDER_OPS = { .ext_get_ipc_handle = trackingGetIpcHandle, .ext_put_ipc_handle = trackingPutIpcHandle, .ext_open_ipc_handle = trackingOpenIpcHandle, - .ext_close_ipc_handle = trackingCloseIpcHandle}; + .ext_close_ipc_handle = trackingCloseIpcHandle, + .ext_ctl = NULL, + .ext_get_allocation_properties = trackingGetAllocationProperties, + .ext_get_allocation_properties_size = trackingGetAllocationPropertiesSize, +}; static void free_ipc_cache_value(void *unused, void *ipc_cache_value) { (void)unused; diff --git a/src/provider/provider_tracking.h b/src/provider/provider_tracking.h index d7ee06c1b..cdbee3973 100644 --- a/src/provider/provider_tracking.h +++ b/src/provider/provider_tracking.h @@ -20,6 +20,7 @@ #include "base_alloc.h" #include "critnib.h" +#include "memory_properties_internal.h" #include "utils_concurrency.h" #ifdef __cplusplus @@ -34,18 +35,23 @@ extern umf_memory_tracker_handle_t TRACKER; umf_result_t umfMemoryTrackerCreate(umf_memory_tracker_handle_t *handle); void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle); -umf_memory_pool_handle_t umfMemoryTrackerGetPool(const void *ptr); +typedef struct tracker_alloc_info_t { + umf_memory_properties_t props; -typedef struct umf_alloc_info_t { - void *base; - size_t baseSize; - umf_memory_pool_handle_t pool; -} umf_alloc_info_t; + // number of overlapping memory regions in the next level of map falling + // within the current range + size_t n_children; +#if !defined(NDEBUG) && defined(UMF_DEVELOPER_MODE) + uint64_t is_freed; +#endif +} tracker_alloc_info_t; umf_result_t umfMemoryTrackerGetAllocInfo(const void *ptr, - umf_alloc_info_t *pAllocInfo); + tracker_alloc_info_t **info); typedef struct umf_ipc_info_t { + umf_memory_properties_handle_t props; + void *base; size_t baseSize; umf_memory_provider_handle_t provider; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index da29448ee..49ddd18ee 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -380,6 +380,10 @@ if(LINUX) # OS-specific functions are implemented NAME provider_tracking_fixture_tests SRCS provider_tracking_fixture_tests.cpp malloc_compliance_tests.cpp LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) + add_umf_test( + NAME provider_properties + SRCS properties/provider_properties.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) # This test requires Linux-only file memory provider if(UMF_POOL_JEMALLOC_ENABLED) @@ -444,6 +448,12 @@ if(UMF_BUILD_GPU_TESTS AND UMF_LEVEL_ZERO_ENABLED) LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) target_compile_definitions(test_provider_level_zero_dlopen_local PUBLIC USE_DLOPEN=1 OPEN_ZE_LIBRARY_GLOBAL=0) + + add_umf_test( + NAME provider_properties_level_zero + SRCS properties/provider_properties_level_zero.cpp + ${UMF_UTILS_DIR}/utils_level_zero.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST} ze_loader) endif() if(NOT UMF_BUILD_LEVEL_ZERO_PROVIDER) @@ -476,6 +486,12 @@ if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_CUDA_PROVIDER) LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) target_compile_definitions(test_provider_cuda_dlopen_local PUBLIC USE_DLOPEN=1 OPEN_CU_LIBRARY_GLOBAL=0) + + add_umf_test( + NAME provider_properties_cuda + SRCS properties/provider_properties_cuda.cpp + providers/cuda_helpers.cpp + LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST} cuda) else() message( STATUS diff --git a/test/common/provider.hpp b/test/common/provider.hpp index 00621b311..c9206b5f6 100644 --- a/test/common/provider.hpp +++ b/test/common/provider.hpp @@ -130,6 +130,29 @@ typedef struct provider_base_t { [[maybe_unused]] size_t size) noexcept { return UMF_RESULT_ERROR_UNKNOWN; } + + umf_result_t ext_ctl([[maybe_unused]] umf_ctl_query_source_t source, + [[maybe_unused]] const char *name, + [[maybe_unused]] void *arg, + [[maybe_unused]] size_t size, + [[maybe_unused]] umf_ctl_query_type_t queryType, + [[maybe_unused]] va_list args) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + + umf_result_t ext_get_allocation_properties( + [[maybe_unused]] const void *ptr, + [[maybe_unused]] umf_memory_property_id_t memory_property_id, + [[maybe_unused]] void *value) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + + umf_result_t ext_get_allocation_properties_size( + [[maybe_unused]] umf_memory_property_id_t memory_property_id, + [[maybe_unused]] size_t *size) noexcept { + return UMF_RESULT_ERROR_UNKNOWN; + } + virtual ~provider_base_t() = default; } provider_base_t; diff --git a/test/common/provider_null.c b/test/common/provider_null.c index 19e2abd23..2ce8c78dd 100644 --- a/test/common/provider_null.c +++ b/test/common/provider_null.c @@ -144,6 +144,24 @@ static umf_result_t nullCloseIpcHandle(void *provider, void *ptr, size_t size) { return UMF_RESULT_SUCCESS; } +static umf_result_t +nullGetAllocationProperties(void *provider, const void *ptr, + umf_memory_property_id_t propertyId, void *value) { + (void)provider; + (void)ptr; + (void)propertyId; + (void)value; + return UMF_RESULT_SUCCESS; +} + +static umf_result_t nullGetAllocationPropertiesSize( + void *provider, umf_memory_property_id_t propertyId, size_t *size) { + (void)provider; + (void)propertyId; + (void)size; + return UMF_RESULT_SUCCESS; +} + umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = nullInitialize, @@ -163,4 +181,6 @@ umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS = { .ext_put_ipc_handle = nullPutIpcHandle, .ext_open_ipc_handle = nullOpenIpcHandle, .ext_close_ipc_handle = nullCloseIpcHandle, + .ext_get_allocation_properties = nullGetAllocationProperties, + .ext_get_allocation_properties_size = nullGetAllocationPropertiesSize, }; diff --git a/test/common/provider_trace.c b/test/common/provider_trace.c index adb808336..403870511 100644 --- a/test/common/provider_trace.c +++ b/test/common/provider_trace.c @@ -5,10 +5,12 @@ #include #include -#include "provider_trace.h" #include #include +#include "memory_properties_internal.h" +#include "provider_trace.h" + static umf_result_t traceInitialize(const void *params, void **pool) { umf_provider_trace_params_t *trace_pool = (umf_provider_trace_params_t *)malloc( @@ -214,4 +216,6 @@ umf_memory_provider_ops_t UMF_TRACE_PROVIDER_OPS = { .ext_put_ipc_handle = tracePutIpcHandle, .ext_open_ipc_handle = traceOpenIpcHandle, .ext_close_ipc_handle = traceCloseIpcHandle, + .ext_ctl = NULL, + .ext_get_allocation_properties = NULL, }; diff --git a/test/memoryProviderAPI.cpp b/test/memoryProviderAPI.cpp index 52a8955d2..5cbc8c2ca 100644 --- a/test/memoryProviderAPI.cpp +++ b/test/memoryProviderAPI.cpp @@ -3,15 +3,16 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // This file contains tests for UMF provider API -#include "provider.hpp" -#include "provider_null.h" -#include "test_helpers.h" - #include #include #include #include +#include "memory_properties_internal.h" +#include "provider.hpp" +#include "provider_null.h" +#include "test_helpers.h" + using umf_test::test; TEST_F(test, memoryProviderTrace) { diff --git a/test/properties/provider_properties.cpp b/test/properties/provider_properties.cpp new file mode 100644 index 000000000..73542b528 --- /dev/null +++ b/test/properties/provider_properties.cpp @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include + +#include "memory_properties_internal.h" +#include "provider.hpp" +#include "provider_properties.hpp" + +void createFixedProvider(umf_memory_provider_handle_t *out_provider, + void *out_data) { + constexpr size_t buffer_size = 1024 * 1024; + + void *memory_buffer = malloc(buffer_size); + ASSERT_NE(memory_buffer, nullptr); + + umf_fixed_memory_provider_params_handle_t params = nullptr; + umf_result_t res = + umfFixedMemoryProviderParamsCreate(memory_buffer, buffer_size, ¶ms); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(params, nullptr); + + res = umfMemoryProviderCreate(umfFixedMemoryProviderOps(), params, + out_provider); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(out_provider, nullptr); + + umfFixedMemoryProviderParamsDestroy(params); + + *(uintptr_t *)out_data = (uintptr_t)memory_buffer; +} + +void destroyFixedProvider(umf_memory_provider_handle_t provider, void *data) { + umfMemoryProviderDestroy(provider); + free(data); +} + +void createOsMemoryProvider(umf_memory_provider_handle_t *out_provider, + void *out_data) { + + umf_os_memory_provider_params_handle_t os_memory_provider_params = nullptr; + umf_result_t res = + umfOsMemoryProviderParamsCreate(&os_memory_provider_params); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(os_memory_provider_params, nullptr); + + umf_memory_provider_handle_t os_memory_provider = nullptr; + res = + umfMemoryProviderCreate(umfOsMemoryProviderOps(), + os_memory_provider_params, &os_memory_provider); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(os_memory_provider, nullptr); + + res = umfOsMemoryProviderParamsDestroy(os_memory_provider_params); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + + *out_provider = os_memory_provider; + *(uintptr_t *)out_data = (uintptr_t)NULL; +} + +void destroyOsMemoryProvider(umf_memory_provider_handle_t provider, + void *data) { + (void)data; // unused + + umfMemoryProviderDestroy(provider); +} + +INSTANTIATE_TEST_SUITE_P( + providerPropsTest, ProviderPropsTest, + ::testing::Values(testParams{createFixedProvider, destroyFixedProvider, + "fixedProvider"}, + testParams{createOsMemoryProvider, + destroyOsMemoryProvider, "osMemoryProvider"}), + nameGen); + +TEST_F(test, CustomPropsTest) { + const uint64_t custom_property_id = UMF_MEMORY_PROPERTY_MAX_RESERVED + 1; + + struct memory_provider : public umf_test::provider_base_t { + umf_result_t alloc(size_t size, size_t alignment, void **ptr) noexcept { + *ptr = umf_ba_global_aligned_alloc(size, alignment); + return UMF_RESULT_SUCCESS; + } + + umf_result_t free(void *ptr, [[maybe_unused]] size_t size) noexcept { + umf_ba_global_free(ptr); + return UMF_RESULT_SUCCESS; + } + + umf_result_t + get_min_page_size([[maybe_unused]] const void *ptr, + [[maybe_unused]] size_t *pageSize) noexcept { + *pageSize = 1024; + return UMF_RESULT_SUCCESS; + } + + umf_result_t ext_get_allocation_properties( + const void *ptr, umf_memory_property_id_t memory_property_id, + void *value) { + + (void)ptr; // unused + + if (memory_property_id == custom_property_id) { + *(uint64_t *)value = 42; // Custom value for the property + return UMF_RESULT_SUCCESS; + } + + return umf_test::provider_base_t::ext_get_allocation_properties( + ptr, memory_property_id, value); + } + + umf_result_t ext_get_allocation_properties_size( + umf_memory_property_id_t memory_property_id, size_t *size) { + if (memory_property_id == custom_property_id) { + *size = sizeof(uint64_t); + return UMF_RESULT_SUCCESS; + } + return umf_test::provider_base_t:: + ext_get_allocation_properties_size(memory_property_id, size); + } + }; + + umf_memory_provider_ops_t provider_ops = + umf_test::providerMakeCOps(); + + umf_memory_provider_handle_t provider = nullptr; + umf_result_t res = + umfMemoryProviderCreate(&provider_ops, nullptr, &provider); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umf_memory_pool_handle_t pool = nullptr; + res = umfPoolCreate(umfProxyPoolOps(), provider, nullptr, + UMF_POOL_CREATE_FLAG_NONE, &pool); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(pool, nullptr); + + void *ptr = umfPoolMalloc(pool, 1024); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t properties = nullptr; + res = umfGetMemoryPropertiesHandle(ptr, &properties); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(properties, nullptr); + + // get value of the custom property from the properties handle + uint64_t value2 = 0; + res = umfGetMemoryProperty(properties, + (umf_memory_property_id_t)custom_property_id, + &value2, sizeof(value2)); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_EQ(value2, 42); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + + res = umfPoolDestroy(pool); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + + res = umfMemoryProviderDestroy(provider); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} diff --git a/test/properties/provider_properties.hpp b/test/properties/provider_properties.hpp new file mode 100644 index 000000000..b501f6ef0 --- /dev/null +++ b/test/properties/provider_properties.hpp @@ -0,0 +1,239 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include +#include +#include +#include + +#include "base.hpp" +#include "test_helpers.h" + +using umf_test::test; + +using testParams = + std::tuple, + std::function, + const char *>; + +std::string nameGen(const testing::TestParamInfo param) { + return std::get<2>(param.param); +} + +struct ProviderPropsTest : umf_test::test, + ::testing::WithParamInterface { + void SetUp() override { + test::SetUp(); + + auto [create_fun, destroy_fun, name] = this->GetParam(); + provider_create = create_fun; + provider_destroy = destroy_fun; + (void)name; // unused + + provider_create(&provider, &data); + ASSERT_NE(provider, nullptr); + + umf_result_t umf_result = + umfPoolCreate(umfProxyPoolOps(), provider, nullptr, 0, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } + + void TearDown() override { + umfPoolDestroy(pool); + provider_destroy(provider, data); + test::TearDown(); + } + + umf_memory_provider_handle_t provider; + umf_memory_pool_handle_t pool; + + std::function provider_create; + std::function provider_destroy; + void *data; +}; + +TEST_P(ProviderPropsTest, genericProps) { + umf_result_t umf_result; + const size_t alloc_size = 8; + + void *ptr = umfPoolMalloc(pool, alloc_size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + umf_memory_provider_handle_t param_provider = nullptr; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_PROVIDER_HANDLE, + ¶m_provider, sizeof(param_provider)); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(param_provider, provider); + + umf_memory_pool_handle_t param_pool = nullptr; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_POOL_HANDLE, + ¶m_pool, sizeof(param_pool)); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(param_pool, pool); + + void *base_address = nullptr; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + &base_address, sizeof(base_address)); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(base_address, ptr); + + size_t size = 0; + umf_result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_BASE_SIZE, &size, sizeof(size)); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(size, alloc_size); + + uint64_t buffer_id = 0; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + &buffer_id, sizeof(buffer_id)); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_GE(buffer_id, 0); + + umf_result = umfPoolFree(pool, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, baseAddressFromMiddle) { + umf_result_t umf_result; + const size_t alloc_size = 8; + + void *ptr = umfPoolMalloc(pool, alloc_size); + ASSERT_NE(ptr, nullptr); + + void *ptr_mid = (void *)((uintptr_t)ptr + (alloc_size / 2)); + umf_memory_properties_handle_t props_handle = nullptr; + umf_result = umfGetMemoryPropertiesHandle(ptr_mid, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + uintptr_t param_base_address = 0; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + ¶m_base_address, sizeof(param_base_address)); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_EQ(param_base_address, (uintptr_t)ptr); + + umf_result = umfPoolFree(pool, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, uniqueBufferId) { + size_t alloc_size = 8; + size_t num_allocs = 10; + umf_result_t umf_result; + std::set buffer_ids; + + for (size_t i = 0; i < num_allocs; ++i) { + void *ptr = umfPoolMalloc(pool, alloc_size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + uint64_t buffer_id = 0; + umf_result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + &buffer_id, sizeof(buffer_id)); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_GE(buffer_id, 0); + + // Ensure that the buffer ID is unique by inserting it into a set and + // checking if it was already present + ASSERT_TRUE(buffer_ids.find(buffer_id) == buffer_ids.end()); + ASSERT_TRUE(buffer_ids.insert(buffer_id).second); + + umf_result = umfPoolFree(pool, ptr); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + } +} + +// Negative tests + +TEST_P(ProviderPropsTest, invalidPointer) { + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t umf_result = + umfGetMemoryPropertiesHandle(nullptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(props_handle, nullptr); + + uintptr_t invalid_ptr = 0xdeadbeef; + umf_result = + umfGetMemoryPropertiesHandle((void *)invalid_ptr, &props_handle); + ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ASSERT_EQ(props_handle, nullptr); +} + +TEST_P(ProviderPropsTest, invalidPropertyId) { + void *ptr = umfPoolMalloc(pool, 8); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t res = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + void *value = nullptr; + res = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_INVALID, + &value, sizeof(value)); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, invalidPropertyValue) { + void *ptr = umfPoolMalloc(pool, 8); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t res = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + res = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + NULL, sizeof(int)); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, invalidPropertySize) { + void *ptr = umfPoolMalloc(pool, 8); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = nullptr; + umf_result_t res = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + int value = 0; + res = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + &value, size_t(0)); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfPoolFree(pool, ptr); + ASSERT_EQ(res, UMF_RESULT_SUCCESS); +} + +TEST_P(ProviderPropsTest, nullPropertiesHandle) { + int val = 0; + umf_result_t res = umfGetMemoryProperty( + NULL, UMF_MEMORY_PROPERTY_BASE_ADDRESS, &val, sizeof(val)); + ASSERT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} diff --git a/test/properties/provider_properties_cuda.cpp b/test/properties/provider_properties_cuda.cpp new file mode 100644 index 000000000..4ed5b8c02 --- /dev/null +++ b/test/properties/provider_properties_cuda.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include "provider_properties.hpp" +#include "providers/cuda_helpers.h" + +void createCudaMemoryProvider(umf_memory_provider_handle_t *out_provider, + void *out_data) { + CUdevice hDevice = -1; + CUcontext hContext = NULL; + + int ret = init_cuda(); + ASSERT_EQ(ret, 0); + + ret = get_cuda_device(&hDevice); + ASSERT_EQ(ret, 0); + ASSERT_NE(hDevice, -1); + + ret = create_context(hDevice, &hContext); + ASSERT_EQ(ret, 0); + ASSERT_NE(hContext, nullptr); + + umf_cuda_memory_provider_params_handle_t cu_params = NULL; + umf_result_t umf_result = umfCUDAMemoryProviderParamsCreate(&cu_params); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(cu_params, nullptr); + + umf_result = umfCUDAMemoryProviderParamsSetContext(cu_params, hContext); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfCUDAMemoryProviderParamsSetDevice(cu_params, hDevice); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_result = umfCUDAMemoryProviderParamsSetMemoryType( + cu_params, UMF_MEMORY_TYPE_DEVICE); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + umf_memory_provider_handle_t provider = nullptr; + umf_result = umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), cu_params, + &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umfCUDAMemoryProviderParamsDestroy(cu_params); + + *out_provider = provider; + *(uintptr_t *)out_data = (uintptr_t)hContext; +} + +void destroyCudaMemoryProvider(umf_memory_provider_handle_t provider, + void *data) { + destroy_context((CUcontext)data); + umfMemoryProviderDestroy(provider); +} + +INSTANTIATE_TEST_SUITE_P(providerPropsTest, ProviderPropsTest, + ::testing::Values(testParams{createCudaMemoryProvider, + destroyCudaMemoryProvider, + "cudaMemoryProvider"}), + nameGen); diff --git a/test/properties/provider_properties_level_zero.cpp b/test/properties/provider_properties_level_zero.cpp new file mode 100644 index 000000000..aa420cf50 --- /dev/null +++ b/test/properties/provider_properties_level_zero.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +#include "provider_properties.hpp" +#include "utils/utils_level_zero.h" + +void levelZeroMemoryProviderCreate(umf_memory_provider_handle_t *out_provider, + void *out_data) { + + ze_driver_handle_t hDriver = nullptr; + ze_device_handle_t hDevice = nullptr; + ze_context_handle_t hContext = nullptr; + uint32_t driver_idx = 0; + + int ret = utils_ze_init_level_zero(); + ASSERT_EQ(ret, 0); + + ret = utils_ze_find_driver_with_gpu(&driver_idx, &hDriver); + ASSERT_EQ(ret, 0); + + ret = utils_ze_find_gpu_device(hDriver, &hDevice); + ASSERT_EQ(ret, 0); + + ret = utils_ze_create_context(hDriver, &hContext); + ASSERT_EQ(ret, 0); + + umf_level_zero_memory_provider_params_handle_t params = nullptr; + umf_result_t result = umfLevelZeroMemoryProviderParamsCreate(¶ms); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + result = umfLevelZeroMemoryProviderParamsSetContext(params, hContext); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + result = umfLevelZeroMemoryProviderParamsSetDevice(params, hDevice); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + result = umfLevelZeroMemoryProviderParamsSetMemoryType( + params, UMF_MEMORY_TYPE_DEVICE); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + + umf_memory_provider_handle_t provider = nullptr; + umf_result_t umf_result = umfMemoryProviderCreate( + umfLevelZeroMemoryProviderOps(), params, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + result = umfLevelZeroMemoryProviderParamsDestroy(params); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + + *out_provider = provider; + *(uintptr_t *)out_data = (uintptr_t)hContext; +} + +void levelZeroMemoryProviderDestroy(umf_memory_provider_handle_t provider, + void *data) { + umfMemoryProviderDestroy(provider); + utils_ze_destroy_context((ze_context_handle_t)data); +} + +INSTANTIATE_TEST_SUITE_P(providerPropsTest, ProviderPropsTest, + ::testing::Values(testParams{ + levelZeroMemoryProviderCreate, + levelZeroMemoryProviderDestroy, + "levelZeroProvider"}), + nameGen); diff --git a/test/providers/provider_cuda.cpp b/test/providers/provider_cuda.cpp index 585868014..8a3d729eb 100644 --- a/test/providers/provider_cuda.cpp +++ b/test/providers/provider_cuda.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include "cuda_helpers.h" @@ -582,6 +583,87 @@ TEST_P(umfCUDAProviderTest, multiContext) { ASSERT_EQ(ret, 0); } +TEST_P(umfCUDAProviderTest, memProps) { + umf_memory_provider_handle_t provider = nullptr; + umf_result_t umf_result = + umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), params, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umf_memory_pool_handle_t pool = NULL; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider, NULL, 0, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_t size = 1024; + void *ptr = umfPoolMalloc(pool, size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = NULL; + umf_result_t result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + umf_usm_memory_type_t type = UMF_MEMORY_TYPE_UNKNOWN; + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, &type, sizeof(type)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, expected_memory_type); + + void *baseAddress = nullptr; + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + &baseAddress, sizeof(baseAddress)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + size_t baseSize = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_SIZE, + &baseSize, sizeof(baseSize)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(baseSize, size); + + int64_t bufferId = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + &bufferId, sizeof(bufferId)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(bufferId, 0); + + if (expected_memory_type != UMF_MEMORY_TYPE_HOST) { + CUdevice device = -1; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_DEVICE, + &device, sizeof(device)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(device, cudaTestHelper.get_test_device()); + } + + CUcontext context = nullptr; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_CONTEXT, + &context, sizeof(context)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(context, cudaTestHelper.get_test_context()); + + // check the props of pointer from the middle of alloc + void *midPtr = static_cast(ptr) + size / 2; + result = umfGetMemoryPropertiesHandle(midPtr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, &type, sizeof(type)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, expected_memory_type); + + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + &baseAddress, sizeof(baseAddress)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + umfFree(ptr); + + umfPoolDestroy(pool); + umfMemoryProviderDestroy(provider); +} + struct umfCUDAProviderAllocFlagsTest : umf_test::test, ::testing::WithParamInterface< diff --git a/test/providers/provider_level_zero.cpp b/test/providers/provider_level_zero.cpp index 7aa3f048b..d91660399 100644 --- a/test/providers/provider_level_zero.cpp +++ b/test/providers/provider_level_zero.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include "ipcFixtures.hpp" @@ -256,6 +257,7 @@ struct umfLevelZeroProviderTest test::SetUp(); umf_usm_memory_type_t memory_type = this->GetParam(); + umfExpectedMemoryType = memory_type; params = nullptr; memAccessor = nullptr; @@ -308,6 +310,7 @@ struct umfLevelZeroProviderTest std::unique_ptr memAccessor = nullptr; ze_context_handle_t hContext = nullptr; ze_memory_type_t zeMemoryTypeExpected = ZE_MEMORY_TYPE_UNKNOWN; + umf_usm_memory_type_t umfExpectedMemoryType = UMF_MEMORY_TYPE_UNKNOWN; }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(umfLevelZeroProviderTest); @@ -578,6 +581,88 @@ TEST_P(umfLevelZeroProviderTest, setDeviceOrdinalValid) { } } +TEST_P(umfLevelZeroProviderTest, memProps) { + umf_memory_provider_handle_t provider = nullptr; + umf_result_t umf_result = umfMemoryProviderCreate( + umfLevelZeroMemoryProviderOps(), params, &provider); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + ASSERT_NE(provider, nullptr); + + umf_memory_pool_handle_t pool = NULL; + umf_result = umfPoolCreate(umfProxyPoolOps(), provider, NULL, 0, &pool); + ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); + + size_t size = 1024; + void *ptr = umfPoolMalloc(pool, size); + ASSERT_NE(ptr, nullptr); + + umf_memory_properties_handle_t props_handle = NULL; + umf_result_t result = umfGetMemoryPropertiesHandle(ptr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + + umf_usm_memory_type_t type = UMF_MEMORY_TYPE_UNKNOWN; + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, &type, sizeof(type)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, umfExpectedMemoryType); + + // base address and size + void *baseAddress = nullptr; + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + &baseAddress, sizeof(baseAddress)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + size_t baseSize = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_SIZE, + &baseSize, sizeof(baseSize)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(baseSize, size); + + int64_t bufferId = 0; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BUFFER_ID, + &bufferId, sizeof(bufferId)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_GE(bufferId, 0); + + if (umfExpectedMemoryType != UMF_MEMORY_TYPE_HOST) { + ze_device_handle_t device = nullptr; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_DEVICE, + &device, sizeof(device)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(device, l0TestHelper.get_test_device()); + } + + ze_context_handle_t context = nullptr; + result = umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_CONTEXT, + &context, sizeof(context)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(context, l0TestHelper.get_test_context()); + + // check the props of pointer from the middle of alloc + void *midPtr = static_cast(ptr) + size / 2; + result = umfGetMemoryPropertiesHandle(midPtr, &props_handle); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_NE(props_handle, nullptr); + result = umfGetMemoryProperty( + props_handle, UMF_MEMORY_PROPERTY_POINTER_TYPE, &type, sizeof(type)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(type, umfExpectedMemoryType); + + result = + umfGetMemoryProperty(props_handle, UMF_MEMORY_PROPERTY_BASE_ADDRESS, + &baseAddress, sizeof(baseAddress)); + ASSERT_EQ(result, UMF_RESULT_SUCCESS); + ASSERT_EQ(baseAddress, ptr); + + umfFree(ptr); + + umfPoolDestroy(pool); + umfMemoryProviderDestroy(provider); +} + // TODO add tests that mixes Level Zero Memory Provider and Disjoint Pool INSTANTIATE_TEST_SUITE_P( diff --git a/test/utils/cpp_helpers.hpp b/test/utils/cpp_helpers.hpp index 6b8e840d8..69a01a39f 100644 --- a/test/utils/cpp_helpers.hpp +++ b/test/utils/cpp_helpers.hpp @@ -114,6 +114,10 @@ template constexpr umf_memory_provider_ops_t providerOpsBase() { UMF_ASSIGN_OP(ops, T, ext_put_ipc_handle, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, ext_open_ipc_handle, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops, T, ext_close_ipc_handle, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, ext_get_allocation_properties, + UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP(ops, T, ext_get_allocation_properties_size, + UMF_RESULT_ERROR_UNKNOWN); return ops; } } // namespace detail