diff --git a/examples/cuda_shared_memory/cuda_shared_memory.c b/examples/cuda_shared_memory/cuda_shared_memory.c index 22e5e3f10a..50c8f92400 100644 --- a/examples/cuda_shared_memory/cuda_shared_memory.c +++ b/examples/cuda_shared_memory/cuda_shared_memory.c @@ -43,24 +43,51 @@ int main(void) { // Create a context on the device cuCtxCreate(&cuContext, 0, cuDevice); - // Setup parameters for the CUDA memory provider. It will be used for + // Setup parameters for the CUDA Memory Provider. It will be used for // allocating memory from CUDA devices. - cuda_memory_provider_params_t cu_memory_provider_params; - cu_memory_provider_params.cuda_context_handle = cuContext; - cu_memory_provider_params.cuda_device_handle = cuDevice; + umf_cuda_memory_provider_params_handle_t cu_memory_provider_params = NULL; + res = umfCUDAMemoryProviderParamsCreate(&cu_memory_provider_params); + if (res != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Failed to create memory provider params!\n"); + ret = -1; + goto cuda_destroy; + } + + res = umfCUDAMemoryProviderParamsSetContext(cu_memory_provider_params, + cuContext); + if (res != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Failed to set context in memory provider params!\n"); + ret = -1; + goto provider_params_destroy; + } + + res = umfCUDAMemoryProviderParamsSetDevice(cu_memory_provider_params, + cuDevice); + if (res != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Failed to set device in memory provider params!\n"); + ret = -1; + goto provider_params_destroy; + } // Set the memory type to shared to allow the memory to be accessed on both // CPU and GPU. - cu_memory_provider_params.memory_type = UMF_MEMORY_TYPE_SHARED; + res = umfCUDAMemoryProviderParamsSetMemoryType(cu_memory_provider_params, + UMF_MEMORY_TYPE_SHARED); + if (res != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "Failed to set memory type in memory provider params!\n"); + ret = -1; + goto provider_params_destroy; + } // Create CUDA memory provider umf_memory_provider_handle_t cu_memory_provider; - res = umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), - &cu_memory_provider_params, - &cu_memory_provider); + res = + umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), + cu_memory_provider_params, &cu_memory_provider); if (res != UMF_RESULT_SUCCESS) { fprintf(stderr, "Failed to create a memory provider!\n"); ret = -1; - goto cuda_destroy; + goto provider_params_destroy; } printf("CUDA memory provider created at %p\n", (void *)cu_memory_provider); @@ -147,6 +174,9 @@ int main(void) { memory_provider_destroy: umfMemoryProviderDestroy(cu_memory_provider); +provider_params_destroy: + umfCUDAMemoryProviderParamsDestroy(cu_memory_provider_params); + cuda_destroy: ret = cuCtxDestroy(cuContext); return ret; diff --git a/include/umf/providers/provider_cuda.h b/include/umf/providers/provider_cuda.h index 2f6a07d816..5f1d5a6e2a 100644 --- a/include/umf/providers/provider_cuda.h +++ b/include/umf/providers/provider_cuda.h @@ -14,12 +14,44 @@ extern "C" { #endif -/// @brief CUDA Memory Provider settings struct -typedef struct cuda_memory_provider_params_t { - void *cuda_context_handle; ///< Handle to the CUDA context - int cuda_device_handle; ///< Handle to the CUDA device - umf_usm_memory_type_t memory_type; ///< Allocation memory type -} cuda_memory_provider_params_t; +struct umf_cuda_memory_provider_params_t; + +typedef struct umf_cuda_memory_provider_params_t + *umf_cuda_memory_provider_params_handle_t; + +/// @brief Create a struct to store parameters of the CUDA Memory Provider. +/// @param hParams [out] handle to the newly created parameters struct. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfCUDAMemoryProviderParamsCreate( + umf_cuda_memory_provider_params_handle_t *hParams); + +/// @brief Destroy parameters struct. +/// @param hParams handle to the parameters of the CUDA Memory Provider. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfCUDAMemoryProviderParamsDestroy( + umf_cuda_memory_provider_params_handle_t hParams); + +/// @brief Set the CUDA context handle in the parameters struct. +/// @param hParams handle to the parameters of the CUDA Memory Provider. +/// @param hContext handle to the CUDA context. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfCUDAMemoryProviderParamsSetContext( + umf_cuda_memory_provider_params_handle_t hParams, void *hContext); + +/// @brief Set the CUDA device handle in the parameters struct. +/// @param hParams handle to the parameters of the CUDA Memory Provider. +/// @param hDevice handle to the CUDA device. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfCUDAMemoryProviderParamsSetDevice( + umf_cuda_memory_provider_params_handle_t hParams, int hDevice); + +/// @brief Set the memory type in the parameters struct. +/// @param hParams handle to the parameters of the CUDA Memory Provider. +/// @param memoryType memory type. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfCUDAMemoryProviderParamsSetMemoryType( + umf_cuda_memory_provider_params_handle_t hParams, + umf_usm_memory_type_t memoryType); umf_memory_provider_ops_t *umfCUDAMemoryProviderOps(void); diff --git a/src/libumf.def b/src/libumf.def index 56e26050c8..8f29f4579b 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -17,6 +17,11 @@ EXPORTS umfCoarseMemoryProviderGetStats umfCoarseMemoryProviderOps umfCUDAMemoryProviderOps + umfCUDAMemoryProviderParamsCreate + umfCUDAMemoryProviderParamsDestroy + umfCUDAMemoryProviderParamsSetContext + umfCUDAMemoryProviderParamsSetDevice + umfCUDAMemoryProviderParamsSetMemoryType umfDevDaxMemoryProviderOps umfFree umfFileMemoryProviderOps diff --git a/src/libumf.map b/src/libumf.map index 19235705c9..f70d247b52 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -11,6 +11,11 @@ UMF_1.0 { umfCoarseMemoryProviderGetStats; umfCoarseMemoryProviderOps; umfCUDAMemoryProviderOps; + umfCUDAMemoryProviderParamsCreate; + umfCUDAMemoryProviderParamsDestroy; + umfCUDAMemoryProviderParamsSetContext; + umfCUDAMemoryProviderParamsSetDevice; + umfCUDAMemoryProviderParamsSetMemoryType; umfDevDaxMemoryProviderOps; umfFree; umfFileMemoryProviderOps; diff --git a/src/provider/provider_cuda.c b/src/provider/provider_cuda.c index 5a686d8579..3c4e39451b 100644 --- a/src/provider/provider_cuda.c +++ b/src/provider/provider_cuda.c @@ -14,6 +14,40 @@ #if defined(UMF_NO_CUDA_PROVIDER) +umf_result_t umfCUDAMemoryProviderParamsCreate( + umf_cuda_memory_provider_params_handle_t *hParams) { + (void)hParams; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +umf_result_t umfCUDAMemoryProviderParamsDestroy( + umf_cuda_memory_provider_params_handle_t hParams) { + (void)hParams; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +umf_result_t umfCUDAMemoryProviderParamsSetContext( + umf_cuda_memory_provider_params_handle_t hParams, void *hContext) { + (void)hParams; + (void)hContext; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +umf_result_t umfCUDAMemoryProviderParamsSetDevice( + umf_cuda_memory_provider_params_handle_t hParams, int hDevice) { + (void)hParams; + (void)hDevice; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + +umf_result_t umfCUDAMemoryProviderParamsSetMemoryType( + umf_cuda_memory_provider_params_handle_t hParams, + umf_usm_memory_type_t memoryType) { + (void)hParams; + (void)memoryType; + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + umf_memory_provider_ops_t *umfCUDAMemoryProviderOps(void) { // not supported return NULL; @@ -48,6 +82,13 @@ typedef struct cu_memory_provider_t { size_t min_alignment; } cu_memory_provider_t; +// CUDA Memory Provider settings struct +typedef struct umf_cuda_memory_provider_params_t { + void *cuda_context_handle; ///< Handle to the CUDA context + int cuda_device_handle; ///< Handle to the CUDA device + umf_usm_memory_type_t memory_type; ///< Allocation memory type +} umf_cuda_memory_provider_params_t; + typedef struct cu_ops_t { CUresult (*cuMemGetAllocationGranularity)( size_t *granularity, const CUmemAllocationProp *prop, @@ -158,14 +199,81 @@ static void init_cu_global_state(void) { } } +umf_result_t umfCUDAMemoryProviderParamsCreate( + umf_cuda_memory_provider_params_handle_t *hParams) { + if (!hParams) { + LOG_ERR("CUDA Memory Provider params handle is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + umf_cuda_memory_provider_params_handle_t params_data = + umf_ba_global_alloc(sizeof(umf_cuda_memory_provider_params_t)); + if (!params_data) { + LOG_ERR("Cannot allocate memory for CUDA Memory Provider params"); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + params_data->cuda_context_handle = NULL; + params_data->cuda_device_handle = -1; + params_data->memory_type = UMF_MEMORY_TYPE_UNKNOWN; + + *hParams = params_data; + + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfCUDAMemoryProviderParamsDestroy( + umf_cuda_memory_provider_params_handle_t hParams) { + umf_ba_global_free(hParams); + + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfCUDAMemoryProviderParamsSetContext( + umf_cuda_memory_provider_params_handle_t hParams, void *hContext) { + if (!hParams) { + LOG_ERR("CUDA Memory Provider params handle is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + hParams->cuda_context_handle = hContext; + + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfCUDAMemoryProviderParamsSetDevice( + umf_cuda_memory_provider_params_handle_t hParams, int hDevice) { + if (!hParams) { + LOG_ERR("CUDA Memory Provider params handle is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + hParams->cuda_device_handle = hDevice; + + return UMF_RESULT_SUCCESS; +} + +umf_result_t umfCUDAMemoryProviderParamsSetMemoryType( + umf_cuda_memory_provider_params_handle_t hParams, + umf_usm_memory_type_t memoryType) { + if (!hParams) { + LOG_ERR("CUDA Memory Provider params handle is NULL"); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + hParams->memory_type = memoryType; + + return UMF_RESULT_SUCCESS; +} + static umf_result_t cu_memory_provider_initialize(void *params, void **provider) { if (params == NULL) { return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - cuda_memory_provider_params_t *cu_params = - (cuda_memory_provider_params_t *)params; + umf_cuda_memory_provider_params_handle_t cu_params = + (umf_cuda_memory_provider_params_handle_t)params; if (cu_params->memory_type == UMF_MEMORY_TYPE_UNKNOWN || cu_params->memory_type > UMF_MEMORY_TYPE_SHARED) { diff --git a/test/providers/cuda_helpers.cpp b/test/providers/cuda_helpers.cpp index 37e71bd6a8..9c41d9382f 100644 --- a/test/providers/cuda_helpers.cpp +++ b/test/providers/cuda_helpers.cpp @@ -39,9 +39,63 @@ struct libcu_ops { } libcu_ops; #if USE_DLOPEN +// Generic no-op stub function for all callbacks +template CUresult noop_stub(Args &&...) { + return CUDA_SUCCESS; // Always return CUDA_SUCCESS +} + struct DlHandleCloser { void operator()(void *dlHandle) { if (dlHandle) { + libcu_ops.cuInit = [](auto... args) { return noop_stub(args...); }; + libcu_ops.cuCtxCreate = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuCtxDestroy = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuCtxGetCurrent = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuCtxSetCurrent = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuDeviceGet = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuMemAlloc = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuMemFree = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuMemAllocHost = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuMemAllocManaged = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuMemFreeHost = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuMemsetD32 = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuMemcpy = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuPointerGetAttribute = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuPointerGetAttributes = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuStreamSynchronize = [](auto... args) { + return noop_stub(args...); + }; + libcu_ops.cuCtxSynchronize = [](auto... args) { + return noop_stub(args...); + }; utils_close_library(dlHandle); } } @@ -355,38 +409,40 @@ int init_cuda() { return InitResult; } -cuda_memory_provider_params_t -create_cuda_prov_params(umf_usm_memory_type_t memory_type) { - cuda_memory_provider_params_t params = {NULL, 0, UMF_MEMORY_TYPE_UNKNOWN}; - int ret = -1; +int get_cuda_device(CUdevice *device) { + CUdevice cuDevice = -1; - ret = init_cuda(); + int ret = init_cuda(); if (ret != 0) { - // Return empty params. Test will be skipped. - return params; + fprintf(stderr, "init_cuda() failed!\n"); + return ret; } - // Get the first CUDA device - CUdevice cuDevice = -1; CUresult res = libcu_ops.cuDeviceGet(&cuDevice, 0); if (res != CUDA_SUCCESS || cuDevice < 0) { - // Return empty params. Test will be skipped. - return params; + return -1; } - // Create a CUDA context + *device = cuDevice; + return 0; +} + +int create_context(CUdevice device, CUcontext *context) { CUcontext cuContext = nullptr; - res = libcu_ops.cuCtxCreate(&cuContext, 0, cuDevice); - if (res != CUDA_SUCCESS || cuContext == nullptr) { - // Return empty params. Test will be skipped. - return params; + + int ret = init_cuda(); + if (ret != 0) { + fprintf(stderr, "init_cuda() failed!\n"); + return ret; } - params.cuda_context_handle = cuContext; - params.cuda_device_handle = cuDevice; - params.memory_type = memory_type; + CUresult res = libcu_ops.cuCtxCreate(&cuContext, 0, device); + if (res != CUDA_SUCCESS || cuContext == nullptr) { + return -1; + } - return params; + *context = cuContext; + return 0; } int destroy_context(CUcontext context) { diff --git a/test/providers/cuda_helpers.h b/test/providers/cuda_helpers.h index fc349fc143..fc06c1fcff 100644 --- a/test/providers/cuda_helpers.h +++ b/test/providers/cuda_helpers.h @@ -26,6 +26,10 @@ extern "C" { #endif +int get_cuda_device(CUdevice *device); + +int create_context(CUdevice device, CUcontext *context); + int destroy_context(CUcontext context); int cuda_fill(CUcontext context, CUdevice device, void *ptr, size_t size, @@ -40,9 +44,6 @@ CUcontext get_mem_context(void *ptr); CUcontext get_current_context(); -cuda_memory_provider_params_t -create_cuda_prov_params(umf_usm_memory_type_t memory_type); - #ifdef __cplusplus } #endif diff --git a/test/providers/ipc_cuda_prov_common.c b/test/providers/ipc_cuda_prov_common.c index ac00bb01b5..a38e4d061d 100644 --- a/test/providers/ipc_cuda_prov_common.c +++ b/test/providers/ipc_cuda_prov_common.c @@ -13,10 +13,9 @@ #include "ipc_cuda_prov_common.h" void memcopy(void *dst, const void *src, size_t size, void *context) { - cuda_memory_provider_params_t *cu_params = - (cuda_memory_provider_params_t *)context; - int ret = cuda_copy(cu_params->cuda_context_handle, - cu_params->cuda_device_handle, dst, (void *)src, size); + cuda_copy_ctx_t *cu_params = (cuda_copy_ctx_t *)context; + int ret = cuda_copy(cu_params->context, cu_params->device, dst, (void *)src, + size); if (ret != 0) { fprintf(stderr, "cuda_copy failed with error %d\n", ret); } diff --git a/test/providers/ipc_cuda_prov_common.h b/test/providers/ipc_cuda_prov_common.h index cecdc2bb8b..e50546efd1 100644 --- a/test/providers/ipc_cuda_prov_common.h +++ b/test/providers/ipc_cuda_prov_common.h @@ -10,6 +10,11 @@ #include +typedef struct cuda_copy_ctx_t { + CUcontext context; + CUdevice device; +} cuda_copy_ctx_t; + void memcopy(void *dst, const void *src, size_t size, void *context); #endif // UMF_TEST_IPC_CUDA_PROV_COMMON_H diff --git a/test/providers/ipc_cuda_prov_consumer.c b/test/providers/ipc_cuda_prov_consumer.c index 3286cee28f..1aeb5b15cb 100644 --- a/test/providers/ipc_cuda_prov_consumer.c +++ b/test/providers/ipc_cuda_prov_consumer.c @@ -22,23 +22,76 @@ int main(int argc, char *argv[]) { } int port = atoi(argv[1]); + CUdevice hDevice = -1; + CUcontext hContext = NULL; - cuda_memory_provider_params_t cu_params = - create_cuda_prov_params(UMF_MEMORY_TYPE_DEVICE); + int ret = get_cuda_device(&hDevice); + if (ret != 0) { + fprintf(stderr, "get_cuda_device() failed!\n"); + return -1; + } + + ret = create_context(hDevice, &hContext); + if (ret != 0) { + fprintf(stderr, "create_context() failed!\n"); + return -1; + } + + umf_cuda_memory_provider_params_handle_t cu_params = NULL; + umf_result_t umf_result = umfCUDAMemoryProviderParamsCreate(&cu_params); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Failed to create CUDA params!\n"); + ret = -1; + goto destroy_context; + } + + umf_result = umfCUDAMemoryProviderParamsSetContext(cu_params, hContext); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "Failed to set context in CUDA Memory Provider params!\n"); + ret = -1; + goto destroy_provider_params; + } + + umf_result = umfCUDAMemoryProviderParamsSetDevice(cu_params, hDevice); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "Failed to set device in CUDA Memory Provider params!\n"); + ret = -1; + goto destroy_provider_params; + } + + umf_result = umfCUDAMemoryProviderParamsSetMemoryType( + cu_params, UMF_MEMORY_TYPE_DEVICE); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Failed to set memory type in CUDA memory " + "provider params!\n"); + ret = -1; + goto destroy_provider_params; + } umf_disjoint_pool_params_handle_t pool_params = NULL; - umf_result_t umf_result = umfDisjointPoolParamsCreate(&pool_params); + umf_result = umfDisjointPoolParamsCreate(&pool_params); if (umf_result != UMF_RESULT_SUCCESS) { fprintf(stderr, "Failed to create pool params!\n"); - return -1; + ret = -1; + goto destroy_provider_params; } - int ret = run_consumer(port, umfDisjointPoolOps(), pool_params, - umfCUDAMemoryProviderOps(), &cu_params, memcopy, - &cu_params); + cuda_copy_ctx_t copy_ctx = {hContext, hDevice}; + + ret = + run_consumer(port, umfDisjointPoolOps(), pool_params, + umfCUDAMemoryProviderOps(), cu_params, memcopy, ©_ctx); umfDisjointPoolParamsDestroy(pool_params); +destroy_provider_params: + umfCUDAMemoryProviderParamsDestroy(cu_params); + +destroy_context: + destroy_context(hContext); + return ret; } diff --git a/test/providers/ipc_cuda_prov_producer.c b/test/providers/ipc_cuda_prov_producer.c index d11004d6d4..c2cd1d1325 100644 --- a/test/providers/ipc_cuda_prov_producer.c +++ b/test/providers/ipc_cuda_prov_producer.c @@ -22,23 +22,76 @@ int main(int argc, char *argv[]) { } int port = atoi(argv[1]); + CUdevice hDevice = -1; + CUcontext hContext = NULL; - cuda_memory_provider_params_t cu_params = - create_cuda_prov_params(UMF_MEMORY_TYPE_DEVICE); + int ret = get_cuda_device(&hDevice); + if (ret != 0) { + fprintf(stderr, "get_cuda_device() failed!\n"); + return -1; + } + + ret = create_context(hDevice, &hContext); + if (ret != 0) { + fprintf(stderr, "create_context() failed!\n"); + return -1; + } + + umf_cuda_memory_provider_params_handle_t cu_params = NULL; + umf_result_t umf_result = umfCUDAMemoryProviderParamsCreate(&cu_params); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Failed to create CUDA params!\n"); + ret = -1; + goto destroy_context; + } + + umf_result = umfCUDAMemoryProviderParamsSetContext(cu_params, hContext); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "Failed to set context in CUDA Memory Provider params!\n"); + ret = -1; + goto destroy_provider_params; + } + + umf_result = umfCUDAMemoryProviderParamsSetDevice(cu_params, hDevice); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, + "Failed to set device in CUDA Memory Provider params!\n"); + ret = -1; + goto destroy_provider_params; + } + + umf_result = umfCUDAMemoryProviderParamsSetMemoryType( + cu_params, UMF_MEMORY_TYPE_DEVICE); + if (umf_result != UMF_RESULT_SUCCESS) { + fprintf(stderr, "Failed to set memory type in CUDA memory " + "provider params!\n"); + ret = -1; + goto destroy_provider_params; + } umf_disjoint_pool_params_handle_t pool_params = NULL; - umf_result_t umf_result = umfDisjointPoolParamsCreate(&pool_params); + umf_result = umfDisjointPoolParamsCreate(&pool_params); if (umf_result != UMF_RESULT_SUCCESS) { fprintf(stderr, "Failed to create pool params!\n"); - return -1; + ret = -1; + goto destroy_provider_params; } - int ret = run_producer(port, umfDisjointPoolOps(), pool_params, - umfCUDAMemoryProviderOps(), &cu_params, memcopy, - &cu_params); + cuda_copy_ctx_t copy_ctx = {hContext, hDevice}; + + ret = + run_producer(port, umfDisjointPoolOps(), pool_params, + umfCUDAMemoryProviderOps(), cu_params, memcopy, ©_ctx); umfDisjointPoolParamsDestroy(pool_params); +destroy_provider_params: + umfCUDAMemoryProviderParamsDestroy(cu_params); + +destroy_context: + destroy_context(hContext); + return ret; } diff --git a/test/providers/provider_cuda.cpp b/test/providers/provider_cuda.cpp index 58e3beb9e4..4f1d359113 100644 --- a/test/providers/provider_cuda.cpp +++ b/test/providers/provider_cuda.cpp @@ -19,6 +19,82 @@ using umf_test::test; using namespace umf_test; +class CUDATestHelper { + public: + CUDATestHelper(); + + ~CUDATestHelper() { + if (hContext_) { + destroy_context(hContext_); + } + } + + CUcontext get_test_context() const { return hContext_; } + + CUdevice get_test_device() const { return hDevice_; } + + private: + CUcontext hContext_ = nullptr; + CUdevice hDevice_ = -1; +}; + +CUDATestHelper::CUDATestHelper() { + int ret = get_cuda_device(&hDevice_); + if (ret != 0) { + fprintf(stderr, "get_cuda_device() failed!\n"); + return; + } + + ret = create_context(hDevice_, &hContext_); + if (ret != 0) { + fprintf(stderr, "create_context() failed!\n"); + return; + } +} + +using cuda_params_unique_handle_t = + std::unique_ptr; + +cuda_params_unique_handle_t +create_cuda_prov_params(CUcontext context, CUdevice device, + umf_usm_memory_type_t memory_type) { + umf_cuda_memory_provider_params_handle_t params = nullptr; + + umf_result_t res = umfCUDAMemoryProviderParamsCreate(¶ms); + if (res != UMF_RESULT_SUCCESS) { + return cuda_params_unique_handle_t(nullptr, + &umfCUDAMemoryProviderParamsDestroy); + } + + res = umfCUDAMemoryProviderParamsSetContext(params, context); + if (res != UMF_RESULT_SUCCESS) { + umfCUDAMemoryProviderParamsDestroy(params); + return cuda_params_unique_handle_t(nullptr, + &umfCUDAMemoryProviderParamsDestroy); + ; + } + + res = umfCUDAMemoryProviderParamsSetDevice(params, device); + if (res != UMF_RESULT_SUCCESS) { + umfCUDAMemoryProviderParamsDestroy(params); + return cuda_params_unique_handle_t(nullptr, + &umfCUDAMemoryProviderParamsDestroy); + ; + } + + res = umfCUDAMemoryProviderParamsSetMemoryType(params, memory_type); + if (res != UMF_RESULT_SUCCESS) { + umfCUDAMemoryProviderParamsDestroy(params); + return cuda_params_unique_handle_t(nullptr, + &umfCUDAMemoryProviderParamsDestroy); + ; + } + + return cuda_params_unique_handle_t(params, + &umfCUDAMemoryProviderParamsDestroy); +} + class CUDAMemoryAccessor : public MemoryAccessor { public: CUDAMemoryAccessor(CUcontext hContext, CUdevice hDevice) @@ -51,7 +127,8 @@ class CUDAMemoryAccessor : public MemoryAccessor { }; using CUDAProviderTestParams = - std::tuple; + std::tuple; struct umfCUDAProviderTest : umf_test::test, @@ -60,15 +137,20 @@ struct umfCUDAProviderTest void SetUp() override { test::SetUp(); - auto [cuda_params, accessor] = this->GetParam(); + auto [cuda_params, cu_context, memory_type, accessor] = + this->GetParam(); params = cuda_params; memAccessor = accessor; + expected_context = cu_context; + expected_memory_type = memory_type; } void TearDown() override { test::TearDown(); } - cuda_memory_provider_params_t params; + umf_cuda_memory_provider_params_handle_t params; MemoryAccessor *memAccessor = nullptr; + CUcontext expected_context; + umf_usm_memory_type_t expected_memory_type; }; TEST_P(umfCUDAProviderTest, basic) { @@ -79,7 +161,7 @@ TEST_P(umfCUDAProviderTest, basic) { // create CUDA provider umf_memory_provider_handle_t provider = nullptr; umf_result_t umf_result = - umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), ¶ms, &provider); + umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), params, &provider); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(provider, nullptr); @@ -102,14 +184,14 @@ TEST_P(umfCUDAProviderTest, basic) { memAccessor->fill(ptr, size, &pattern, sizeof(pattern)); CUcontext actual_mem_context = get_mem_context(ptr); - ASSERT_EQ(actual_mem_context, (CUcontext)params.cuda_context_handle); + ASSERT_EQ(actual_mem_context, expected_context); CUcontext actual_current_context = get_current_context(); ASSERT_EQ(actual_current_context, expected_current_context); umf_usm_memory_type_t memoryTypeActual = - get_mem_type((CUcontext)params.cuda_context_handle, ptr); - ASSERT_EQ(memoryTypeActual, params.memory_type); + get_mem_type(actual_current_context, ptr); + ASSERT_EQ(memoryTypeActual, expected_memory_type); // check if the pattern was successfully applied uint32_t *hostMemory = (uint32_t *)calloc(1, size); @@ -128,7 +210,7 @@ TEST_P(umfCUDAProviderTest, basic) { TEST_P(umfCUDAProviderTest, getPageSize) { umf_memory_provider_handle_t provider = nullptr; umf_result_t umf_result = - umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), ¶ms, &provider); + umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), params, &provider); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(provider, nullptr); @@ -152,7 +234,7 @@ TEST_P(umfCUDAProviderTest, getPageSize) { TEST_P(umfCUDAProviderTest, getName) { umf_memory_provider_handle_t provider = nullptr; umf_result_t umf_result = - umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), ¶ms, &provider); + umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), params, &provider); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(provider, nullptr); @@ -167,14 +249,14 @@ TEST_P(umfCUDAProviderTest, allocInvalidSize) { // create CUDA provider umf_memory_provider_handle_t provider = nullptr; umf_result_t umf_result = - umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), ¶ms, &provider); + umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), params, &provider); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(provider, nullptr); void *ptr = nullptr; // NOTE: some scenarios are invalid only for the DEVICE allocations - if (params.memory_type == UMF_MEMORY_TYPE_DEVICE) { + if (expected_memory_type == UMF_MEMORY_TYPE_DEVICE) { // try to alloc SIZE_MAX umf_result = umfMemoryProviderAlloc(provider, SIZE_MAX, 0, &ptr); ASSERT_EQ(ptr, nullptr); @@ -198,14 +280,14 @@ TEST_P(umfCUDAProviderTest, providerCreateInvalidArgs) { umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), nullptr, &provider); ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); - umf_result = umfMemoryProviderCreate(nullptr, ¶ms, nullptr); + umf_result = umfMemoryProviderCreate(nullptr, params, nullptr); ASSERT_EQ(umf_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); } TEST_P(umfCUDAProviderTest, getPageSizeInvalidArgs) { umf_memory_provider_handle_t provider = nullptr; umf_result_t umf_result = - umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), ¶ms, &provider); + umfMemoryProviderCreate(umfCUDAMemoryProviderOps(), params, &provider); ASSERT_EQ(umf_result, UMF_RESULT_SUCCESS); ASSERT_NE(provider, nullptr); @@ -218,26 +300,51 @@ TEST_P(umfCUDAProviderTest, getPageSizeInvalidArgs) { umfMemoryProviderDestroy(provider); } +TEST_P(umfCUDAProviderTest, cudaProviderNullParams) { + umf_result_t res = umfCUDAMemoryProviderParamsCreate(nullptr); + EXPECT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfCUDAMemoryProviderParamsSetContext(nullptr, expected_context); + EXPECT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = umfCUDAMemoryProviderParamsSetDevice(nullptr, 1); + EXPECT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); + + res = + umfCUDAMemoryProviderParamsSetMemoryType(nullptr, expected_memory_type); + EXPECT_EQ(res, UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + // TODO add tests that mixes CUDA Memory Provider and Disjoint Pool -cuda_memory_provider_params_t cuParams_device_memory = - create_cuda_prov_params(UMF_MEMORY_TYPE_DEVICE); -cuda_memory_provider_params_t cuParams_shared_memory = - create_cuda_prov_params(UMF_MEMORY_TYPE_SHARED); -cuda_memory_provider_params_t cuParams_host_memory = - create_cuda_prov_params(UMF_MEMORY_TYPE_HOST); +CUDATestHelper cudaTestHelper; + +cuda_params_unique_handle_t cuParams_device_memory = create_cuda_prov_params( + cudaTestHelper.get_test_context(), cudaTestHelper.get_test_device(), + UMF_MEMORY_TYPE_DEVICE); +cuda_params_unique_handle_t cuParams_shared_memory = create_cuda_prov_params( + cudaTestHelper.get_test_context(), cudaTestHelper.get_test_device(), + UMF_MEMORY_TYPE_SHARED); +cuda_params_unique_handle_t cuParams_host_memory = create_cuda_prov_params( + cudaTestHelper.get_test_context(), cudaTestHelper.get_test_device(), + UMF_MEMORY_TYPE_HOST); -CUDAMemoryAccessor - cuAccessor((CUcontext)cuParams_device_memory.cuda_context_handle, - (CUdevice)cuParams_device_memory.cuda_device_handle); +CUDAMemoryAccessor cuAccessor(cudaTestHelper.get_test_context(), + cudaTestHelper.get_test_device()); HostMemoryAccessor hostAccessor; INSTANTIATE_TEST_SUITE_P( umfCUDAProviderTestSuite, umfCUDAProviderTest, ::testing::Values( - CUDAProviderTestParams{cuParams_device_memory, &cuAccessor}, - CUDAProviderTestParams{cuParams_shared_memory, &hostAccessor}, - CUDAProviderTestParams{cuParams_host_memory, &hostAccessor})); + CUDAProviderTestParams{cuParams_device_memory.get(), + cudaTestHelper.get_test_context(), + UMF_MEMORY_TYPE_DEVICE, &cuAccessor}, + CUDAProviderTestParams{cuParams_shared_memory.get(), + cudaTestHelper.get_test_context(), + UMF_MEMORY_TYPE_SHARED, &hostAccessor}, + CUDAProviderTestParams{cuParams_host_memory.get(), + cudaTestHelper.get_test_context(), + UMF_MEMORY_TYPE_HOST, &hostAccessor})); // TODO: add IPC API GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(umfIpcTest); @@ -246,5 +353,5 @@ INSTANTIATE_TEST_SUITE_P(umfCUDAProviderTestSuite, umfIpcTest, ::testing::Values(ipcTestParams{ umfProxyPoolOps(), nullptr, umfCUDAMemoryProviderOps(), - &cuParams_device_memory, &cuAccessor})); + cuParams_device_memory.get(), &cuAccessor, false})); */