diff --git a/.github/intel-llvm-mirror-base-commit b/.github/intel-llvm-mirror-base-commit index 3d8197503d..fdb4d85658 100644 --- a/.github/intel-llvm-mirror-base-commit +++ b/.github/intel-llvm-mirror-base-commit @@ -1 +1 @@ -81641383bc2714f2a1e0328e52ad79487a884f30 +3e95c0c70850b8b668116d9a491d25dd969c6329 diff --git a/include/ur_api.h b/include/ur_api.h index 8baf407095..1bba8a950e 100644 --- a/include/ur_api.h +++ b/include/ur_api.h @@ -305,26 +305,6 @@ typedef enum ur_function_t { UR_FUNCTION_ENQUEUE_USM_FILL_2D = 151, /// Enumerator for ::urEnqueueUSMMemcpy2D UR_FUNCTION_ENQUEUE_USM_MEMCPY_2D = 152, - /// Enumerator for ::urVirtualMemGranularityGetInfo - UR_FUNCTION_VIRTUAL_MEM_GRANULARITY_GET_INFO = 153, - /// Enumerator for ::urVirtualMemReserve - UR_FUNCTION_VIRTUAL_MEM_RESERVE = 154, - /// Enumerator for ::urVirtualMemFree - UR_FUNCTION_VIRTUAL_MEM_FREE = 155, - /// Enumerator for ::urVirtualMemMap - UR_FUNCTION_VIRTUAL_MEM_MAP = 156, - /// Enumerator for ::urVirtualMemUnmap - UR_FUNCTION_VIRTUAL_MEM_UNMAP = 157, - /// Enumerator for ::urVirtualMemSetAccess - UR_FUNCTION_VIRTUAL_MEM_SET_ACCESS = 158, - /// Enumerator for ::urVirtualMemGetInfo - UR_FUNCTION_VIRTUAL_MEM_GET_INFO = 159, - /// Enumerator for ::urPhysicalMemCreate - UR_FUNCTION_PHYSICAL_MEM_CREATE = 160, - /// Enumerator for ::urPhysicalMemRetain - UR_FUNCTION_PHYSICAL_MEM_RETAIN = 161, - /// Enumerator for ::urPhysicalMemRelease - UR_FUNCTION_PHYSICAL_MEM_RELEASE = 162, /// Enumerator for ::urUSMImportExp UR_FUNCTION_USM_IMPORT_EXP = 163, /// Enumerator for ::urUSMReleaseExp @@ -421,8 +401,6 @@ typedef enum ur_function_t { UR_FUNCTION_BINDLESS_IMAGES_MAP_EXTERNAL_LINEAR_MEMORY_EXP = 245, /// Enumerator for ::urEnqueueEventsWaitWithBarrierExt UR_FUNCTION_ENQUEUE_EVENTS_WAIT_WITH_BARRIER_EXT = 246, - /// Enumerator for ::urPhysicalMemGetInfo - UR_FUNCTION_PHYSICAL_MEM_GET_INFO = 249, /// Enumerator for ::urEnqueueUSMDeviceAllocExp UR_FUNCTION_ENQUEUE_USM_DEVICE_ALLOC_EXP = 250, /// Enumerator for ::urEnqueueUSMSharedAllocExp @@ -467,8 +445,36 @@ typedef enum ur_function_t { UR_FUNCTION_KERNEL_SUGGEST_MAX_COOPERATIVE_GROUP_COUNT = 272, /// Enumerator for ::urUSMContextMemcpyExp UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP = 273, + /// Enumerator for ::urVirtualMemGranularityGetInfo + UR_FUNCTION_VIRTUAL_MEM_GRANULARITY_GET_INFO = 274, + /// Enumerator for ::urVirtualMemReserve + UR_FUNCTION_VIRTUAL_MEM_RESERVE = 275, + /// Enumerator for ::urVirtualMemFree + UR_FUNCTION_VIRTUAL_MEM_FREE = 276, + /// Enumerator for ::urVirtualMemMap + UR_FUNCTION_VIRTUAL_MEM_MAP = 277, + /// Enumerator for ::urVirtualMemUnmap + UR_FUNCTION_VIRTUAL_MEM_UNMAP = 278, + /// Enumerator for ::urVirtualMemSetAccess + UR_FUNCTION_VIRTUAL_MEM_SET_ACCESS = 279, + /// Enumerator for ::urVirtualMemGetInfo + UR_FUNCTION_VIRTUAL_MEM_GET_INFO = 280, + /// Enumerator for ::urPhysicalMemCreate + UR_FUNCTION_PHYSICAL_MEM_CREATE = 281, + /// Enumerator for ::urPhysicalMemRetain + UR_FUNCTION_PHYSICAL_MEM_RETAIN = 282, + /// Enumerator for ::urPhysicalMemRelease + UR_FUNCTION_PHYSICAL_MEM_RELEASE = 283, + /// Enumerator for ::urPhysicalMemGetInfo + UR_FUNCTION_PHYSICAL_MEM_GET_INFO = 284, + /// Enumerator for ::urMemoryExportAllocExportableMemoryExp + UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP = 285, + /// Enumerator for ::urMemoryExportFreeExportableMemoryExp + UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP = 286, + /// Enumerator for ::urMemoryExportExportMemoryHandleExp + UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP = 287, /// Enumerator for ::urBindlessImagesSupportsImportingHandleTypeExp - UR_FUNCTION_BINDLESS_IMAGES_SUPPORTS_IMPORTING_HANDLE_TYPE_EXP = 274, + UR_FUNCTION_BINDLESS_IMAGES_SUPPORTS_IMPORTING_HANDLE_TYPE_EXP = 288, /// @cond UR_FUNCTION_FORCE_UINT32 = 0x7fffffff /// @endcond @@ -534,8 +540,6 @@ typedef enum ur_structure_type_t { UR_STRUCTURE_TYPE_DEVICE_PARTITION_PROPERTIES = 26, /// ::ur_kernel_arg_mem_obj_properties_t UR_STRUCTURE_TYPE_KERNEL_ARG_MEM_OBJ_PROPERTIES = 27, - /// ::ur_physical_mem_properties_t - UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES = 28, /// ::ur_kernel_arg_pointer_properties_t UR_STRUCTURE_TYPE_KERNEL_ARG_POINTER_PROPERTIES = 29, /// ::ur_kernel_arg_sampler_properties_t @@ -550,6 +554,8 @@ typedef enum ur_structure_type_t { UR_STRUCTURE_TYPE_USM_ALLOC_LOCATION_DESC = 35, /// ::ur_usm_pool_buffer_desc_t UR_STRUCTURE_TYPE_USM_POOL_BUFFER_DESC = 36, + /// ::ur_physical_mem_properties_t + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES = 37, /// ::ur_exp_command_buffer_desc_t UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC = 0x1000, /// ::ur_exp_command_buffer_update_kernel_launch_desc_t @@ -2314,6 +2320,10 @@ typedef enum ur_device_info_t { /// [::ur_kernel_launch_properties_flags_t] Bitfield of supported kernel /// launch properties. UR_DEVICE_INFO_KERNEL_LAUNCH_CAPABILITIES = 128, + /// [uint8_t[]][optional-query] return device Windows LUID + UR_DEVICE_INFO_LUID = 129, + /// [uint32_t][optional-query] return device Windows node mask + UR_DEVICE_INFO_NODE_MASK = 130, /// [::ur_bool_t] Returns true if the device supports the use of /// command-buffers. UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP = 0x1000, @@ -2426,6 +2436,10 @@ typedef enum ur_device_info_t { /// [::ur_bool_t] returns true if the device supports /// ::urUSMContextMemcpyExp UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP = 0x7000, + /// [::ur_bool_t] returns true if the device supports the allocation of + /// exportable linear layout device memory and exporting that memory to + /// an interoperable handle. + UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP = 0x8000, /// @cond UR_DEVICE_INFO_FORCE_UINT32 = 0x7fffffff /// @endcond @@ -2451,7 +2465,8 @@ typedef enum ur_device_info_t { /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hDevice` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP < propName` +/// + `::UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP < +/// propName` /// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION /// + If `propName` is not supported by the adapter. /// - ::UR_RESULT_ERROR_INVALID_SIZE @@ -12326,6 +12341,127 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( /// array. ur_event_handle_t *phEvent); +#if !defined(__GNUC__) +#pragma endregion +#endif +// Memory Export Extension APIs +#if !defined(__GNUC__) +#pragma region memory_export_(experimental) +#endif +/////////////////////////////////////////////////////////////////////////////// +/// @brief Allocate an exportable memory region and return a pointer to that +/// allocation. +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// + `(hDevice == nullptr) || (hContext == nullptr)` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `ppMem == nullptr` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT +/// + `alignment != 0 && ((alignment & (alignment-1)) != 0)` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + If `alignment` exceeds largest supported data type by `hDevice` +/// - ::UR_RESULT_ERROR_INVALID_USM_SIZE +/// + `size == 0` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_SIZE +/// + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE. +/// - +/// ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE:DEVICE_INFO_MEMORY_EXPORT_LINEAR_MEMORY_EXPORT_SUPPORT_EXP +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. + size_t size, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Free an exportable memory allocation. +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// + `(hDevice == nullptr) || (hContext == nullptr)` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `pMem == nullptr` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Obtain an exportable handle to a memory allocated with +/// `AllocExportableMemoryExp`.The returned external memory type will be +/// that which was specified upon +/// allocation of the exportable memory (e.g. `opaque_fd` or +/// `win32_nt_handle`). +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// + `(hDevice == nullptr) || (hContext == nullptr)` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `pMemHandleRet == nullptr || pMem == nullptr` +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet); + #if !defined(__GNUC__) #pragma endregion #endif @@ -15280,6 +15416,41 @@ typedef struct ur_command_buffer_get_native_handle_exp_params_t { ur_native_handle_t **pphNativeCommandBuffer; } ur_command_buffer_get_native_handle_exp_params_t; +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urMemoryExportAllocExportableMemoryExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_memory_export_alloc_exportable_memory_exp_params_t { + ur_context_handle_t *phContext; + ur_device_handle_t *phDevice; + size_t *palignment; + size_t *psize; + ur_exp_external_mem_type_t *phandleTypeToExport; + void ***pppMem; +} ur_memory_export_alloc_exportable_memory_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urMemoryExportFreeExportableMemoryExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_memory_export_free_exportable_memory_exp_params_t { + ur_context_handle_t *phContext; + ur_device_handle_t *phDevice; + void **ppMem; +} ur_memory_export_free_exportable_memory_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urMemoryExportExportMemoryHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_memory_export_export_memory_handle_exp_params_t { + ur_context_handle_t *phContext; + ur_device_handle_t *phDevice; + ur_exp_external_mem_type_t *phandleTypeToExport; + void **ppMem; + void **ppMemHandleRet; +} ur_memory_export_export_memory_handle_exp_params_t; + /////////////////////////////////////////////////////////////////////////////// /// @brief Function parameters for urUsmP2PEnablePeerAccessExp /// @details Each entry is a pointer to the parameter passed to the function; diff --git a/include/ur_api_funcs.def b/include/ur_api_funcs.def index cb7e03bba4..f0c92445b9 100644 --- a/include/ur_api_funcs.def +++ b/include/ur_api_funcs.def @@ -206,6 +206,9 @@ _UR_API(urCommandBufferUpdateSignalEventExp) _UR_API(urCommandBufferUpdateWaitEventsExp) _UR_API(urCommandBufferGetInfoExp) _UR_API(urCommandBufferGetNativeHandleExp) +_UR_API(urMemoryExportAllocExportableMemoryExp) +_UR_API(urMemoryExportFreeExportableMemoryExp) +_UR_API(urMemoryExportExportMemoryHandleExp) _UR_API(urUsmP2PEnablePeerAccessExp) _UR_API(urUsmP2PDisablePeerAccessExp) _UR_API(urUsmP2PPeerAccessGetInfoExp) diff --git a/include/ur_ddi.h b/include/ur_ddi.h index 5f58d4c560..8ab686aa58 100644 --- a/include/ur_ddi.h +++ b/include/ur_ddi.h @@ -1787,6 +1787,51 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetCommandBufferExpProcAddrTable( typedef ur_result_t(UR_APICALL *ur_pfnGetCommandBufferExpProcAddrTable_t)( ur_api_version_t, ur_command_buffer_exp_dditable_t *); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urMemoryExportAllocExportableMemoryExp +typedef ur_result_t(UR_APICALL *ur_pfnMemoryExportAllocExportableMemoryExp_t)( + ur_context_handle_t, ur_device_handle_t, size_t, size_t, + ur_exp_external_mem_type_t, void **); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urMemoryExportFreeExportableMemoryExp +typedef ur_result_t(UR_APICALL *ur_pfnMemoryExportFreeExportableMemoryExp_t)( + ur_context_handle_t, ur_device_handle_t, void *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urMemoryExportExportMemoryHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnMemoryExportExportMemoryHandleExp_t)( + ur_context_handle_t, ur_device_handle_t, ur_exp_external_mem_type_t, void *, + void *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Table of MemoryExportExp functions pointers +typedef struct ur_memory_export_exp_dditable_t { + ur_pfnMemoryExportAllocExportableMemoryExp_t pfnAllocExportableMemoryExp; + ur_pfnMemoryExportFreeExportableMemoryExp_t pfnFreeExportableMemoryExp; + ur_pfnMemoryExportExportMemoryHandleExp_t pfnExportMemoryHandleExp; +} ur_memory_export_exp_dditable_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's MemoryExportExp table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +UR_DLLEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + /// [in] API version requested + ur_api_version_t version, + /// [in,out] pointer to table of DDI function pointers + ur_memory_export_exp_dditable_t *pDdiTable); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urGetMemoryExportExpProcAddrTable +typedef ur_result_t(UR_APICALL *ur_pfnGetMemoryExportExpProcAddrTable_t)( + ur_api_version_t, ur_memory_export_exp_dditable_t *); + /////////////////////////////////////////////////////////////////////////////// /// @brief Function-pointer for urUsmP2PEnablePeerAccessExp typedef ur_result_t(UR_APICALL *ur_pfnUsmP2PEnablePeerAccessExp_t)( @@ -2004,6 +2049,7 @@ typedef struct ur_dditable_t { ur_usm_exp_dditable_t USMExp; ur_bindless_images_exp_dditable_t BindlessImagesExp; ur_command_buffer_exp_dditable_t CommandBufferExp; + ur_memory_export_exp_dditable_t MemoryExportExp; ur_usm_p2p_exp_dditable_t UsmP2PExp; ur_virtual_mem_dditable_t VirtualMem; ur_device_dditable_t Device; diff --git a/include/ur_print.h b/include/ur_print.h index dd7d60d11a..8130df0c5b 100644 --- a/include/ur_print.h +++ b/include/ur_print.h @@ -3490,6 +3490,39 @@ urPrintCommandBufferGetNativeHandleExpParams( const struct ur_command_buffer_get_native_handle_exp_params_t *params, char *buffer, const size_t buff_size, size_t *out_size); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_memory_export_alloc_exportable_memory_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL +urPrintMemoryExportAllocExportableMemoryExpParams( + const struct ur_memory_export_alloc_exportable_memory_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_memory_export_free_exportable_memory_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL +urPrintMemoryExportFreeExportableMemoryExpParams( + const struct ur_memory_export_free_exportable_memory_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_memory_export_export_memory_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL +urPrintMemoryExportExportMemoryHandleExpParams( + const struct ur_memory_export_export_memory_handle_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size); + /////////////////////////////////////////////////////////////////////////////// /// @brief Print ur_usm_p2p_enable_peer_access_exp_params_t struct /// @returns diff --git a/include/ur_print.hpp b/include/ur_print.hpp index c7dc701db3..91c9973a3a 100644 --- a/include/ur_print.hpp +++ b/include/ur_print.hpp @@ -1018,36 +1018,6 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_function_t value) { case UR_FUNCTION_ENQUEUE_USM_MEMCPY_2D: os << "UR_FUNCTION_ENQUEUE_USM_MEMCPY_2D"; break; - case UR_FUNCTION_VIRTUAL_MEM_GRANULARITY_GET_INFO: - os << "UR_FUNCTION_VIRTUAL_MEM_GRANULARITY_GET_INFO"; - break; - case UR_FUNCTION_VIRTUAL_MEM_RESERVE: - os << "UR_FUNCTION_VIRTUAL_MEM_RESERVE"; - break; - case UR_FUNCTION_VIRTUAL_MEM_FREE: - os << "UR_FUNCTION_VIRTUAL_MEM_FREE"; - break; - case UR_FUNCTION_VIRTUAL_MEM_MAP: - os << "UR_FUNCTION_VIRTUAL_MEM_MAP"; - break; - case UR_FUNCTION_VIRTUAL_MEM_UNMAP: - os << "UR_FUNCTION_VIRTUAL_MEM_UNMAP"; - break; - case UR_FUNCTION_VIRTUAL_MEM_SET_ACCESS: - os << "UR_FUNCTION_VIRTUAL_MEM_SET_ACCESS"; - break; - case UR_FUNCTION_VIRTUAL_MEM_GET_INFO: - os << "UR_FUNCTION_VIRTUAL_MEM_GET_INFO"; - break; - case UR_FUNCTION_PHYSICAL_MEM_CREATE: - os << "UR_FUNCTION_PHYSICAL_MEM_CREATE"; - break; - case UR_FUNCTION_PHYSICAL_MEM_RETAIN: - os << "UR_FUNCTION_PHYSICAL_MEM_RETAIN"; - break; - case UR_FUNCTION_PHYSICAL_MEM_RELEASE: - os << "UR_FUNCTION_PHYSICAL_MEM_RELEASE"; - break; case UR_FUNCTION_USM_IMPORT_EXP: os << "UR_FUNCTION_USM_IMPORT_EXP"; break; @@ -1192,9 +1162,6 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_function_t value) { case UR_FUNCTION_ENQUEUE_EVENTS_WAIT_WITH_BARRIER_EXT: os << "UR_FUNCTION_ENQUEUE_EVENTS_WAIT_WITH_BARRIER_EXT"; break; - case UR_FUNCTION_PHYSICAL_MEM_GET_INFO: - os << "UR_FUNCTION_PHYSICAL_MEM_GET_INFO"; - break; case UR_FUNCTION_ENQUEUE_USM_DEVICE_ALLOC_EXP: os << "UR_FUNCTION_ENQUEUE_USM_DEVICE_ALLOC_EXP"; break; @@ -1262,6 +1229,48 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_function_t value) { case UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP: os << "UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP"; break; + case UR_FUNCTION_VIRTUAL_MEM_GRANULARITY_GET_INFO: + os << "UR_FUNCTION_VIRTUAL_MEM_GRANULARITY_GET_INFO"; + break; + case UR_FUNCTION_VIRTUAL_MEM_RESERVE: + os << "UR_FUNCTION_VIRTUAL_MEM_RESERVE"; + break; + case UR_FUNCTION_VIRTUAL_MEM_FREE: + os << "UR_FUNCTION_VIRTUAL_MEM_FREE"; + break; + case UR_FUNCTION_VIRTUAL_MEM_MAP: + os << "UR_FUNCTION_VIRTUAL_MEM_MAP"; + break; + case UR_FUNCTION_VIRTUAL_MEM_UNMAP: + os << "UR_FUNCTION_VIRTUAL_MEM_UNMAP"; + break; + case UR_FUNCTION_VIRTUAL_MEM_SET_ACCESS: + os << "UR_FUNCTION_VIRTUAL_MEM_SET_ACCESS"; + break; + case UR_FUNCTION_VIRTUAL_MEM_GET_INFO: + os << "UR_FUNCTION_VIRTUAL_MEM_GET_INFO"; + break; + case UR_FUNCTION_PHYSICAL_MEM_CREATE: + os << "UR_FUNCTION_PHYSICAL_MEM_CREATE"; + break; + case UR_FUNCTION_PHYSICAL_MEM_RETAIN: + os << "UR_FUNCTION_PHYSICAL_MEM_RETAIN"; + break; + case UR_FUNCTION_PHYSICAL_MEM_RELEASE: + os << "UR_FUNCTION_PHYSICAL_MEM_RELEASE"; + break; + case UR_FUNCTION_PHYSICAL_MEM_GET_INFO: + os << "UR_FUNCTION_PHYSICAL_MEM_GET_INFO"; + break; + case UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP: + os << "UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP"; + break; + case UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP: + os << "UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP"; + break; + case UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP: + os << "UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP"; + break; case UR_FUNCTION_BINDLESS_IMAGES_SUPPORTS_IMPORTING_HANDLE_TYPE_EXP: os << "UR_FUNCTION_BINDLESS_IMAGES_SUPPORTS_IMPORTING_HANDLE_TYPE_EXP"; break; @@ -1362,9 +1371,6 @@ inline std::ostream &operator<<(std::ostream &os, case UR_STRUCTURE_TYPE_KERNEL_ARG_MEM_OBJ_PROPERTIES: os << "UR_STRUCTURE_TYPE_KERNEL_ARG_MEM_OBJ_PROPERTIES"; break; - case UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES: - os << "UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES"; - break; case UR_STRUCTURE_TYPE_KERNEL_ARG_POINTER_PROPERTIES: os << "UR_STRUCTURE_TYPE_KERNEL_ARG_POINTER_PROPERTIES"; break; @@ -1386,6 +1392,9 @@ inline std::ostream &operator<<(std::ostream &os, case UR_STRUCTURE_TYPE_USM_POOL_BUFFER_DESC: os << "UR_STRUCTURE_TYPE_USM_POOL_BUFFER_DESC"; break; + case UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES: + os << "UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES"; + break; case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC: os << "UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC"; break; @@ -1608,12 +1617,6 @@ inline ur_result_t printStruct(std::ostream &os, const void *ptr) { printPtr(os, pstruct); } break; - case UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES: { - const ur_physical_mem_properties_t *pstruct = - (const ur_physical_mem_properties_t *)ptr; - printPtr(os, pstruct); - } break; - case UR_STRUCTURE_TYPE_KERNEL_ARG_POINTER_PROPERTIES: { const ur_kernel_arg_pointer_properties_t *pstruct = (const ur_kernel_arg_pointer_properties_t *)ptr; @@ -1656,6 +1659,12 @@ inline ur_result_t printStruct(std::ostream &os, const void *ptr) { printPtr(os, pstruct); } break; + case UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES: { + const ur_physical_mem_properties_t *pstruct = + (const ur_physical_mem_properties_t *)ptr; + printPtr(os, pstruct); + } break; + case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC: { const ur_exp_command_buffer_desc_t *pstruct = (const ur_exp_command_buffer_desc_t *)ptr; @@ -2996,6 +3005,12 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_device_info_t value) { case UR_DEVICE_INFO_KERNEL_LAUNCH_CAPABILITIES: os << "UR_DEVICE_INFO_KERNEL_LAUNCH_CAPABILITIES"; break; + case UR_DEVICE_INFO_LUID: + os << "UR_DEVICE_INFO_LUID"; + break; + case UR_DEVICE_INFO_NODE_MASK: + os << "UR_DEVICE_INFO_NODE_MASK"; + break; case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: os << "UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP"; break; @@ -3110,6 +3125,9 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_device_info_t value) { case UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP: os << "UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP"; break; + case UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP: + os << "UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP"; + break; default: os << "unknown enumerator"; break; @@ -4750,6 +4768,33 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, os << ")"; } break; + case UR_DEVICE_INFO_LUID: { + + const uint8_t *tptr = (const uint8_t *)ptr; + os << "{"; + size_t nelems = size / sizeof(uint8_t); + for (size_t i = 0; i < nelems; ++i) { + if (i != 0) { + os << ", "; + } + + os << static_cast(tptr[i]); + } + os << "}"; + } break; + case UR_DEVICE_INFO_NODE_MASK: { + const uint32_t *tptr = (const uint32_t *)ptr; + if (sizeof(uint32_t) > size) { + os << "invalid size (is: " << size << ", expected: >=" << sizeof(uint32_t) + << ")"; + return UR_RESULT_ERROR_INVALID_SIZE; + } + os << (const void *)(tptr) << " ("; + + os << *tptr; + + os << ")"; + } break; case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: { const ur_bool_t *tptr = (const ur_bool_t *)ptr; if (sizeof(ur_bool_t) > size) { @@ -5248,6 +5293,19 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, os << ")"; } break; + case UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP: { + const ur_bool_t *tptr = (const ur_bool_t *)ptr; + if (sizeof(ur_bool_t) > size) { + os << "invalid size (is: " << size + << ", expected: >=" << sizeof(ur_bool_t) << ")"; + return UR_RESULT_ERROR_INVALID_SIZE; + } + os << (const void *)(tptr) << " ("; + + os << *tptr; + + os << ")"; + } break; default: os << "unknown enumerator"; return UR_RESULT_ERROR_INVALID_ENUMERATION; @@ -20183,6 +20241,109 @@ operator<<(std::ostream &os, [[maybe_unused]] const struct return os; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the +/// ur_memory_export_alloc_exportable_memory_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, [[maybe_unused]] const struct + ur_memory_export_alloc_exportable_memory_exp_params_t *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hDevice = "; + + ur::details::printPtr(os, *(params->phDevice)); + + os << ", "; + os << ".alignment = "; + + os << *(params->palignment); + + os << ", "; + os << ".size = "; + + os << *(params->psize); + + os << ", "; + os << ".handleTypeToExport = "; + + os << *(params->phandleTypeToExport); + + os << ", "; + os << ".ppMem = "; + + os << *(params->pppMem); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the +/// ur_memory_export_free_exportable_memory_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, [[maybe_unused]] const struct + ur_memory_export_free_exportable_memory_exp_params_t *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hDevice = "; + + ur::details::printPtr(os, *(params->phDevice)); + + os << ", "; + os << ".pMem = "; + + os << *(params->ppMem); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the +/// ur_memory_export_export_memory_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, [[maybe_unused]] const struct + ur_memory_export_export_memory_handle_exp_params_t *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hDevice = "; + + ur::details::printPtr(os, *(params->phDevice)); + + os << ", "; + os << ".handleTypeToExport = "; + + os << *(params->phandleTypeToExport); + + os << ", "; + os << ".pMem = "; + + os << *(params->ppMem); + + os << ", "; + os << ".pMemHandleRet = "; + + os << *(params->ppMemHandleRet); + + return os; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Print operator for the ur_usm_p2p_enable_peer_access_exp_params_t /// type @@ -21544,6 +21705,18 @@ inline ur_result_t UR_APICALL printFunctionParams(std::ostream &os, os << (const struct ur_command_buffer_get_native_handle_exp_params_t *) params; } break; + case UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP: { + os << (const struct ur_memory_export_alloc_exportable_memory_exp_params_t *) + params; + } break; + case UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP: { + os << (const struct ur_memory_export_free_exportable_memory_exp_params_t *) + params; + } break; + case UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP: { + os << (const struct ur_memory_export_export_memory_handle_exp_params_t *) + params; + } break; case UR_FUNCTION_USM_P2P_ENABLE_PEER_ACCESS_EXP: { os << (const struct ur_usm_p2p_enable_peer_access_exp_params_t *)params; } break; diff --git a/scripts/core/EXP-MEMORY-EXPORT.rst b/scripts/core/EXP-MEMORY-EXPORT.rst new file mode 100644 index 0000000000..7bb99b1194 --- /dev/null +++ b/scripts/core/EXP-MEMORY-EXPORT.rst @@ -0,0 +1,104 @@ +<% + OneApi=tags['$OneApi'] + x=tags['$x'] + X=x.upper() +%> + +.. _experimental-memory-export: + +================================================================================ +Memory Export +================================================================================ + +.. warning:: + + Experimental features: + + * May be replaced, updated, or removed at any time. + * Do not require maintaining API/ABI stability of their own additions over + time. + * Do not require conformance testing of their own additions. + +Motivation +-------------------------------------------------------------------------------- + + +The `DPC++ Memory Export extension` provides new APIs for +allocating and deallocating exportable device memory, and obtaining a handle to +that memory which can be used in external APIs. This is useful when applications +want to share device memory between different APIs. + +Without the ability to allocate exportable memory and obtain an interoperable +handle, applications have to copy device memory allocated by one API to +the host, then copy that host memory back to the device in a memory region +allocated by a second API. If the second API modifies that memory, then this +process would have to be repeated in the opposite direction in order for the +first API to see the changes made to that memory. + +This extension enables copy-free sharing of SYCL allocated device memory with +external APIs. + +Overview +-------------------------------------------------------------------------------- +In this document, we propose the following experimental additions to the Unified +Runtime: + +* Exporting Memory + + * Allocating an exportable device resident memory region. + * Obtaining a handle to the allocated exportable memory region, for the + purpose of using this handle to import the memory in external APIs such as + Vulkan or DirectX. + * Freeing the exportable device resident memory region. + + +Please note that the following enums and types used by this extension are +dependent on the Bindless Images extension. Their definitions are provided in +`exp-bindless-images.yml`. + +* ${x}_exp_external_mem_handle_t + +* ${x}_exp_external_mem_type_t + * ${X}_EXP_EXTERNAL_MEM_TYPE_OPAQUE_FD + * ${X}_EXP_EXTERNAL_MEM_TYPE_WIN32_NT + +API +-------------------------------------------------------------------------------- + +Enums +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* ${x}_device_info_t + * ${X}_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP + +* ${x}_exp_external_mem_type_t + * ${X}_EXP_EXTERNAL_MEM_TYPE_OPAQUE_FD + * ${X}_EXP_EXTERNAL_MEM_TYPE_WIN32_NT + +* ${x}_function_t + * ${X}_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP + * ${X}_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP + * ${X}_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP + +Types +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* ${x}_exp_external_mem_handle_t + +Functions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* ${x}MemoryExportAllocExportableMemoryExp +* ${x}MemoryExportFreeExportableMemoryExp +* ${x}MemoryExportExportMemoryHandleExp + +Changelog +-------------------------------------------------------------------------------- + ++----------+----------------------------------------------------------+ +| Revision | Changes | ++==========+==========================================================+ +| 1.0 | Initial draft with Level Zero adapter implementation | ++----------+----------------------------------------------------------+ + +Contributors +-------------------------------------------------------------------------------- + +* Przemek Malon `przemek.malon@codeplay.com `_ diff --git a/scripts/core/device.yml b/scripts/core/device.yml index f7575ca9e0..7bef8a45a5 100644 --- a/scripts/core/device.yml +++ b/scripts/core/device.yml @@ -464,6 +464,10 @@ etors: desc: "[$x_bool_t] support for native bfloat16 conversions" - name: KERNEL_LAUNCH_CAPABILITIES desc: "[$x_kernel_launch_properties_flags_t] Bitfield of supported kernel launch properties." + - name: LUID + desc: "[uint8_t[]][optional-query] return device Windows LUID" + - name: NODE_MASK + desc: "[uint32_t][optional-query] return device Windows node mask" --- #-------------------------------------------------------------------------- type: function desc: "Retrieves various information about device" diff --git a/scripts/core/exp-memory-export.yml b/scripts/core/exp-memory-export.yml new file mode 100644 index 0000000000..4c388e9ae6 --- /dev/null +++ b/scripts/core/exp-memory-export.yml @@ -0,0 +1,145 @@ +# +# Copyright (C) 2025 Codeplay +# +# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +# Exceptions. +# +# See LICENSE.TXT +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# See YaML.md for syntax definition +# +--- #-------------------------------------------------------------------------- +type: header +desc: "Memory Export Extension APIs" +ordinal: "99" +--- #-------------------------------------------------------------------------- +type: enum +extend: true +typed_etors: true +desc: "Extension enums to $x_device_info_t to support Memory Export." +name: $x_device_info_t +etors: + - name: MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP + value: "0x8000" + desc: |- + [$x_bool_t] returns true if the device supports the allocation of + exportable linear layout device memory and exporting that memory to + an interoperable handle. +--- #-------------------------------------------------------------------------- +type: function +desc: |- + Allocate an exportable memory region and return a pointer to that + allocation. +class: $xMemoryExport +name: AllocExportableMemoryExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] Handle to context in which to allocate memory." + - type: $x_device_handle_t + name: hDevice + desc: "[in] Handle to device on which to allocate memory." + - type: size_t + name: alignment + desc: "[in] Requested alignment of the allocation." + - type: size_t + name: size + desc: "[in] Requested size of the allocation." + - type: $x_exp_external_mem_type_t + name: handleTypeToExport + desc: |- + [in] Type of the memory handle to be exported (e.g. file descriptor, + or win32 NT handle). + - type: void ** + name: ppMem + desc: "[out][alloc] Pointer to allocated exportable memory." +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_DEVICE + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`(hDevice == nullptr) || (hContext == nullptr)`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`ppMem == nullptr`" + - $X_RESULT_ERROR_UNSUPPORTED_ALIGNMENT: + - "`alignment != 0 && ((alignment & (alignment-1)) != 0)`" + - $X_RESULT_ERROR_INVALID_VALUE: + - "If `alignment` exceeds largest supported data type by `hDevice`" + - $X_RESULT_ERROR_INVALID_USM_SIZE: + - "`size == 0`" + - $X_RESULT_ERROR_UNSUPPORTED_SIZE: + - "`size` is greater than $X_DEVICE_INFO_MAX_MEM_ALLOC_SIZE." + - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:DEVICE_INFO_MEMORY_EXPORT_LINEAR_MEMORY_EXPORT_SUPPORT_EXP + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +--- #-------------------------------------------------------------------------- +type: function +desc: "Free an exportable memory allocation." +class: $xMemoryExport +name: FreeExportableMemoryExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] Handle to context in which to free memory." + - type: $x_device_handle_t + name: hDevice + desc: "[in] Handle to device on which to free memory." + - type: void * + name: pMem + desc: |- + [in][release] Pointer to exportable memory to be deallocated. +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_DEVICE + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`(hDevice == nullptr) || (hContext == nullptr)`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`pMem == nullptr`" + - $X_RESULT_ERROR_INVALID_VALUE + - $X_RESULT_ERROR_DEVICE_LOST + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +--- #-------------------------------------------------------------------------- +type: function +desc: |- + Obtain an exportable handle to a memory allocated with + `AllocExportableMemoryExp`. + The returned external memory type will be that which was specified upon + allocation of the exportable memory (e.g. `opaque_fd` or + `win32_nt_handle`). +class: $xMemoryExport +name: ExportMemoryHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: |- + [in] Handle to context in which the exportable memory was allocated. + - type: $x_device_handle_t + name: hDevice + desc: |- + [in] Handle to device on which the exportable memory was allocated. + - type: $x_exp_external_mem_type_t + name: handleTypeToExport + desc: |- + [in] Type of the memory handle to be exported (e.g. file descriptor, + or win32 NT handle). + - type: void * + name: pMem + desc: "[in] Pointer to exportable memory handle." + - type: void * + name: pMemHandleRet + desc: "[out] Returned exportable handle to memory allocated in `pMem`" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_DEVICE + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`(hDevice == nullptr) || (hContext == nullptr)`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`pMemHandleRet == nullptr || pMem == nullptr`" + - $X_RESULT_ERROR_INVALID_ENUMERATION + - $X_RESULT_ERROR_DEVICE_LOST + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_DEVICE_MEMORY diff --git a/scripts/core/registry.yml b/scripts/core/registry.yml index 4eeb16f3c5..349ac97a27 100644 --- a/scripts/core/registry.yml +++ b/scripts/core/registry.yml @@ -415,36 +415,6 @@ etors: - name: ENQUEUE_USM_MEMCPY_2D desc: Enumerator for $xEnqueueUSMMemcpy2D value: '152' -- name: VIRTUAL_MEM_GRANULARITY_GET_INFO - desc: Enumerator for $xVirtualMemGranularityGetInfo - value: '153' -- name: VIRTUAL_MEM_RESERVE - desc: Enumerator for $xVirtualMemReserve - value: '154' -- name: VIRTUAL_MEM_FREE - desc: Enumerator for $xVirtualMemFree - value: '155' -- name: VIRTUAL_MEM_MAP - desc: Enumerator for $xVirtualMemMap - value: '156' -- name: VIRTUAL_MEM_UNMAP - desc: Enumerator for $xVirtualMemUnmap - value: '157' -- name: VIRTUAL_MEM_SET_ACCESS - desc: Enumerator for $xVirtualMemSetAccess - value: '158' -- name: VIRTUAL_MEM_GET_INFO - desc: Enumerator for $xVirtualMemGetInfo - value: '159' -- name: PHYSICAL_MEM_CREATE - desc: Enumerator for $xPhysicalMemCreate - value: '160' -- name: PHYSICAL_MEM_RETAIN - desc: Enumerator for $xPhysicalMemRetain - value: '161' -- name: PHYSICAL_MEM_RELEASE - desc: Enumerator for $xPhysicalMemRelease - value: '162' - name: USM_IMPORT_EXP desc: Enumerator for $xUSMImportExp value: '163' @@ -589,9 +559,6 @@ etors: - name: ENQUEUE_EVENTS_WAIT_WITH_BARRIER_EXT desc: Enumerator for $xEnqueueEventsWaitWithBarrierExt value: '246' -- name: PHYSICAL_MEM_GET_INFO - desc: Enumerator for $xPhysicalMemGetInfo - value: '249' - name: ENQUEUE_USM_DEVICE_ALLOC_EXP desc: Enumerator for $xEnqueueUSMDeviceAllocExp value: '250' @@ -658,9 +625,51 @@ etors: - name: USM_CONTEXT_MEMCPY_EXP desc: Enumerator for $xUSMContextMemcpyExp value: '273' +- name: VIRTUAL_MEM_GRANULARITY_GET_INFO + desc: Enumerator for $xVirtualMemGranularityGetInfo + value: '274' +- name: VIRTUAL_MEM_RESERVE + desc: Enumerator for $xVirtualMemReserve + value: '275' +- name: VIRTUAL_MEM_FREE + desc: Enumerator for $xVirtualMemFree + value: '276' +- name: VIRTUAL_MEM_MAP + desc: Enumerator for $xVirtualMemMap + value: '277' +- name: VIRTUAL_MEM_UNMAP + desc: Enumerator for $xVirtualMemUnmap + value: '278' +- name: VIRTUAL_MEM_SET_ACCESS + desc: Enumerator for $xVirtualMemSetAccess + value: '279' +- name: VIRTUAL_MEM_GET_INFO + desc: Enumerator for $xVirtualMemGetInfo + value: '280' +- name: PHYSICAL_MEM_CREATE + desc: Enumerator for $xPhysicalMemCreate + value: '281' +- name: PHYSICAL_MEM_RETAIN + desc: Enumerator for $xPhysicalMemRetain + value: '282' +- name: PHYSICAL_MEM_RELEASE + desc: Enumerator for $xPhysicalMemRelease + value: '283' +- name: PHYSICAL_MEM_GET_INFO + desc: Enumerator for $xPhysicalMemGetInfo + value: '284' +- name: MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP + desc: Enumerator for $xMemoryExportAllocExportableMemoryExp + value: '285' +- name: MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP + desc: Enumerator for $xMemoryExportFreeExportableMemoryExp + value: '286' +- name: MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP + desc: Enumerator for $xMemoryExportExportMemoryHandleExp + value: '287' - name: BINDLESS_IMAGES_SUPPORTS_IMPORTING_HANDLE_TYPE_EXP desc: Enumerator for $xBindlessImagesSupportsImportingHandleTypeExp - value: '274' + value: '288' --- type: enum desc: Defines structure types @@ -750,9 +759,6 @@ etors: - name: KERNEL_ARG_MEM_OBJ_PROPERTIES desc: $x_kernel_arg_mem_obj_properties_t value: '27' -- name: PHYSICAL_MEM_PROPERTIES - desc: $x_physical_mem_properties_t - value: '28' - name: KERNEL_ARG_POINTER_PROPERTIES desc: $x_kernel_arg_pointer_properties_t value: '29' @@ -774,3 +780,6 @@ etors: - name: USM_POOL_BUFFER_DESC desc: $x_usm_pool_buffer_desc_t value: '36' +- name: PHYSICAL_MEM_PROPERTIES + desc: $x_physical_mem_properties_t + value: '37' diff --git a/source/adapters/adapter.def.in b/source/adapters/adapter.def.in index 73a360b36d..edccce444d 100644 --- a/source/adapters/adapter.def.in +++ b/source/adapters/adapter.def.in @@ -9,6 +9,7 @@ EXPORTS urGetEventProcAddrTable urGetKernelProcAddrTable urGetMemProcAddrTable + urGetMemoryExportExpProcAddrTable urGetPhysicalMemProcAddrTable urGetPlatformProcAddrTable urGetProgramProcAddrTable diff --git a/source/adapters/adapter.map.in b/source/adapters/adapter.map.in index 4ba14eacb7..54ff7d6b93 100644 --- a/source/adapters/adapter.map.in +++ b/source/adapters/adapter.map.in @@ -9,6 +9,7 @@ urGetEventProcAddrTable; urGetKernelProcAddrTable; urGetMemProcAddrTable; + urGetMemoryExportExpProcAddrTable; urGetPhysicalMemProcAddrTable; urGetPlatformProcAddrTable; urGetProgramProcAddrTable; diff --git a/source/adapters/cuda/CMakeLists.txt b/source/adapters/cuda/CMakeLists.txt index 8b8754fad2..5d7ba70e31 100644 --- a/source/adapters/cuda/CMakeLists.txt +++ b/source/adapters/cuda/CMakeLists.txt @@ -27,6 +27,7 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/image.cpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.hpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory_export.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp diff --git a/source/adapters/cuda/device.cpp b/source/adapters/cuda/device.cpp index a77fba9aff..03d9a13999 100644 --- a/source/adapters/cuda/device.cpp +++ b/source/adapters/cuda/device.cpp @@ -1186,6 +1186,46 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, return ReturnValue(0); } + case UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP: + return ReturnValue(false); + case UR_DEVICE_INFO_LUID: { + // LUID is only available on Windows. + // Intel extension for device LUID. This returns the LUID as + // std::array. For details about this extension, + // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. + std::array LUID{}; + cuDeviceGetLuid(LUID.data(), nullptr, hDevice->get()); + + bool isAllZeros = true; + for (char num : LUID) { + if (num != 0) { + isAllZeros = false; + } + } + + if (isAllZeros) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + std::array Name{}; + std::copy(LUID.begin(), LUID.end(), Name.begin()); + return ReturnValue(Name.data(), 8); + } + case UR_DEVICE_INFO_NODE_MASK: { + // Device node mask is only available on Windows. + // Intel extension for device node mask. This returns the node mask as + // uint32_t. For details about this extension, + // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. + uint32_t nodeMask = 0; + cuDeviceGetLuid(nullptr, &nodeMask, hDevice->get()); + + // If nodeMask has not changed, return unsupported. + if (nodeMask == 0) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + + return ReturnValue(nodeMask); + } default: break; } diff --git a/source/adapters/cuda/memory_export.cpp b/source/adapters/cuda/memory_export.cpp new file mode 100644 index 0000000000..d6b4001c00 --- /dev/null +++ b/source/adapters/cuda/memory_export.cpp @@ -0,0 +1,35 @@ +//===--------- memory_export.cpp - CUDA Adapter ---------------------------===// +// +// Copyright (C) 2025 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common/logger/ur_logger.hpp" +#include "ur_api.h" + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + size_t /*aligment*/, size_t /*size*/, + ur_exp_external_mem_type_t /*handleTypeToExport*/, void ** /*ppMem*/) { + UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + void * /*pMem*/) { + UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + ur_exp_external_mem_type_t /*handleTypeToExport*/, void * /*pMem*/, + void * /*pMemHandleRet*/) { + UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/source/adapters/cuda/ur_interface_loader.cpp b/source/adapters/cuda/ur_interface_loader.cpp index e3060d1c33..8430df0ab0 100644 --- a/source/adapters/cuda/ur_interface_loader.cpp +++ b/source/adapters/cuda/ur_interface_loader.cpp @@ -369,6 +369,19 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetBindlessImagesExpProcAddrTable( return UR_RESULT_SUCCESS; } +UR_DLLEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + ur_api_version_t version, ur_memory_export_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + pDdiTable->pfnAllocExportableMemoryExp = + urMemoryExportAllocExportableMemoryExp; + pDdiTable->pfnFreeExportableMemoryExp = urMemoryExportFreeExportableMemoryExp; + pDdiTable->pfnExportMemoryHandleExp = urMemoryExportExportMemoryHandleExp; + return UR_RESULT_SUCCESS; +} + UR_DLLEXPORT ur_result_t UR_APICALL urGetUSMExpProcAddrTable( ur_api_version_t version, ur_usm_exp_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); @@ -481,6 +494,7 @@ UR_DLLEXPORT ur_result_t UR_APICALL urAllAddrTable(ur_api_version_t version, urGetUsmP2PExpProcAddrTable(version, &pDdiTable->UsmP2PExp); urGetVirtualMemProcAddrTable(version, &pDdiTable->VirtualMem); urGetDeviceProcAddrTable(version, &pDdiTable->Device); + urGetMemoryExportExpProcAddrTable(version, &pDdiTable->MemoryExportExp); return UR_RESULT_SUCCESS; } diff --git a/source/adapters/hip/CMakeLists.txt b/source/adapters/hip/CMakeLists.txt index 914e36d41d..31f784e700 100644 --- a/source/adapters/hip/CMakeLists.txt +++ b/source/adapters/hip/CMakeLists.txt @@ -75,6 +75,7 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/image.cpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.hpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory_export.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp diff --git a/source/adapters/hip/device.cpp b/source/adapters/hip/device.cpp index f8751031d4..c48033ec88 100644 --- a/source/adapters/hip/device.cpp +++ b/source/adapters/hip/device.cpp @@ -1043,6 +1043,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, return ReturnValue(false); case UR_DEVICE_INFO_KERNEL_LAUNCH_CAPABILITIES: return ReturnValue(0); + case UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP: + return ReturnValue(false); default: break; } diff --git a/source/adapters/hip/memory_export.cpp b/source/adapters/hip/memory_export.cpp new file mode 100644 index 0000000000..73c1d597bd --- /dev/null +++ b/source/adapters/hip/memory_export.cpp @@ -0,0 +1,35 @@ +//===--------- memory_export.cpp - HIP Adapter ----------------------------===// +// +// Copyright (C) 2025 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common/logger/ur_logger.hpp" +#include "ur_api.h" + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + size_t /*aligment*/, size_t /*size*/, + ur_exp_external_mem_type_t /*handleTypeToExport*/, void ** /*ppMem*/) { + UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + void * /*pMem*/) { + UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + ur_exp_external_mem_type_t /*handleTypeToExport*/, void * /*pMem*/, + void * /*pMemHandleRet*/) { + UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/source/adapters/hip/ur_interface_loader.cpp b/source/adapters/hip/ur_interface_loader.cpp index 69e9cb27c9..dfb4382cad 100644 --- a/source/adapters/hip/ur_interface_loader.cpp +++ b/source/adapters/hip/ur_interface_loader.cpp @@ -366,6 +366,19 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetBindlessImagesExpProcAddrTable( return UR_RESULT_SUCCESS; } +UR_DLLEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + ur_api_version_t version, ur_memory_export_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + pDdiTable->pfnAllocExportableMemoryExp = + urMemoryExportAllocExportableMemoryExp; + pDdiTable->pfnFreeExportableMemoryExp = urMemoryExportFreeExportableMemoryExp; + pDdiTable->pfnExportMemoryHandleExp = urMemoryExportExportMemoryHandleExp; + return UR_RESULT_SUCCESS; +} + UR_DLLEXPORT ur_result_t UR_APICALL urGetUSMExpProcAddrTable( ur_api_version_t version, ur_usm_exp_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); @@ -474,6 +487,7 @@ UR_DLLEXPORT ur_result_t UR_APICALL urAllAddrTable(ur_api_version_t version, urGetUsmP2PExpProcAddrTable(version, &pDdiTable->UsmP2PExp); urGetVirtualMemProcAddrTable(version, &pDdiTable->VirtualMem); urGetDeviceProcAddrTable(version, &pDdiTable->Device); + urGetMemoryExportExpProcAddrTable(version, &pDdiTable->MemoryExportExp); return UR_RESULT_SUCCESS; } diff --git a/source/adapters/level_zero/CMakeLists.txt b/source/adapters/level_zero/CMakeLists.txt index 8532263dd8..8130322cf3 100644 --- a/source/adapters/level_zero/CMakeLists.txt +++ b/source/adapters/level_zero/CMakeLists.txt @@ -57,6 +57,7 @@ if(UR_BUILD_ADAPTER_L0) ${CMAKE_CURRENT_SOURCE_DIR}/helpers/mutable_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp ${CMAKE_CURRENT_SOURCE_DIR}/enqueued_pool.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory_export.cpp ) install_ur_library(ur_adapter_level_zero) @@ -148,6 +149,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp ${CMAKE_CURRENT_SOURCE_DIR}/sampler.hpp ${CMAKE_CURRENT_SOURCE_DIR}/sampler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory_export.cpp # v2-only sources ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_buffer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_list_cache.hpp diff --git a/source/adapters/level_zero/adapter.cpp b/source/adapters/level_zero/adapter.cpp index 388af44695..362a2479cd 100644 --- a/source/adapters/level_zero/adapter.cpp +++ b/source/adapters/level_zero/adapter.cpp @@ -373,140 +373,138 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() setEnvVar("ZEL_ENABLE_BASIC_LEAK_CHECKER", "1"); } - uint32_t UserForcedSysManInit = 0; - // Check if the user has disabled the default L0 Env initialization. - const int UrSysManEnvInitEnabled = [&UserForcedSysManInit] { - const char *UrRet = std::getenv("UR_L0_ENABLE_SYSMAN_ENV_DEFAULT"); - if (!UrRet) - return 1; - UserForcedSysManInit &= 1; - return std::atoi(UrRet); - }(); - - // Dynamically load the new L0 apis separately. - // This must be done to avoid attempting to use symbols that do - // not exist in older loader runtimes. + uint32_t UserForcedSysManInit = 0; + // Check if the user has disabled the default L0 Env initialization. + const int UrSysManEnvInitEnabled = [&UserForcedSysManInit] { + const char *UrRet = std::getenv("UR_L0_ENABLE_SYSMAN_ENV_DEFAULT"); + if (!UrRet) + return 1; + UserForcedSysManInit &= 1; + return std::atoi(UrRet); + }(); + + // Dynamically load the new L0 apis separately. + // This must be done to avoid attempting to use symbols that do + // not exist in older loader runtimes. #ifndef UR_STATIC_LEVEL_ZERO #ifdef _WIN32 - processHandle = GetModuleHandle(NULL); + processHandle = GetModuleHandle(NULL); #else - processHandle = nullptr; + processHandle = nullptr; #endif #endif - // Setting these environment variables before running zeInit will enable - // the validation layer in the Level Zero loader. - if (UrL0Debug & UR_L0_DEBUG_VALIDATION) { - setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); - setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1"); - } - - if (getenv("SYCL_ENABLE_PCI") != nullptr) { - UR_LOG( - WARN, - "WARNING: SYCL_ENABLE_PCI is deprecated and no longer needed.\n"); - } + // Setting these environment variables before running zeInit will enable + // the validation layer in the Level Zero loader. + if (UrL0Debug & UR_L0_DEBUG_VALIDATION) { + setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); + setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1"); + } - // TODO: We can still safely recover if something goes wrong during the - // init. Implement handling segfault using sigaction. + if (getenv("SYCL_ENABLE_PCI") != nullptr) { + UR_LOG(WARN, + "WARNING: SYCL_ENABLE_PCI is deprecated and no longer needed.\n"); + } - // We must only initialize the driver once, even if urPlatformGet() is - // called multiple times. Declaring the return value as "static" ensures - // it's only called once. + // TODO: We can still safely recover if something goes wrong during the + // init. Implement handling segfault using sigaction. - // Set ZES_ENABLE_SYSMAN by default if the user has not set it. - if (UrSysManEnvInitEnabled) { - setEnvVar("ZES_ENABLE_SYSMAN", "1"); - } + // We must only initialize the driver once, even if urPlatformGet() is + // called multiple times. Declaring the return value as "static" ensures + // it's only called once. - // Init with all flags set to enable for all driver types to be init in - // the application. - ze_init_flags_t L0InitFlags = ZE_INIT_FLAG_GPU_ONLY; - if (UrL0InitAllDrivers) { - L0InitFlags |= ZE_INIT_FLAG_VPU_ONLY; - } - UR_LOG(DEBUG, "\nzeInit with flags value of {}\n", - static_cast(L0InitFlags)); - ZeInitResult = ZE_CALL_NOCHECK(zeInit, (L0InitFlags)); - if (ZeInitResult != ZE_RESULT_SUCCESS) { - const char *ErrorString = "Unknown"; - zeParseError(ZeInitResult, ErrorString); - UR_LOG(ERR, "\nzeInit failed with {}\n", ErrorString); - } + // Set ZES_ENABLE_SYSMAN by default if the user has not set it. + if (UrSysManEnvInitEnabled) { + setEnvVar("ZES_ENABLE_SYSMAN", "1"); + } - bool useInitDrivers = false; - zel_version_t loader_version = {}; - size_t num_components; - auto result = zelLoaderGetVersions(&num_components, nullptr); - if (result == ZE_RESULT_SUCCESS) { - zel_component_version_t *versions = - new zel_component_version_t[num_components]; - result = zelLoaderGetVersions(&num_components, versions); - if (result == ZE_RESULT_SUCCESS) { - for (size_t i = 0; i < num_components; ++i) { - if (strncmp(versions[i].component_name, "loader", - strlen("loader")) == 0) { - loader_version = versions[i].component_lib_version; - UR_LOG(DEBUG, "\nLevel Zero Loader Version: {}.{}.{}\n", - loader_version.major, loader_version.minor, - loader_version.patch); - break; - } - } - } - delete[] versions; - if (loader_version.major > 1 || - (loader_version.major == 1 && loader_version.minor > 19) || - (loader_version.major == 1 && loader_version.minor == 19 && - loader_version.patch >= 2)) { - useInitDrivers = true; - } + // Init with all flags set to enable for all driver types to be init in + // the application. + ze_init_flags_t L0InitFlags = ZE_INIT_FLAG_GPU_ONLY; + if (UrL0InitAllDrivers) { + L0InitFlags |= ZE_INIT_FLAG_VPU_ONLY; + } + UR_LOG(DEBUG, "\nzeInit with flags value of {}\n", + static_cast(L0InitFlags)); + ZeInitResult = ZE_CALL_NOCHECK(zeInit, (L0InitFlags)); + if (ZeInitResult != ZE_RESULT_SUCCESS) { + const char *ErrorString = "Unknown"; + zeParseError(ZeInitResult, ErrorString); + UR_LOG(ERR, "\nzeInit failed with {}\n", ErrorString); + } - if ((loader_version.major == 1 && loader_version.minor < 21) || - (loader_version.major == 1 && loader_version.minor == 21 && - loader_version.patch < 2)) { - UR_LOG( - WARN, - "WARNING: Level Zero Loader version is older than 1.21.2. " - "Please update to the latest version for API logging support.\n"); + bool useInitDrivers = false; + zel_version_t loader_version = {}; + size_t num_components; + auto result = zelLoaderGetVersions(&num_components, nullptr); + if (result == ZE_RESULT_SUCCESS) { + zel_component_version_t *versions = + new zel_component_version_t[num_components]; + result = zelLoaderGetVersions(&num_components, versions); + if (result == ZE_RESULT_SUCCESS) { + for (size_t i = 0; i < num_components; ++i) { + if (strncmp(versions[i].component_name, "loader", strlen("loader")) == + 0) { + loader_version = versions[i].component_lib_version; + UR_LOG(DEBUG, "\nLevel Zero Loader Version: {}.{}.{}\n", + loader_version.major, loader_version.minor, + loader_version.patch); + break; } } + } + delete[] versions; + if (loader_version.major > 1 || + (loader_version.major == 1 && loader_version.minor > 19) || + (loader_version.major == 1 && loader_version.minor == 19 && + loader_version.patch >= 2)) { + useInitDrivers = true; + } + + if ((loader_version.major == 1 && loader_version.minor < 21) || + (loader_version.major == 1 && loader_version.minor == 21 && + loader_version.patch < 2)) { + UR_LOG(WARN, + "WARNING: Level Zero Loader version is older than 1.21.2. " + "Please update to the latest version for API logging support.\n"); + } + } - if (useInitDrivers) { + if (useInitDrivers) { #ifdef UR_STATIC_LEVEL_ZERO - initDriversFunctionPtr = zeInitDrivers; + initDriversFunctionPtr = zeInitDrivers; #else - initDriversFunctionPtr = - (ze_pfnInitDrivers_t)ur_loader::LibLoader::getFunctionPtr( - processHandle, "zeInitDrivers"); + initDriversFunctionPtr = + (ze_pfnInitDrivers_t)ur_loader::LibLoader::getFunctionPtr( + processHandle, "zeInitDrivers"); #endif - if (initDriversFunctionPtr) { - UR_LOG(DEBUG, "\nzeInitDrivers with flags value of {}\n", - static_cast(InitDriversDesc.flags)); - ZeInitDriversResult = - ZE_CALL_NOCHECK(initDriversFunctionPtr, - (&ZeInitDriversCount, nullptr, &InitDriversDesc)); - if (ZeInitDriversResult == ZE_RESULT_SUCCESS) { - InitDriversSupported = true; - } else { - const char *ErrorString = "Unknown"; - zeParseError(ZeInitDriversResult, ErrorString); - UR_LOG(ERR, "\nzeInitDrivers failed with {}\n", ErrorString); - } - } + if (initDriversFunctionPtr) { + UR_LOG(DEBUG, "\nzeInitDrivers with flags value of {}\n", + static_cast(InitDriversDesc.flags)); + ZeInitDriversResult = + ZE_CALL_NOCHECK(initDriversFunctionPtr, + (&ZeInitDriversCount, nullptr, &InitDriversDesc)); + if (ZeInitDriversResult == ZE_RESULT_SUCCESS) { + InitDriversSupported = true; + } else { + const char *ErrorString = "Unknown"; + zeParseError(ZeInitDriversResult, ErrorString); + UR_LOG(ERR, "\nzeInitDrivers failed with {}\n", ErrorString); } + } + } - if (ZeInitResult != ZE_RESULT_SUCCESS && - ZeInitDriversResult != ZE_RESULT_SUCCESS) { - // Absorb the ZE_RESULT_ERROR_UNINITIALIZED and just return 0 Platforms. - UR_LOG(ERR, "Level Zero Uninitialized\n"); - return; - } + if (ZeInitResult != ZE_RESULT_SUCCESS && + ZeInitDriversResult != ZE_RESULT_SUCCESS) { + // Absorb the ZE_RESULT_ERROR_UNINITIALIZED and just return 0 Platforms. + UR_LOG(ERR, "Level Zero Uninitialized\n"); + return; + } - PlatformVec platforms; + PlatformVec platforms; - bool forceLoadedAdapter = ur_getenv("UR_ADAPTERS_FORCE_LOAD").has_value(); - if (!forceLoadedAdapter) { + bool forceLoadedAdapter = ur_getenv("UR_ADAPTERS_FORCE_LOAD").has_value(); + if (!forceLoadedAdapter) { #ifdef UR_ADAPTER_LEVEL_ZERO_V2 auto [useV2, reason] = shouldUseV2Adapter(); if (!useV2) { @@ -520,7 +518,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() return; } #endif - } + } // Check if the user has enabled the default L0 SysMan initialization. const int UrSysmanZesinitEnable = [&UserForcedSysManInit] { diff --git a/source/adapters/level_zero/device.cpp b/source/adapters/level_zero/device.cpp index 167c7cb856..32c3ea19b2 100644 --- a/source/adapters/level_zero/device.cpp +++ b/source/adapters/level_zero/device.cpp @@ -1381,6 +1381,50 @@ ur_result_t urDeviceGetInfo( } case UR_DEVICE_INFO_KERNEL_LAUNCH_CAPABILITIES: return ReturnValue(UR_KERNEL_LAUNCH_PROPERTIES_FLAG_COOPERATIVE); + case UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP: + return ReturnValue(true); + case UR_DEVICE_INFO_LUID: { + // LUID is only available on Windows. + // Intel extension for device LUID. This returns the LUID as + // std::array. For details about this extension, + // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. + if (Device->Platform->ZeLUIDSupported) { + ze_device_properties_t DeviceProp = {}; + DeviceProp.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + ze_device_luid_ext_properties_t LuidDesc = {}; + LuidDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_LUID_EXT_PROPERTIES; + DeviceProp.pNext = (void *)&LuidDesc; + + ZE2UR_CALL(zeDeviceGetProperties, (ZeDevice, &DeviceProp)); + + const auto &LUID = LuidDesc.luid.id; + return ReturnValue(LUID, sizeof(LUID)); + } else { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + } + case UR_DEVICE_INFO_NODE_MASK: { + // Device node mask is only available on Windows. + // Intel extension for device node mask. This returns the node mask as + // uint32_t. For details about this extension, + // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. + + // Node mask is provided through the L0 LUID extension so support for this + // extension must be checked. + if (Device->Platform->ZeLUIDSupported) { + ze_device_properties_t DeviceProp = {}; + DeviceProp.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES; + ze_device_luid_ext_properties_t LuidDesc = {}; + LuidDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_LUID_EXT_PROPERTIES; + DeviceProp.pNext = (void *)&LuidDesc; + + ZE2UR_CALL(zeDeviceGetProperties, (ZeDevice, &DeviceProp)); + + return ReturnValue(LuidDesc.nodeMask); + } else { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + } default: UR_LOG(ERR, "Unsupported ParamName in urGetDeviceInfo"); UR_LOG(ERR, "ParamNameParamName={}(0x{})", ParamName, diff --git a/source/adapters/level_zero/memory_export.cpp b/source/adapters/level_zero/memory_export.cpp new file mode 100644 index 0000000000..d5c9d91f50 --- /dev/null +++ b/source/adapters/level_zero/memory_export.cpp @@ -0,0 +1,100 @@ +//===--------- memory_export.cpp - Level Zero Adapter ---------------------===// +// +// Copyright (C) 2025 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common.hpp" +#ifdef UR_ADAPTER_LEVEL_ZERO_V2 +#include "v2/context.hpp" +#else +#include "context.hpp" +#endif +#include "ur_api.h" + +namespace ur::level_zero { + +ur_result_t urMemoryExportAllocExportableMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t alignment, + size_t size, ur_exp_external_mem_type_t handleTypeToExport, void **ppMem) { + + UR_ASSERT(handleTypeToExport == UR_EXP_EXTERNAL_MEM_TYPE_OPAQUE_FD || + handleTypeToExport == UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT, + UR_RESULT_ERROR_INVALID_ENUMERATION); + + ze_external_memory_export_desc_t MemExportDesc{}; + MemExportDesc.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_DESC; + + switch (handleTypeToExport) { + case UR_EXP_EXTERNAL_MEM_TYPE_OPAQUE_FD: + MemExportDesc.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_FD; + break; + case UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT: + MemExportDesc.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_WIN32; + break; + default: + return UR_RESULT_ERROR_INVALID_ENUMERATION; + } + + ze_device_mem_alloc_desc_t MemAllocDesc{}; + MemAllocDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + MemAllocDesc.pNext = &MemExportDesc; + + ZE2UR_CALL(zeMemAllocDevice, (hContext->getZeHandle(), &MemAllocDesc, size, + alignment, hDevice->ZeDevice, ppMem)); + + return UR_RESULT_SUCCESS; +} + +ur_result_t urMemoryExportFreeExportableMemoryExp( + ur_context_handle_t hContext, [[maybe_unused]] ur_device_handle_t hDevice, + void *pMem) { + ZE2UR_CALL(zeMemFree, (hContext->getZeHandle(), pMem)); + return UR_RESULT_SUCCESS; +} + +ur_result_t urMemoryExportExportMemoryHandleExp( + ur_context_handle_t hContext, [[maybe_unused]] ur_device_handle_t hDevice, + ur_exp_external_mem_type_t handleTypeToExport, void *pMem, + void *pMemHandleRet) { + + ze_memory_allocation_properties_t MemAllocProps{}; + MemAllocProps.stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES; + + ze_external_memory_export_fd_t MemExportFD{}; + ze_external_memory_export_win32_handle_t MemExportWin32{}; + + switch (handleTypeToExport) { + case UR_EXP_EXTERNAL_MEM_TYPE_OPAQUE_FD: { + MemExportFD.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_FD; + MemExportFD.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_FD; + MemAllocProps.pNext = &MemExportFD; + ZE2UR_CALL(zeMemGetAllocProperties, + (hContext->getZeHandle(), pMem, &MemAllocProps, nullptr)); + int *pMemHandleRetIntPtr = static_cast(pMemHandleRet); + *pMemHandleRetIntPtr = MemExportFD.fd; + break; + } + case UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT: { + MemExportWin32.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_WIN32; + MemExportWin32.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_WIN32; + MemAllocProps.pNext = &MemExportWin32; + ZE2UR_CALL(zeMemGetAllocProperties, + (hContext->getZeHandle(), pMem, &MemAllocProps, nullptr)); + void **ppMemHandleRet = static_cast(&pMemHandleRet); + *ppMemHandleRet = MemExportWin32.handle; + break; + } + default: { + return UR_RESULT_ERROR_INVALID_ENUMERATION; + } + } + + return UR_RESULT_SUCCESS; +} + +} // namespace ur::level_zero diff --git a/source/adapters/level_zero/platform.cpp b/source/adapters/level_zero/platform.cpp index 47f2e519bb..32b89a6a72 100644 --- a/source/adapters/level_zero/platform.cpp +++ b/source/adapters/level_zero/platform.cpp @@ -32,11 +32,11 @@ ur_result_t urPlatformGet( if (NumPlatforms) { *NumPlatforms = nplatforms; } - if (Platforms) { - for (uint32_t i = 0; i < std::min(nplatforms, NumEntries); ++i) { - Platforms[i] = GlobalAdapter->Platforms.at(i).get(); - } + if (Platforms) { + for (uint32_t i = 0; i < std::min(nplatforms, NumEntries); ++i) { + Platforms[i] = GlobalAdapter->Platforms.at(i).get(); } + } return UR_RESULT_SUCCESS; } @@ -295,6 +295,12 @@ ur_result_t ur_platform_handle_t_::initialize() { ZeBindlessImagesExtensionSupported = true; } } + if (strncmp(extension.name, ZE_DEVICE_LUID_EXT_NAME, + strlen(ZE_DEVICE_LUID_EXT_NAME) + 1) == 0) { + if (extension.version == ZE_DEVICE_LUID_EXT_VERSION_1_0) { + ZeLUIDSupported = true; + } + } zeDriverExtensionMap[extension.name] = extension.version; } diff --git a/source/adapters/level_zero/platform.hpp b/source/adapters/level_zero/platform.hpp index 192cd063c1..5f767120ce 100644 --- a/source/adapters/level_zero/platform.hpp +++ b/source/adapters/level_zero/platform.hpp @@ -69,6 +69,7 @@ struct ur_platform_handle_t_ : ur::handle_base, bool ZeDriverEuCountExtensionFound{false}; bool ZeCopyOffloadExtensionSupported{false}; bool ZeBindlessImagesExtensionSupported{false}; + bool ZeLUIDSupported{false}; // Cache UR devices for reuse std::vector> URDevicesCache; diff --git a/source/adapters/level_zero/ur_interface_loader.cpp b/source/adapters/level_zero/ur_interface_loader.cpp index b1cb5218ce..13d7274e7a 100644 --- a/source/adapters/level_zero/ur_interface_loader.cpp +++ b/source/adapters/level_zero/ur_interface_loader.cpp @@ -312,6 +312,23 @@ urGetMemProcAddrTable(ur_api_version_t version, ur_mem_dditable_t *pDdiTable) { return result; } +UR_APIEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + ur_api_version_t version, ur_memory_export_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + + pDdiTable->pfnAllocExportableMemoryExp = + ur::level_zero::urMemoryExportAllocExportableMemoryExp; + pDdiTable->pfnFreeExportableMemoryExp = + ur::level_zero::urMemoryExportFreeExportableMemoryExp; + pDdiTable->pfnExportMemoryHandleExp = + ur::level_zero::urMemoryExportExportMemoryHandleExp; + + return result; +} + UR_APIEXPORT ur_result_t UR_APICALL urGetPhysicalMemProcAddrTable( ur_api_version_t version, ur_physical_mem_dditable_t *pDdiTable) { auto result = validateProcInputs(version, pDdiTable); @@ -583,6 +600,10 @@ ur_result_t populateDdiTable(ur_dditable_t *ddi) { if (result != UR_RESULT_SUCCESS) return result; result = NAMESPACE_::urGetMemProcAddrTable(UR_API_VERSION_CURRENT, &ddi->Mem); + if (result != UR_RESULT_SUCCESS) + return result; + result = NAMESPACE_::urGetMemoryExportExpProcAddrTable(UR_API_VERSION_CURRENT, + &ddi->MemoryExportExp); if (result != UR_RESULT_SUCCESS) return result; result = NAMESPACE_::urGetPhysicalMemProcAddrTable(UR_API_VERSION_CURRENT, diff --git a/source/adapters/level_zero/ur_interface_loader.hpp b/source/adapters/level_zero/ur_interface_loader.hpp index bbbe1fce96..df8e93c1f7 100644 --- a/source/adapters/level_zero/ur_interface_loader.hpp +++ b/source/adapters/level_zero/ur_interface_loader.hpp @@ -768,6 +768,16 @@ urCommandBufferGetNativeHandleExp(ur_exp_command_buffer_handle_t hCommandBuffer, ur_result_t urEnqueueTimestampRecordingExp( ur_queue_handle_t hQueue, bool blocking, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); +ur_result_t urMemoryExportAllocExportableMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t alignment, + size_t size, ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); +ur_result_t urMemoryExportFreeExportableMemoryExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + void *pMem); +ur_result_t urMemoryExportExportMemoryHandleExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_external_mem_type_t handleTypeToExport, void *pMem, + void *pMemHandleRet); ur_result_t urProgramBuildExp(ur_program_handle_t hProgram, uint32_t numDevices, ur_device_handle_t *phDevices, const char *pOptions); diff --git a/source/adapters/mock/ur_mockddi.cpp b/source/adapters/mock/ur_mockddi.cpp index 3ab79444f3..39d67fff43 100644 --- a/source/adapters/mock/ur_mockddi.cpp +++ b/source/adapters/mock/ur_mockddi.cpp @@ -11304,6 +11304,168 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( return exceptionToResult(std::current_exception()); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. + size_t size, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_memory_export_alloc_exportable_memory_exp_params_t params = { + &hContext, &hDevice, &alignment, &size, &handleTypeToExport, &ppMem}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback( + "urMemoryExportAllocExportableMemoryExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback( + "urMemoryExportAllocExportableMemoryExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback( + "urMemoryExportAllocExportableMemoryExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_memory_export_free_exportable_memory_exp_params_t params = { + &hContext, &hDevice, &pMem}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback( + "urMemoryExportFreeExportableMemoryExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback( + "urMemoryExportFreeExportableMemoryExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback( + "urMemoryExportFreeExportableMemoryExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportExportMemoryHandleExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_memory_export_export_memory_handle_exp_params_t params = { + &hContext, &hDevice, &handleTypeToExport, &pMem, &pMemHandleRet}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback( + "urMemoryExportExportMemoryHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback( + "urMemoryExportExportMemoryHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback( + "urMemoryExportExportMemoryHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urProgramBuildExp __urdlllocal ur_result_t UR_APICALL urProgramBuildExp( @@ -12430,6 +12592,41 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetMemProcAddrTable( return exceptionToResult(std::current_exception()); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's MemoryExportExp table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +UR_DLLEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + /// [in] API version requested + ur_api_version_t version, + /// [in,out] pointer to table of DDI function pointers + ur_memory_export_exp_dditable_t *pDdiTable) try { + if (nullptr == pDdiTable) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (driver::d_context.version < version) + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + + ur_result_t result = UR_RESULT_SUCCESS; + + pDdiTable->pfnAllocExportableMemoryExp = + driver::urMemoryExportAllocExportableMemoryExp; + + pDdiTable->pfnFreeExportableMemoryExp = + driver::urMemoryExportFreeExportableMemoryExp; + + pDdiTable->pfnExportMemoryHandleExp = + driver::urMemoryExportExportMemoryHandleExp; + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Exported function for filling application's PhysicalMem table /// with current process' addresses diff --git a/source/adapters/native_cpu/CMakeLists.txt b/source/adapters/native_cpu/CMakeLists.txt index c197d544fd..a5af02213f 100644 --- a/source/adapters/native_cpu/CMakeLists.txt +++ b/source/adapters/native_cpu/CMakeLists.txt @@ -24,6 +24,7 @@ add_ur_adapter(${TARGET_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/image.cpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory_export.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp diff --git a/source/adapters/native_cpu/memory_export.cpp b/source/adapters/native_cpu/memory_export.cpp new file mode 100644 index 0000000000..95cce666f9 --- /dev/null +++ b/source/adapters/native_cpu/memory_export.cpp @@ -0,0 +1,32 @@ +//===--------- memory_export.cpp - Native CPU Adapter ---------------------===// +// +// Copyright (C) 2025 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common.hpp" +#include "ur_api.h" + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + size_t /*aligment*/, size_t /*size*/, + ur_exp_external_mem_type_t /*handleTypeToExport*/, void ** /*ppMem*/) { + DIE_NO_IMPLEMENTATION; +} + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + void * /*pMem*/) { + DIE_NO_IMPLEMENTATION; +} + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + ur_exp_external_mem_type_t /*handleTypeToExport*/, void * /*pMem*/, + void * /*pMemHandleRet*/) { + DIE_NO_IMPLEMENTATION; +} diff --git a/source/adapters/native_cpu/ur_interface_loader.cpp b/source/adapters/native_cpu/ur_interface_loader.cpp index 56ea0c76cc..3f6fe061b4 100644 --- a/source/adapters/native_cpu/ur_interface_loader.cpp +++ b/source/adapters/native_cpu/ur_interface_loader.cpp @@ -361,6 +361,19 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetBindlessImagesExpProcAddrTable( return UR_RESULT_SUCCESS; } +UR_DLLEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + ur_api_version_t version, ur_memory_export_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + pDdiTable->pfnAllocExportableMemoryExp = + urMemoryExportAllocExportableMemoryExp; + pDdiTable->pfnFreeExportableMemoryExp = urMemoryExportFreeExportableMemoryExp; + pDdiTable->pfnExportMemoryHandleExp = urMemoryExportExportMemoryHandleExp; + return UR_RESULT_SUCCESS; +} + UR_DLLEXPORT ur_result_t UR_APICALL urGetPhysicalMemProcAddrTable( ur_api_version_t version, ur_physical_mem_dditable_t *pDdiTable) { auto retVal = validateProcInputs(version, pDdiTable); @@ -458,6 +471,7 @@ UR_DLLEXPORT ur_result_t UR_APICALL urAllAddrTable(ur_api_version_t version, urGetUsmP2PExpProcAddrTable(version, &pDdiTable->UsmP2PExp); urGetVirtualMemProcAddrTable(version, &pDdiTable->VirtualMem); urGetDeviceProcAddrTable(version, &pDdiTable->Device); + urGetMemoryExportExpProcAddrTable(version, &pDdiTable->MemoryExportExp); return UR_RESULT_SUCCESS; } diff --git a/source/adapters/opencl/CMakeLists.txt b/source/adapters/opencl/CMakeLists.txt index 415a85d5b7..fc2a4b7862 100644 --- a/source/adapters/opencl/CMakeLists.txt +++ b/source/adapters/opencl/CMakeLists.txt @@ -30,6 +30,7 @@ add_ur_adapter(${TARGET_NAME} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/image.cpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/memory_export.cpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp diff --git a/source/adapters/opencl/device.cpp b/source/adapters/opencl/device.cpp index d0ab2376f5..9c9c82ea47 100644 --- a/source/adapters/opencl/device.cpp +++ b/source/adapters/opencl/device.cpp @@ -1418,6 +1418,59 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, return ReturnValue(true); case UR_DEVICE_INFO_KERNEL_LAUNCH_CAPABILITIES: return ReturnValue(0); + case UR_DEVICE_INFO_LUID: { + // LUID is only available on Windows. + // Intel extension for device LUID. This returns the LUID as + // std::array. For details about this extension, + // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. + + // Use the cl_khr_device_uuid extension, if available. + bool isKhrDeviceLuidSupported = false; + if (hDevice->checkDeviceExtensions({"cl_khr_device_uuid"}, + isKhrDeviceLuidSupported) != + UR_RESULT_SUCCESS || + !isKhrDeviceLuidSupported) { + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } + + cl_bool isLuidValid; + CL_RETURN_ON_FAILURE( + clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_LUID_VALID_KHR, + sizeof(cl_bool), &isLuidValid, nullptr)); + + if (!isLuidValid) { + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } + + static_assert(CL_LUID_SIZE_KHR == 8); + std::array UUID{}; + CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, CL_DEVICE_LUID_KHR, + UUID.size(), UUID.data(), nullptr)); + return ReturnValue(UUID); + } + case UR_DEVICE_INFO_NODE_MASK: { + // Device node mask is only available on Windows. + // Intel extension for device node mask. This returns the node mask as + // uint32_t. For details about this extension, + // see sycl/doc/extensions/supported/sycl_ext_intel_device_info.md. + + // Use the cl_khr_device_uuid extension, if available. + bool isKhrDeviceLuidSupported = false; + if (hDevice->checkDeviceExtensions({"cl_khr_device_uuid"}, + isKhrDeviceLuidSupported) != + UR_RESULT_SUCCESS || + !isKhrDeviceLuidSupported) { + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } + + cl_int nodeMask = 0; + + CL_RETURN_ON_FAILURE(clGetDeviceInfo(hDevice->CLDevice, + CL_DEVICE_NODE_MASK_KHR, + sizeof(cl_int), &nodeMask, nullptr)); + + return ReturnValue(nodeMask); + } // TODO: We can't query to check if these are supported, they will need to be // manually updated if support is ever implemented. case UR_DEVICE_INFO_KERNEL_SET_SPECIALIZATION_CONSTANTS: @@ -1448,6 +1501,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, case UR_DEVICE_INFO_BINDLESS_SAMPLE_2D_USM_SUPPORT_EXP: case UR_DEVICE_INFO_BINDLESS_IMAGES_GATHER_SUPPORT_EXP: case UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP: + case UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP: return ReturnValue(false); case UR_DEVICE_INFO_IMAGE_PITCH_ALIGN_EXP: case UR_DEVICE_INFO_MAX_IMAGE_LINEAR_WIDTH_EXP: diff --git a/source/adapters/opencl/memory_export.cpp b/source/adapters/opencl/memory_export.cpp new file mode 100644 index 0000000000..faeaa42372 --- /dev/null +++ b/source/adapters/opencl/memory_export.cpp @@ -0,0 +1,35 @@ +//===--------- memory_export.cpp - OpenCL Adapter -------------------------===// +// +// Copyright (C) 2025 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common/logger/ur_logger.hpp" +#include "ur_api.h" + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + size_t /*aligment*/, size_t /*size*/, + ur_exp_external_mem_type_t /*handleTypeToExport*/, void ** /*ppMem*/) { + UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + void * /*pMem*/) { + UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + ur_exp_external_mem_type_t /*handleTypeToExport*/, void * /*pMem*/, + void * /*pMemHandleRet*/) { + UR_LOG(ERR, "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/source/adapters/opencl/ur_interface_loader.cpp b/source/adapters/opencl/ur_interface_loader.cpp index 05117dd829..c619fa36b1 100644 --- a/source/adapters/opencl/ur_interface_loader.cpp +++ b/source/adapters/opencl/ur_interface_loader.cpp @@ -378,6 +378,19 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetBindlessImagesExpProcAddrTable( return UR_RESULT_SUCCESS; } +UR_DLLEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + ur_api_version_t version, ur_memory_export_exp_dditable_t *pDdiTable) { + auto result = validateProcInputs(version, pDdiTable); + if (UR_RESULT_SUCCESS != result) { + return result; + } + pDdiTable->pfnAllocExportableMemoryExp = + urMemoryExportAllocExportableMemoryExp; + pDdiTable->pfnFreeExportableMemoryExp = urMemoryExportFreeExportableMemoryExp; + pDdiTable->pfnExportMemoryHandleExp = urMemoryExportExportMemoryHandleExp; + return UR_RESULT_SUCCESS; +} + UR_DLLEXPORT ur_result_t UR_APICALL urGetVirtualMemProcAddrTable( ur_api_version_t version, ur_virtual_mem_dditable_t *pDdiTable) { auto retVal = validateProcInputs(version, pDdiTable); @@ -461,6 +474,7 @@ UR_DLLEXPORT ur_result_t UR_APICALL urAllAddrTable(ur_api_version_t version, urGetUsmP2PExpProcAddrTable(version, &pDdiTable->UsmP2PExp); urGetVirtualMemProcAddrTable(version, &pDdiTable->VirtualMem); urGetDeviceProcAddrTable(version, &pDdiTable->Device); + urGetMemoryExportExpProcAddrTable(version, &pDdiTable->MemoryExportExp); return UR_RESULT_SUCCESS; } diff --git a/source/common/stype_map_helpers.def b/source/common/stype_map_helpers.def index db72221147..7970582639 100644 --- a/source/common/stype_map_helpers.def +++ b/source/common/stype_map_helpers.def @@ -92,9 +92,6 @@ template <> struct stype_map : stype_map_impl {}; template <> -struct stype_map - : stype_map_impl {}; -template <> struct stype_map : stype_map_impl {}; template <> @@ -116,6 +113,9 @@ template <> struct stype_map : stype_map_impl {}; template <> +struct stype_map + : stype_map_impl {}; +template <> struct stype_map : stype_map_impl {}; template <> diff --git a/source/loader/layers/tracing/ur_trcddi.cpp b/source/loader/layers/tracing/ur_trcddi.cpp index 0abbb7604c..e96e1cbffd 100644 --- a/source/loader/layers/tracing/ur_trcddi.cpp +++ b/source/loader/layers/tracing/ur_trcddi.cpp @@ -9574,6 +9574,148 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. + size_t size, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) { + auto pfnAllocExportableMemoryExp = + getContext()->urDdiTable.MemoryExportExp.pfnAllocExportableMemoryExp; + + if (nullptr == pfnAllocExportableMemoryExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_memory_export_alloc_exportable_memory_exp_params_t params = { + &hContext, &hDevice, &alignment, &size, &handleTypeToExport, &ppMem}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP, + "urMemoryExportAllocExportableMemoryExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urMemoryExportAllocExportableMemoryExp\n"); + + ur_result_t result = pfnAllocExportableMemoryExp( + hContext, hDevice, alignment, size, handleTypeToExport, ppMem); + + getContext()->notify_end( + UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP, + "urMemoryExportAllocExportableMemoryExp", ¶ms, &result, instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP, + ¶ms); + UR_LOG_L(logger, INFO, + " <--- urMemoryExportAllocExportableMemoryExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) { + auto pfnFreeExportableMemoryExp = + getContext()->urDdiTable.MemoryExportExp.pfnFreeExportableMemoryExp; + + if (nullptr == pfnFreeExportableMemoryExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_memory_export_free_exportable_memory_exp_params_t params = { + &hContext, &hDevice, &pMem}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP, + "urMemoryExportFreeExportableMemoryExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urMemoryExportFreeExportableMemoryExp\n"); + + ur_result_t result = pfnFreeExportableMemoryExp(hContext, hDevice, pMem); + + getContext()->notify_end(UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP, + "urMemoryExportFreeExportableMemoryExp", ¶ms, + &result, instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP, + ¶ms); + UR_LOG_L(logger, INFO, + " <--- urMemoryExportFreeExportableMemoryExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportExportMemoryHandleExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) { + auto pfnExportMemoryHandleExp = + getContext()->urDdiTable.MemoryExportExp.pfnExportMemoryHandleExp; + + if (nullptr == pfnExportMemoryHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_memory_export_export_memory_handle_exp_params_t params = { + &hContext, &hDevice, &handleTypeToExport, &pMem, &pMemHandleRet}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP, + "urMemoryExportExportMemoryHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urMemoryExportExportMemoryHandleExp\n"); + + ur_result_t result = pfnExportMemoryHandleExp( + hContext, hDevice, handleTypeToExport, pMem, pMemHandleRet); + + getContext()->notify_end(UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP, + "urMemoryExportExportMemoryHandleExp", ¶ms, + &result, instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, + " <--- urMemoryExportExportMemoryHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urProgramBuildExp __urdlllocal ur_result_t UR_APICALL urProgramBuildExp( @@ -10789,6 +10931,46 @@ __urdlllocal ur_result_t UR_APICALL urGetMemProcAddrTable( return result; } /////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's MemoryExportExp table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +__urdlllocal ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + /// [in] API version requested + ur_api_version_t version, + /// [in,out] pointer to table of DDI function pointers + ur_memory_export_exp_dditable_t *pDdiTable) { + auto &dditable = ur_tracing_layer::getContext()->urDdiTable.MemoryExportExp; + + if (nullptr == pDdiTable) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (UR_MAJOR_VERSION(ur_tracing_layer::getContext()->version) != + UR_MAJOR_VERSION(version) || + UR_MINOR_VERSION(ur_tracing_layer::getContext()->version) > + UR_MINOR_VERSION(version)) + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + + ur_result_t result = UR_RESULT_SUCCESS; + + dditable.pfnAllocExportableMemoryExp = pDdiTable->pfnAllocExportableMemoryExp; + pDdiTable->pfnAllocExportableMemoryExp = + ur_tracing_layer::urMemoryExportAllocExportableMemoryExp; + + dditable.pfnFreeExportableMemoryExp = pDdiTable->pfnFreeExportableMemoryExp; + pDdiTable->pfnFreeExportableMemoryExp = + ur_tracing_layer::urMemoryExportFreeExportableMemoryExp; + + dditable.pfnExportMemoryHandleExp = pDdiTable->pfnExportMemoryHandleExp; + pDdiTable->pfnExportMemoryHandleExp = + ur_tracing_layer::urMemoryExportExportMemoryHandleExp; + + return result; +} +/////////////////////////////////////////////////////////////////////////////// /// @brief Exported function for filling application's PhysicalMem table /// with current process' addresses /// @@ -11418,6 +11600,11 @@ ur_result_t context_t::init(ur_dditable_t *dditable, &dditable->Mem); } + if (UR_RESULT_SUCCESS == result) { + result = ur_tracing_layer::urGetMemoryExportExpProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->MemoryExportExp); + } + if (UR_RESULT_SUCCESS == result) { result = ur_tracing_layer::urGetPhysicalMemProcAddrTable( UR_API_VERSION_CURRENT, &dditable->PhysicalMem); diff --git a/source/loader/layers/validation/ur_valddi.cpp b/source/loader/layers/validation/ur_valddi.cpp index b61356afd2..6f33aaa856 100644 --- a/source/loader/layers/validation/ur_valddi.cpp +++ b/source/loader/layers/validation/ur_valddi.cpp @@ -569,7 +569,7 @@ __urdlllocal ur_result_t UR_APICALL urDeviceGetInfo( if (NULL == hDevice) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP < propName) + if (UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP < propName) return UR_RESULT_ERROR_INVALID_ENUMERATION; if (propSize == 0 && pPropValue != NULL) @@ -10337,6 +10337,167 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. + size_t size, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) { + auto pfnAllocExportableMemoryExp = + getContext()->urDdiTable.MemoryExportExp.pfnAllocExportableMemoryExp; + + if (nullptr == pfnAllocExportableMemoryExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (ppMem == nullptr) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if ((hDevice == nullptr) || (hContext == nullptr)) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < handleTypeToExport) + return UR_RESULT_ERROR_INVALID_ENUMERATION; + + if (alignment != 0 && ((alignment & (alignment - 1)) != 0)) + return UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT; + + if (size == 0) + return UR_RESULT_ERROR_INVALID_USM_SIZE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); + } + + ur_result_t result = pfnAllocExportableMemoryExp( + hContext, hDevice, alignment, size, handleTypeToExport, ppMem); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) { + auto pfnFreeExportableMemoryExp = + getContext()->urDdiTable.MemoryExportExp.pfnFreeExportableMemoryExp; + + if (nullptr == pfnFreeExportableMemoryExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (pMem == nullptr) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if ((hDevice == nullptr) || (hContext == nullptr)) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); + } + + ur_result_t result = pfnFreeExportableMemoryExp(hContext, hDevice, pMem); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportExportMemoryHandleExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) { + auto pfnExportMemoryHandleExp = + getContext()->urDdiTable.MemoryExportExp.pfnExportMemoryHandleExp; + + if (nullptr == pfnExportMemoryHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (pMemHandleRet == nullptr || pMem == nullptr) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if ((hDevice == nullptr) || (hContext == nullptr)) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < handleTypeToExport) + return UR_RESULT_ERROR_INVALID_ENUMERATION; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); + } + + ur_result_t result = pfnExportMemoryHandleExp( + hContext, hDevice, handleTypeToExport, pMem, pMemHandleRet); + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urProgramBuildExp __urdlllocal ur_result_t UR_APICALL urProgramBuildExp( @@ -11574,6 +11735,48 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetMemProcAddrTable( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's MemoryExportExp table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +UR_DLLEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + /// [in] API version requested + ur_api_version_t version, + /// [in,out] pointer to table of DDI function pointers + ur_memory_export_exp_dditable_t *pDdiTable) { + auto &dditable = + ur_validation_layer::getContext()->urDdiTable.MemoryExportExp; + + if (nullptr == pDdiTable) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (UR_MAJOR_VERSION(ur_validation_layer::getContext()->version) != + UR_MAJOR_VERSION(version) || + UR_MINOR_VERSION(ur_validation_layer::getContext()->version) > + UR_MINOR_VERSION(version)) + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + + ur_result_t result = UR_RESULT_SUCCESS; + + dditable.pfnAllocExportableMemoryExp = pDdiTable->pfnAllocExportableMemoryExp; + pDdiTable->pfnAllocExportableMemoryExp = + ur_validation_layer::urMemoryExportAllocExportableMemoryExp; + + dditable.pfnFreeExportableMemoryExp = pDdiTable->pfnFreeExportableMemoryExp; + pDdiTable->pfnFreeExportableMemoryExp = + ur_validation_layer::urMemoryExportFreeExportableMemoryExp; + + dditable.pfnExportMemoryHandleExp = pDdiTable->pfnExportMemoryHandleExp; + pDdiTable->pfnExportMemoryHandleExp = + ur_validation_layer::urMemoryExportExportMemoryHandleExp; + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Exported function for filling application's PhysicalMem table /// with current process' addresses @@ -12234,6 +12437,11 @@ ur_result_t context_t::init(ur_dditable_t *dditable, &dditable->Mem); } + if (UR_RESULT_SUCCESS == result) { + result = ur_validation_layer::urGetMemoryExportExpProcAddrTable( + UR_API_VERSION_CURRENT, &dditable->MemoryExportExp); + } + if (UR_RESULT_SUCCESS == result) { result = ur_validation_layer::urGetPhysicalMemProcAddrTable( UR_API_VERSION_CURRENT, &dditable->PhysicalMem); diff --git a/source/loader/loader.def.in b/source/loader/loader.def.in index 48cbcf0606..3ad4714931 100644 --- a/source/loader/loader.def.in +++ b/source/loader/loader.def.in @@ -120,6 +120,7 @@ EXPORTS urGetEventProcAddrTable urGetKernelProcAddrTable urGetMemProcAddrTable + urGetMemoryExportExpProcAddrTable urGetPhysicalMemProcAddrTable urGetPlatformProcAddrTable urGetProgramExpProcAddrTable @@ -166,6 +167,9 @@ EXPORTS urMemImageGetInfo urMemRelease urMemRetain + urMemoryExportAllocExportableMemoryExp + urMemoryExportExportMemoryHandleExp + urMemoryExportFreeExportableMemoryExp urPhysicalMemCreate urPhysicalMemGetInfo urPhysicalMemRelease @@ -415,6 +419,9 @@ EXPORTS urPrintMemReleaseParams urPrintMemRetainParams urPrintMemType + urPrintMemoryExportAllocExportableMemoryExpParams + urPrintMemoryExportExportMemoryHandleExpParams + urPrintMemoryExportFreeExportableMemoryExpParams urPrintMemoryOrderCapabilityFlags urPrintMemoryScopeCapabilityFlags urPrintPhysicalMemCreateParams diff --git a/source/loader/loader.map.in b/source/loader/loader.map.in index 177748b142..fde803f9aa 100644 --- a/source/loader/loader.map.in +++ b/source/loader/loader.map.in @@ -120,6 +120,7 @@ urGetEventProcAddrTable; urGetKernelProcAddrTable; urGetMemProcAddrTable; + urGetMemoryExportExpProcAddrTable; urGetPhysicalMemProcAddrTable; urGetPlatformProcAddrTable; urGetProgramExpProcAddrTable; @@ -166,6 +167,9 @@ urMemImageGetInfo; urMemRelease; urMemRetain; + urMemoryExportAllocExportableMemoryExp; + urMemoryExportExportMemoryHandleExp; + urMemoryExportFreeExportableMemoryExp; urPhysicalMemCreate; urPhysicalMemGetInfo; urPhysicalMemRelease; @@ -415,6 +419,9 @@ urPrintMemReleaseParams; urPrintMemRetainParams; urPrintMemType; + urPrintMemoryExportAllocExportableMemoryExpParams; + urPrintMemoryExportExportMemoryHandleExpParams; + urPrintMemoryExportFreeExportableMemoryExpParams; urPrintMemoryOrderCapabilityFlags; urPrintMemoryScopeCapabilityFlags; urPrintPhysicalMemCreateParams; diff --git a/source/loader/ur_ldrddi.cpp b/source/loader/ur_ldrddi.cpp index 74712c5c4d..5c2c3a41af 100644 --- a/source/loader/ur_ldrddi.cpp +++ b/source/loader/ur_ldrddi.cpp @@ -5452,6 +5452,83 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( phEventWaitList, phEvent); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. + size_t size, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnAllocExportableMemoryExp = + dditable->MemoryExportExp.pfnAllocExportableMemoryExp; + if (nullptr == pfnAllocExportableMemoryExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnAllocExportableMemoryExp(hContext, hDevice, alignment, size, + handleTypeToExport, ppMem); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnFreeExportableMemoryExp = + dditable->MemoryExportExp.pfnFreeExportableMemoryExp; + if (nullptr == pfnFreeExportableMemoryExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnFreeExportableMemoryExp(hContext, hDevice, pMem); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportExportMemoryHandleExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnExportMemoryHandleExp = + dditable->MemoryExportExp.pfnExportMemoryHandleExp; + if (nullptr == pfnExportMemoryHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnExportMemoryHandleExp(hContext, hDevice, handleTypeToExport, pMem, + pMemHandleRet); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urProgramBuildExp __urdlllocal ur_result_t UR_APICALL urProgramBuildExp( @@ -6380,6 +6457,64 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetMemProcAddrTable( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Exported function for filling application's MemoryExportExp table +/// with current process' addresses +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// - ::UR_RESULT_ERROR_UNSUPPORTED_VERSION +UR_DLLEXPORT ur_result_t UR_APICALL urGetMemoryExportExpProcAddrTable( + /// [in] API version requested + ur_api_version_t version, + /// [in,out] pointer to table of DDI function pointers + ur_memory_export_exp_dditable_t *pDdiTable) { + if (nullptr == pDdiTable) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (ur_loader::getContext()->version < version) + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + + ur_result_t result = UR_RESULT_SUCCESS; + + // Load the device-platform DDI tables + for (auto &platform : ur_loader::getContext()->platforms) { + // statically linked adapter inside of the loader + if (platform.handle == nullptr) + continue; + + if (platform.initStatus != UR_RESULT_SUCCESS) + continue; + auto getTable = reinterpret_cast( + ur_loader::LibLoader::getFunctionPtr( + platform.handle.get(), "urGetMemoryExportExpProcAddrTable")); + if (!getTable) + continue; + platform.initStatus = getTable(version, &platform.dditable.MemoryExportExp); + } + + if (UR_RESULT_SUCCESS == result) { + if (ur_loader::getContext()->platforms.size() != 1 || + ur_loader::getContext()->forceIntercept) { + // return pointers to loader's DDIs + pDdiTable->pfnAllocExportableMemoryExp = + ur_loader::urMemoryExportAllocExportableMemoryExp; + pDdiTable->pfnFreeExportableMemoryExp = + ur_loader::urMemoryExportFreeExportableMemoryExp; + pDdiTable->pfnExportMemoryHandleExp = + ur_loader::urMemoryExportExportMemoryHandleExp; + } else { + // return pointers directly to platform's DDIs + *pDdiTable = + ur_loader::getContext()->platforms.front().dditable.MemoryExportExp; + } + } + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Exported function for filling application's PhysicalMem table /// with current process' addresses diff --git a/source/loader/ur_libapi.cpp b/source/loader/ur_libapi.cpp index cad6de4dd9..a31b639ae5 100644 --- a/source/loader/ur_libapi.cpp +++ b/source/loader/ur_libapi.cpp @@ -926,7 +926,8 @@ ur_result_t UR_APICALL urDeviceGetSelected( /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hDevice` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP < propName` +/// + `::UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP < +/// propName` /// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION /// + If `propName` is not supported by the adapter. /// - ::UR_RESULT_ERROR_INVALID_SIZE @@ -9994,6 +9995,151 @@ ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( return exceptionToResult(std::current_exception()); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Allocate an exportable memory region and return a pointer to that +/// allocation. +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// + `(hDevice == nullptr) || (hContext == nullptr)` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `ppMem == nullptr` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT +/// + `alignment != 0 && ((alignment & (alignment-1)) != 0)` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + If `alignment` exceeds largest supported data type by `hDevice` +/// - ::UR_RESULT_ERROR_INVALID_USM_SIZE +/// + `size == 0` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_SIZE +/// + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE. +/// - +/// ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE:DEVICE_INFO_MEMORY_EXPORT_LINEAR_MEMORY_EXPORT_SUPPORT_EXP +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. + size_t size, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) try { + auto pfnAllocExportableMemoryExp = + ur_lib::getContext() + ->urDdiTable.MemoryExportExp.pfnAllocExportableMemoryExp; + if (nullptr == pfnAllocExportableMemoryExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnAllocExportableMemoryExp(hContext, hDevice, alignment, size, + handleTypeToExport, ppMem); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Free an exportable memory allocation. +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// + `(hDevice == nullptr) || (hContext == nullptr)` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `pMem == nullptr` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) try { + auto pfnFreeExportableMemoryExp = + ur_lib::getContext() + ->urDdiTable.MemoryExportExp.pfnFreeExportableMemoryExp; + if (nullptr == pfnFreeExportableMemoryExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnFreeExportableMemoryExp(hContext, hDevice, pMem); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Obtain an exportable handle to a memory allocated with +/// `AllocExportableMemoryExp`.The returned external memory type will be +/// that which was specified upon +/// allocation of the exportable memory (e.g. `opaque_fd` or +/// `win32_nt_handle`). +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// + `(hDevice == nullptr) || (hContext == nullptr)` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `pMemHandleRet == nullptr || pMem == nullptr` +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) try { + auto pfnExportMemoryHandleExp = + ur_lib::getContext()->urDdiTable.MemoryExportExp.pfnExportMemoryHandleExp; + if (nullptr == pfnExportMemoryHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnExportMemoryHandleExp(hContext, hDevice, handleTypeToExport, pMem, + pMemHandleRet); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Produces an executable program from one program, negates need for the /// linking step. diff --git a/source/loader/ur_libddi.cpp b/source/loader/ur_libddi.cpp index 616b79ef54..b06ee531cd 100644 --- a/source/loader/ur_libddi.cpp +++ b/source/loader/ur_libddi.cpp @@ -65,6 +65,11 @@ __urdlllocal ur_result_t context_t::ddiInit() { result = urGetMemProcAddrTable(UR_API_VERSION_CURRENT, &urDdiTable.Mem); } + if (UR_RESULT_SUCCESS == result) { + result = urGetMemoryExportExpProcAddrTable(UR_API_VERSION_CURRENT, + &urDdiTable.MemoryExportExp); + } + if (UR_RESULT_SUCCESS == result) { result = urGetPhysicalMemProcAddrTable(UR_API_VERSION_CURRENT, &urDdiTable.PhysicalMem); diff --git a/source/loader/ur_print.cpp b/source/loader/ur_print.cpp index a1cc0a0ae5..f3d5c96e37 100644 --- a/source/loader/ur_print.cpp +++ b/source/loader/ur_print.cpp @@ -2298,6 +2298,30 @@ ur_result_t urPrintMemImageGetInfoParams( return str_copy(&ss, buffer, buff_size, out_size); } +ur_result_t urPrintMemoryExportAllocExportableMemoryExpParams( + const struct ur_memory_export_alloc_exportable_memory_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintMemoryExportFreeExportableMemoryExpParams( + const struct ur_memory_export_free_exportable_memory_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintMemoryExportExportMemoryHandleExpParams( + const struct ur_memory_export_export_memory_handle_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + ur_result_t urPrintPhysicalMemCreateParams( const struct ur_physical_mem_create_params_t *params, char *buffer, const size_t buff_size, size_t *out_size) { diff --git a/source/ur_api.cpp b/source/ur_api.cpp index 426ca95027..da84b7f50f 100644 --- a/source/ur_api.cpp +++ b/source/ur_api.cpp @@ -833,7 +833,8 @@ ur_result_t UR_APICALL urDeviceGetSelected( /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hDevice` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP < propName` +/// + `::UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP < +/// propName` /// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION /// + If `propName` is not supported by the adapter. /// - ::UR_RESULT_ERROR_INVALID_SIZE @@ -8701,6 +8702,129 @@ ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Allocate an exportable memory region and return a pointer to that +/// allocation. +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// + `(hDevice == nullptr) || (hContext == nullptr)` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `ppMem == nullptr` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT +/// + `alignment != 0 && ((alignment & (alignment-1)) != 0)` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + If `alignment` exceeds largest supported data type by `hDevice` +/// - ::UR_RESULT_ERROR_INVALID_USM_SIZE +/// + `size == 0` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_SIZE +/// + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE. +/// - +/// ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE:DEVICE_INFO_MEMORY_EXPORT_LINEAR_MEMORY_EXPORT_SUPPORT_EXP +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. + size_t size, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Free an exportable memory allocation. +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// + `(hDevice == nullptr) || (hContext == nullptr)` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `pMem == nullptr` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Obtain an exportable handle to a memory allocated with +/// `AllocExportableMemoryExp`.The returned external memory type will be +/// that which was specified upon +/// allocation of the exportable memory (e.g. `opaque_fd` or +/// `win32_nt_handle`). +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// + `(hDevice == nullptr) || (hContext == nullptr)` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `pMemHandleRet == nullptr || pMem == nullptr` +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Produces an executable program from one program, negates need for the /// linking step. diff --git a/test/conformance/testing/include/uur/optional_queries.h b/test/conformance/testing/include/uur/optional_queries.h index 3f3bdb2953..154273131d 100644 --- a/test/conformance/testing/include/uur/optional_queries.h +++ b/test/conformance/testing/include/uur/optional_queries.h @@ -46,6 +46,8 @@ constexpr std::array optional_ur_device_info_t = { UR_DEVICE_INFO_FAN_SPEED, UR_DEVICE_INFO_MIN_POWER_LIMIT, UR_DEVICE_INFO_MAX_POWER_LIMIT, + UR_DEVICE_INFO_LUID, + UR_DEVICE_INFO_NODE_MASK, }; template <> inline bool isQueryOptional(ur_device_info_t query) { diff --git a/tools/urinfo/urinfo.hpp b/tools/urinfo/urinfo.hpp index d099c6365b..b08661787c 100644 --- a/tools/urinfo/urinfo.hpp +++ b/tools/urinfo/urinfo.hpp @@ -347,6 +347,10 @@ inline void printDeviceInfos(ur_device_handle_t hDevice, printDeviceInfo( hDevice, UR_DEVICE_INFO_KERNEL_LAUNCH_CAPABILITIES); std::cout << prefix; + printDeviceInfo(hDevice, UR_DEVICE_INFO_LUID); + std::cout << prefix; + printDeviceInfo(hDevice, UR_DEVICE_INFO_NODE_MASK); + std::cout << prefix; printDeviceInfo(hDevice, UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP); std::cout << prefix; @@ -451,5 +455,8 @@ inline void printDeviceInfos(ur_device_handle_t hDevice, std::cout << prefix; printDeviceInfo(hDevice, UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP); + std::cout << prefix; + printDeviceInfo( + hDevice, UR_DEVICE_INFO_MEMORY_EXPORT_EXPORTABLE_DEVICE_MEM_EXP); } } // namespace urinfo