diff --git a/unified-runtime/include/unified-runtime/ur_api.h b/unified-runtime/include/unified-runtime/ur_api.h index 5fe17e1eed9c3..1a7bdeca32d46 100644 --- a/unified-runtime/include/unified-runtime/ur_api.h +++ b/unified-runtime/include/unified-runtime/ur_api.h @@ -512,6 +512,14 @@ typedef enum ur_function_t { UR_FUNCTION_QUEUE_GET_GRAPH_EXP = 314, /// Enumerator for ::urGraphSetDestructionCallbackExp UR_FUNCTION_GRAPH_SET_DESTRUCTION_CALLBACK_EXP = 315, + /// Enumerator for ::urIPCGetPhysMemHandleExp + UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP = 316, + /// Enumerator for ::urIPCPutPhysMemHandleExp + UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP = 317, + /// Enumerator for ::urIPCOpenPhysMemHandleExp + UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP = 318, + /// Enumerator for ::urIPCClosePhysMemHandleExp + UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP = 319, /// @cond UR_FUNCTION_FORCE_UINT32 = 0x7fffffff /// @endcond @@ -2504,6 +2512,9 @@ typedef enum ur_device_info_t { /// [::ur_bool_t] returns true if the device supports inter-process /// communicable memory handles UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP = 0x2023, + /// [::ur_bool_t] returns true if the device supports inter-process + /// communicable physical memory handles + UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP = 0x2024, /// [::ur_bool_t] returns true if the device supports enqueueing of /// allocations and frees. UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP = 0x2050, @@ -5335,15 +5346,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urVirtualMemGetInfo( /// @brief Physical memory creation properties. typedef uint32_t ur_physical_mem_flags_t; typedef enum ur_physical_mem_flag_t { - /// reserved for future use. - UR_PHYSICAL_MEM_FLAG_TBD = UR_BIT(0), + /// allocate physical memory that can be shared via IPC handles. + UR_PHYSICAL_MEM_FLAG_ENABLE_IPC = UR_BIT(1), /// @cond UR_PHYSICAL_MEM_FLAG_FORCE_UINT32 = 0x7fffffff /// @endcond } ur_physical_mem_flag_t; /// @brief Bit Mask for validating ur_physical_mem_flags_t -#define UR_PHYSICAL_MEM_FLAGS_MASK 0xfffffffe +#define UR_PHYSICAL_MEM_FLAGS_MASK 0xfffffffd /////////////////////////////////////////////////////////////////////////////// /// @brief Physical memory creation properties. @@ -11241,6 +11252,113 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp( /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp void *pMem); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Gets an inter-process handle for a physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCPhysMemHandleData` +/// + `NULL == pIPCPhysMemHandleDataSizeRet` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_ARGUMENT +/// + `hPhysMem` was not created with the +/// `::UR_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Releases an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Opens an inter-process physical memory handle to get the +/// corresponding +/// physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPhysMem` +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcPhysMemHandleDataSize is not the same as the size of IPC +/// physical memory handle data +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Closes an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem); + #if !defined(__GNUC__) #pragma endregion #endif @@ -16503,6 +16621,47 @@ typedef struct ur_ipc_close_mem_handle_exp_params_t { void **ppMem; } ur_ipc_close_mem_handle_exp_params_t; +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCGetPhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_get_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + ur_physical_mem_handle_t *phPhysMem; + void ***pppIPCPhysMemHandleData; + size_t **ppIPCPhysMemHandleDataSizeRet; +} ur_ipc_get_phys_mem_handle_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCPutPhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_put_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + const void **ppIPCPhysMemHandleData; +} ur_ipc_put_phys_mem_handle_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCOpenPhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_open_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + ur_device_handle_t *phDevice; + const void **ppIPCPhysMemHandleData; + size_t *pipcPhysMemHandleDataSize; + ur_physical_mem_handle_t **pphPhysMem; +} ur_ipc_open_phys_mem_handle_exp_params_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function parameters for urIPCClosePhysMemHandleExp +/// @details Each entry is a pointer to the parameter passed to the function; +/// allowing the callback the ability to modify the parameter's value +typedef struct ur_ipc_close_phys_mem_handle_exp_params_t { + ur_context_handle_t *phContext; + ur_physical_mem_handle_t *phPhysMem; +} ur_ipc_close_phys_mem_handle_exp_params_t; + /////////////////////////////////////////////////////////////////////////////// /// @brief Function parameters for urMemoryExportAllocExportableMemoryExp /// @details Each entry is a pointer to the parameter passed to the function; diff --git a/unified-runtime/include/unified-runtime/ur_api_funcs.def b/unified-runtime/include/unified-runtime/ur_api_funcs.def index fc481d947a8c7..f0d68bcf009f0 100644 --- a/unified-runtime/include/unified-runtime/ur_api_funcs.def +++ b/unified-runtime/include/unified-runtime/ur_api_funcs.def @@ -223,6 +223,10 @@ _UR_API(urIPCGetMemHandleExp) _UR_API(urIPCPutMemHandleExp) _UR_API(urIPCOpenMemHandleExp) _UR_API(urIPCCloseMemHandleExp) +_UR_API(urIPCGetPhysMemHandleExp) +_UR_API(urIPCPutPhysMemHandleExp) +_UR_API(urIPCOpenPhysMemHandleExp) +_UR_API(urIPCClosePhysMemHandleExp) _UR_API(urMemoryExportAllocExportableMemoryExp) _UR_API(urMemoryExportFreeExportableMemoryExp) _UR_API(urMemoryExportExportMemoryHandleExp) diff --git a/unified-runtime/include/unified-runtime/ur_ddi.h b/unified-runtime/include/unified-runtime/ur_ddi.h index cc734e6194d92..8f45375c369c1 100644 --- a/unified-runtime/include/unified-runtime/ur_ddi.h +++ b/unified-runtime/include/unified-runtime/ur_ddi.h @@ -1950,6 +1950,27 @@ typedef ur_result_t(UR_APICALL *ur_pfnIPCOpenMemHandleExp_t)( typedef ur_result_t(UR_APICALL *ur_pfnIPCCloseMemHandleExp_t)( ur_context_handle_t, void *); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCGetPhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCGetPhysMemHandleExp_t)( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCPutPhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCPutPhysMemHandleExp_t)( + ur_context_handle_t, const void *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCOpenPhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCOpenPhysMemHandleExp_t)( + ur_context_handle_t, ur_device_handle_t, const void *, size_t, + ur_physical_mem_handle_t *); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urIPCClosePhysMemHandleExp +typedef ur_result_t(UR_APICALL *ur_pfnIPCClosePhysMemHandleExp_t)( + ur_context_handle_t, ur_physical_mem_handle_t); + /////////////////////////////////////////////////////////////////////////////// /// @brief Table of IPCExp functions pointers typedef struct ur_ipc_exp_dditable_t { @@ -1957,6 +1978,10 @@ typedef struct ur_ipc_exp_dditable_t { ur_pfnIPCPutMemHandleExp_t pfnPutMemHandleExp; ur_pfnIPCOpenMemHandleExp_t pfnOpenMemHandleExp; ur_pfnIPCCloseMemHandleExp_t pfnCloseMemHandleExp; + ur_pfnIPCGetPhysMemHandleExp_t pfnGetPhysMemHandleExp; + ur_pfnIPCPutPhysMemHandleExp_t pfnPutPhysMemHandleExp; + ur_pfnIPCOpenPhysMemHandleExp_t pfnOpenPhysMemHandleExp; + ur_pfnIPCClosePhysMemHandleExp_t pfnClosePhysMemHandleExp; } ur_ipc_exp_dditable_t; /////////////////////////////////////////////////////////////////////////////// diff --git a/unified-runtime/include/unified-runtime/ur_print.h b/unified-runtime/include/unified-runtime/ur_print.h index d9acae65c7d07..70ff14c95a499 100644 --- a/unified-runtime/include/unified-runtime/ur_print.h +++ b/unified-runtime/include/unified-runtime/ur_print.h @@ -3785,6 +3785,46 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcCloseMemHandleExpParams( const struct ur_ipc_close_mem_handle_exp_params_t *params, char *buffer, const size_t buff_size, size_t *out_size); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_get_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcGetPhysMemHandleExpParams( + const struct ur_ipc_get_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_put_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcPutPhysMemHandleExpParams( + const struct ur_ipc_put_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_open_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcOpenPhysMemHandleExpParams( + const struct ur_ipc_open_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_ipc_close_phys_mem_handle_exp_params_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintIpcClosePhysMemHandleExpParams( + const struct ur_ipc_close_phys_mem_handle_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size); + /////////////////////////////////////////////////////////////////////////////// /// @brief Print ur_memory_export_alloc_exportable_memory_exp_params_t struct /// @returns diff --git a/unified-runtime/include/unified-runtime/ur_print.hpp b/unified-runtime/include/unified-runtime/ur_print.hpp index c29373a7d162a..a8393a59a9b4c 100644 --- a/unified-runtime/include/unified-runtime/ur_print.hpp +++ b/unified-runtime/include/unified-runtime/ur_print.hpp @@ -1380,6 +1380,18 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_function_t value) { case UR_FUNCTION_GRAPH_SET_DESTRUCTION_CALLBACK_EXP: os << "UR_FUNCTION_GRAPH_SET_DESTRUCTION_CALLBACK_EXP"; break; + case UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP"; + break; + case UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP"; + break; + case UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP"; + break; + case UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP: + os << "UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP"; + break; default: os << "unknown enumerator"; break; @@ -3309,6 +3321,9 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_device_info_t value) { case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: os << "UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP"; break; + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + os << "UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP"; + break; case UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP: os << "UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP"; break; @@ -5594,6 +5609,19 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, os << ")"; } break; + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: { + const ur_bool_t *tptr = (const ur_bool_t *)ptr; + if (sizeof(ur_bool_t) > size) { + os << "invalid size (is: " << size + << ", expected: >=" << sizeof(ur_bool_t) << ")"; + return UR_RESULT_ERROR_INVALID_SIZE; + } + os << (const void *)(tptr) << " ("; + + os << *tptr; + + os << ")"; + } break; case UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP: { const ur_bool_t *tptr = (const ur_bool_t *)ptr; if (sizeof(ur_bool_t) > size) { @@ -9144,8 +9172,8 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, inline std::ostream &operator<<(std::ostream &os, enum ur_physical_mem_flag_t value) { switch (value) { - case UR_PHYSICAL_MEM_FLAG_TBD: - os << "UR_PHYSICAL_MEM_FLAG_TBD"; + case UR_PHYSICAL_MEM_FLAG_ENABLE_IPC: + os << "UR_PHYSICAL_MEM_FLAG_ENABLE_IPC"; break; default: os << "unknown enumerator"; @@ -9163,14 +9191,15 @@ inline ur_result_t printFlag(std::ostream &os, uint32_t val = flag; bool first = true; - if ((val & UR_PHYSICAL_MEM_FLAG_TBD) == (uint32_t)UR_PHYSICAL_MEM_FLAG_TBD) { - val ^= (uint32_t)UR_PHYSICAL_MEM_FLAG_TBD; + if ((val & UR_PHYSICAL_MEM_FLAG_ENABLE_IPC) == + (uint32_t)UR_PHYSICAL_MEM_FLAG_ENABLE_IPC) { + val ^= (uint32_t)UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; if (!first) { os << " | "; } else { first = false; } - os << UR_PHYSICAL_MEM_FLAG_TBD; + os << UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; } if (val != 0) { std::bitset<32> bits(val); @@ -21724,6 +21753,115 @@ operator<<(std::ostream &os, return os; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_get_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, + [[maybe_unused]] const struct ur_ipc_get_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hPhysMem = "; + + ur::details::printPtr(os, *(params->phPhysMem)); + + os << ", "; + os << ".ppIPCPhysMemHandleData = "; + + ur::details::printPtr(os, *(params->pppIPCPhysMemHandleData)); + + os << ", "; + os << ".pIPCPhysMemHandleDataSizeRet = "; + + ur::details::printPtr(os, *(params->ppIPCPhysMemHandleDataSizeRet)); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_put_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, + [[maybe_unused]] const struct ur_ipc_put_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".pIPCPhysMemHandleData = "; + + ur::details::printPtr(os, *(params->ppIPCPhysMemHandleData)); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_open_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream &operator<<( + std::ostream &os, + [[maybe_unused]] const struct ur_ipc_open_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hDevice = "; + + ur::details::printPtr(os, *(params->phDevice)); + + os << ", "; + os << ".pIPCPhysMemHandleData = "; + + os << *(params->ppIPCPhysMemHandleData); + + os << ", "; + os << ".ipcPhysMemHandleDataSize = "; + + os << *(params->pipcPhysMemHandleDataSize); + + os << ", "; + os << ".phPhysMem = "; + + ur::details::printPtr(os, *(params->pphPhysMem)); + + return os; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_ipc_close_phys_mem_handle_exp_params_t type +/// @returns +/// std::ostream & +inline std::ostream &operator<<( + std::ostream &os, + [[maybe_unused]] const struct ur_ipc_close_phys_mem_handle_exp_params_t + *params) { + + os << ".hContext = "; + + ur::details::printPtr(os, *(params->phContext)); + + os << ", "; + os << ".hPhysMem = "; + + ur::details::printPtr(os, *(params->phPhysMem)); + + return os; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Print operator for the /// ur_memory_export_alloc_exportable_memory_exp_params_t type @@ -23280,6 +23418,18 @@ inline ur_result_t UR_APICALL printFunctionParams(std::ostream &os, case UR_FUNCTION_IPC_CLOSE_MEM_HANDLE_EXP: { os << (const struct ur_ipc_close_mem_handle_exp_params_t *)params; } break; + case UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_get_phys_mem_handle_exp_params_t *)params; + } break; + case UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_put_phys_mem_handle_exp_params_t *)params; + } break; + case UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_open_phys_mem_handle_exp_params_t *)params; + } break; + case UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP: { + os << (const struct ur_ipc_close_phys_mem_handle_exp_params_t *)params; + } break; case UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP: { os << (const struct ur_memory_export_alloc_exportable_memory_exp_params_t *) params; diff --git a/unified-runtime/scripts/core/exp-inter-process-communication.yml b/unified-runtime/scripts/core/exp-inter-process-communication.yml index e5019bb40836d..71c15e6b9d71a 100644 --- a/unified-runtime/scripts/core/exp-inter-process-communication.yml +++ b/unified-runtime/scripts/core/exp-inter-process-communication.yml @@ -23,6 +23,16 @@ etors: value: "0x2023" desc: "[$x_bool_t] returns true if the device supports inter-process communicable memory handles" --- #-------------------------------------------------------------------------- +type: enum +extend: true +typed_etors: true +desc: "Extension enums to $x_device_info_t to support inter-process communicable physical memory handles." +name: $x_device_info_t +etors: + - name: IPC_PHYSICAL_MEMORY_SUPPORT_EXP + value: "0x2024" + desc: "[$x_bool_t] returns true if the device supports inter-process communicable physical memory handles" +--- #-------------------------------------------------------------------------- type: function desc: "Gets an inter-process memory handle for a pointer to device USM memory" class: $xIPC @@ -126,3 +136,109 @@ returns: - "`NULL == pMem`" - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Gets an inter-process handle for a physical memory object" +class: $xIPC +name: GetPhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: $x_physical_mem_handle_t + name: hPhysMem + desc: "[in] handle of the physical memory object" + - type: void** + name: ppIPCPhysMemHandleData + desc: "[out] a pointer to the IPC physical memory handle data" + - type: size_t* + name: pIPCPhysMemHandleDataSizeRet + desc: "[out] size of the resulting IPC physical memory handle data" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - "`NULL == hPhysMem`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`NULL == ppIPCPhysMemHandleData`" + - "`NULL == pIPCPhysMemHandleDataSizeRet`" + - $X_RESULT_ERROR_INVALID_ARGUMENT: + - "`hPhysMem` was not created with the `$X_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Releases an inter-process physical memory handle" +class: $xIPC +name: PutPhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: const void* + name: pIPCPhysMemHandleData + desc: "[in] a pointer to the IPC physical memory handle data obtained with urIPCGetPhysMemHandleExp" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`NULL == pIPCPhysMemHandleData`" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Opens an inter-process physical memory handle to get the corresponding physical memory object" +class: $xIPC +name: OpenPhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: $x_device_handle_t + name: hDevice + desc: "[in] handle of the device object the physical memory was allocated on" + - type: const void * + name: pIPCPhysMemHandleData + desc: "[in] the IPC physical memory handle data obtained with urIPCGetPhysMemHandleExp" + - type: size_t + name: ipcPhysMemHandleDataSize + desc: "[in] size of the IPC physical memory handle data" + - type: $x_physical_mem_handle_t* + name: phPhysMem + desc: "[out] pointer to the physical memory handle" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - "`NULL == hDevice`" + - $X_RESULT_ERROR_INVALID_NULL_POINTER: + - "`NULL == pIPCPhysMemHandleData`" + - "`NULL == phPhysMem`" + - $X_RESULT_ERROR_INVALID_VALUE: + - "ipcPhysMemHandleDataSize is not the same as the size of IPC physical memory handle data" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES +--- #-------------------------------------------------------------------------- +type: function +desc: "Closes an inter-process physical memory handle" +class: $xIPC +name: ClosePhysMemHandleExp +ordinal: "0" +params: + - type: $x_context_handle_t + name: hContext + desc: "[in] handle of the context object" + - type: $x_physical_mem_handle_t + name: hPhysMem + desc: "[in] physical memory handle opened through urIPCOpenPhysMemHandleExp" +returns: + - $X_RESULT_ERROR_INVALID_CONTEXT + - $X_RESULT_ERROR_INVALID_NULL_HANDLE: + - "`NULL == hContext`" + - "`NULL == hPhysMem`" + - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY + - $X_RESULT_ERROR_OUT_OF_RESOURCES diff --git a/unified-runtime/scripts/core/registry.yml b/unified-runtime/scripts/core/registry.yml index f7a11b40c6547..3d4f037329ad9 100644 --- a/unified-runtime/scripts/core/registry.yml +++ b/unified-runtime/scripts/core/registry.yml @@ -727,7 +727,19 @@ etors: - name: GRAPH_SET_DESTRUCTION_CALLBACK_EXP desc: Enumerator for $xGraphSetDestructionCallbackExp value: '315' -max_id: '315' +- name: IPC_GET_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCGetPhysMemHandleExp + value: '316' +- name: IPC_PUT_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCPutPhysMemHandleExp + value: '317' +- name: IPC_OPEN_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCOpenPhysMemHandleExp + value: '318' +- name: IPC_CLOSE_PHYS_MEM_HANDLE_EXP + desc: Enumerator for $xIPCClosePhysMemHandleExp + value: '319' +max_id: '319' --- type: enum desc: Defines structure types diff --git a/unified-runtime/scripts/core/virtual_memory.yml b/unified-runtime/scripts/core/virtual_memory.yml index 00d03c54a5726..4fe51b74dd550 100644 --- a/unified-runtime/scripts/core/virtual_memory.yml +++ b/unified-runtime/scripts/core/virtual_memory.yml @@ -246,8 +246,9 @@ desc: "Physical memory creation properties." class: $xPhysicalMem name: $x_physical_mem_flags_t etors: - - name: TBD - desc: "reserved for future use." + - name: ENABLE_IPC + value: "$X_BIT(1)" + desc: "allocate physical memory that can be shared via IPC handles." --- #-------------------------------------------------------------------------- type: struct diff --git a/unified-runtime/source/adapters/cuda/device.cpp b/unified-runtime/source/adapters/cuda/device.cpp index d213d51eaeb9b..9ef56d297e330 100644 --- a/unified-runtime/source/adapters/cuda/device.cpp +++ b/unified-runtime/source/adapters/cuda/device.cpp @@ -1173,6 +1173,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, hDevice->get())); return ReturnValue(static_cast(IPCSupported)); } + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: case UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP: return ReturnValue(true); diff --git a/unified-runtime/source/adapters/cuda/memory.cpp b/unified-runtime/source/adapters/cuda/memory.cpp index 81b86d7445381..cf73ac1cbca65 100644 --- a/unified-runtime/source/adapters/cuda/memory.cpp +++ b/unified-runtime/source/adapters/cuda/memory.cpp @@ -639,3 +639,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *pMem) { return umf::umf2urResult(umfCloseIPCHandle(pMem)); } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp b/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp index 1f6434b2c140f..e1cce057c6320 100644 --- a/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/cuda/ur_interface_loader.cpp @@ -510,6 +510,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/hip/device.cpp b/unified-runtime/source/adapters/hip/device.cpp index f07a31c03ce4e..e43f7e58164f0 100644 --- a/unified-runtime/source/adapters/hip/device.cpp +++ b/unified-runtime/source/adapters/hip/device.cpp @@ -1021,6 +1021,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, static_cast(0)); case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: return ReturnValue(false); + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: { int RuntimeVersion = 0; UR_CHECK_ERROR(hipRuntimeGetVersion(&RuntimeVersion)); diff --git a/unified-runtime/source/adapters/hip/memory.cpp b/unified-runtime/source/adapters/hip/memory.cpp index 6360ab0f92352..78c4a7c93ea24 100644 --- a/unified-runtime/source/adapters/hip/memory.cpp +++ b/unified-runtime/source/adapters/hip/memory.cpp @@ -658,3 +658,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/hip/ur_interface_loader.cpp b/unified-runtime/source/adapters/hip/ur_interface_loader.cpp index 1d6fbdf7a9c0c..48c6c1b161546 100644 --- a/unified-runtime/source/adapters/hip/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/hip/ur_interface_loader.cpp @@ -503,6 +503,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/CMakeLists.txt b/unified-runtime/source/adapters/level_zero/CMakeLists.txt index 42e63e952f392..6e5f29af3610f 100644 --- a/unified-runtime/source/adapters/level_zero/CMakeLists.txt +++ b/unified-runtime/source/adapters/level_zero/CMakeLists.txt @@ -133,6 +133,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/image_common.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.hpp @@ -145,7 +146,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/image_common.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index f72b49790bac6..4541c8645b5d6 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -1366,6 +1366,12 @@ ur_result_t urDeviceGetInfo( return ReturnValue(false); #else return ReturnValue(true); +#endif + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: +#if defined(UR_ADAPTER_LEVEL_ZERO_V2) && defined(__linux__) + return ReturnValue(true); +#else + return ReturnValue(false); #endif case UR_DEVICE_INFO_ASYNC_BARRIER: return ReturnValue(false); diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/physical_mem.cpp index a0663253f42e3..96f7c9d0c2f99 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.cpp @@ -9,13 +9,8 @@ #include "physical_mem.hpp" #include "common.hpp" -#include "device.hpp" - -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 -#include "v2/context.hpp" -#else #include "context.hpp" -#endif +#include "device.hpp" namespace ur::level_zero { @@ -31,10 +26,14 @@ ur_result_t urPhysicalMemCreate( ZE2UR_CALL(zePhysicalMemCreate, (hContext->getZeHandle(), hDevice->ZeDevice, &PhysicalMemDesc, &ZePhysicalMem)); try { - *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext); + *phPhysicalMem = + new ur_physical_mem_handle_t_(ZePhysicalMem, hContext, hDevice, size, + /*EnableIpc=*/false); } catch (const std::bad_alloc &) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; } catch (...) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); return UR_RESULT_ERROR_UNKNOWN; } return UR_RESULT_SUCCESS; @@ -66,13 +65,45 @@ ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); switch (propName) { - case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: { - return ReturnValue(hPhysicalMem->RefCount.getCount()); + case UR_PHYSICAL_MEM_INFO_CONTEXT: + return ReturnValue(hPhysicalMem->Context); + case UR_PHYSICAL_MEM_INFO_DEVICE: + return ReturnValue(hPhysicalMem->Device); + case UR_PHYSICAL_MEM_INFO_SIZE: + return ReturnValue(hPhysicalMem->Size); + case UR_PHYSICAL_MEM_INFO_PROPERTIES: { + ur_physical_mem_flags_t Flags = static_cast(0); + ur_physical_mem_properties_t Props = { + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, Flags}; + return ReturnValue(Props); } + case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: + return ReturnValue(hPhysicalMem->RefCount.getCount()); default: return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; } return UR_RESULT_SUCCESS; } +ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t, + ur_physical_mem_handle_t, void **, + size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, + const void *, size_t, + ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t, + ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + } // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/physical_mem.hpp index a5db639575e8d..10938d1e7e282 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.hpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.hpp @@ -13,8 +13,11 @@ struct ur_physical_mem_handle_t_ : ur_object { ur_physical_mem_handle_t_(ze_physical_mem_handle_t ZePhysicalMem, - ur_context_handle_t Context) - : ZePhysicalMem{ZePhysicalMem}, Context{Context} {} + ur_context_handle_t Context, + ur_device_handle_t Device, size_t Size, + bool EnableIpc) + : ZePhysicalMem{ZePhysicalMem}, Context{Context}, Device{Device}, + Size{Size}, EnableIpc{EnableIpc} {} // Level Zero physical memory handle. ze_physical_mem_handle_t ZePhysicalMem; @@ -22,5 +25,14 @@ struct ur_physical_mem_handle_t_ : ur_object { // Keeps the PI context of this memory handle. ur_context_handle_t Context; + // Device this physical memory was allocated on. + ur_device_handle_t Device; + + // Size in bytes of this physical memory allocation. + size_t Size; + + // Whether this allocation was created with IPC export enabled. + bool EnableIpc; + ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp index dc76f7729f937..ce63be24c0a64 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp @@ -293,6 +293,12 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = ur::level_zero::urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = ur::level_zero::urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur::level_zero::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = ur::level_zero::urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = ur::level_zero::urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = + ur::level_zero::urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur::level_zero::urIPCClosePhysMemHandleExp; return result; } diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp index b56cc5e803c5d..bc4956f6eed2f 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp @@ -629,6 +629,19 @@ ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, void *pIPCMemHandleData, size_t ipcMemHandleDataSize, void **ppMem); ur_result_t urIPCCloseMemHandleExp(ur_context_handle_t hContext, void *pMem); +ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem, + void **ppIPCPhysMemHandleData, + size_t *pIPCPhysMemHandleDataSizeRet); +ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t hContext, + const void *pIPCPhysMemHandleData); +ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const void *pIPCPhysMemHandleData, + size_t ipcPhysMemHandleDataSize, + ur_physical_mem_handle_t *phPhysMem); +ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t hContext, + ur_physical_mem_handle_t hPhysMem); ur_result_t urMemoryExportAllocExportableMemoryExp( ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t alignment, size_t size, ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); diff --git a/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp new file mode 100644 index 0000000000000..9e360fb8fb01c --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/physical_mem.cpp @@ -0,0 +1,276 @@ +//===---------- physical_mem.cpp - Level Zero Adapter v2 ----------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "physical_mem.hpp" +#include "../common.hpp" +#include "../device.hpp" + +#ifdef __linux__ +#include +#include +#endif + +#include "ur_util.hpp" +#include + +#include "context.hpp" + +namespace ur::level_zero { + +ur_result_t urPhysicalMemCreate(ur_context_handle_t hContext, + ur_device_handle_t hDevice, size_t size, + const ur_physical_mem_properties_t *pProperties, + ur_physical_mem_handle_t *phPhysicalMem) { + ZeStruct PhysicalMemDesc; + PhysicalMemDesc.flags = 0; + PhysicalMemDesc.size = size; + + // If IPC export is requested, chain in the export descriptor so the + // physical memory can later be shared via urIPCGetPhysMemHandleExp. + bool EnableIpc = + pProperties && (pProperties->flags & UR_PHYSICAL_MEM_FLAG_ENABLE_IPC); + std::optional ExportDesc; + if (EnableIpc) { + ExportDesc = ze_external_memory_export_desc_t{}; + ExportDesc->stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_DESC; + ExportDesc->pNext = nullptr; + ExportDesc->flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_FD; + PhysicalMemDesc.pNext = &ExportDesc.value(); + } + + ze_physical_mem_handle_t ZePhysicalMem; + ZE2UR_CALL(zePhysicalMemCreate, (hContext->getZeHandle(), hDevice->ZeDevice, + &PhysicalMemDesc, &ZePhysicalMem)); + try { + *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext, + hDevice, size, EnableIpc); + } catch (const std::bad_alloc &) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysicalMem); + return UR_RESULT_ERROR_UNKNOWN; + } + return UR_RESULT_SUCCESS; +} + +ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { + hPhysicalMem->RefCount.retain(); + return UR_RESULT_SUCCESS; +} + +ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { + if (!hPhysicalMem->RefCount.release()) + return UR_RESULT_SUCCESS; + + if (checkL0LoaderTeardown()) { + ZE2UR_CALL(zePhysicalMemDestroy, (hPhysicalMem->Context->getZeHandle(), + hPhysicalMem->ZePhysicalMem)); + } + delete hPhysicalMem; + + return UR_RESULT_SUCCESS; +} + +ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, + ur_physical_mem_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { + + UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); + + switch (propName) { + case UR_PHYSICAL_MEM_INFO_CONTEXT: + return ReturnValue(hPhysicalMem->Context); + case UR_PHYSICAL_MEM_INFO_DEVICE: + return ReturnValue(hPhysicalMem->Device); + case UR_PHYSICAL_MEM_INFO_SIZE: + return ReturnValue(hPhysicalMem->Size); + case UR_PHYSICAL_MEM_INFO_PROPERTIES: { + ur_physical_mem_flags_t Flags = static_cast(0); + if (hPhysicalMem->EnableIpc) + Flags = UR_PHYSICAL_MEM_FLAG_ENABLE_IPC; + ur_physical_mem_properties_t Props = { + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, Flags}; + return ReturnValue(Props); + } + case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: + return ReturnValue(hPhysicalMem->RefCount.getCount()); + default: + return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + } + return UR_RESULT_SUCCESS; +} + +ur_result_t urIPCGetPhysMemHandleExp(ur_context_handle_t, + ur_physical_mem_handle_t hPhysMem, + void **ppIPCPhysMemHandleData, + size_t *pIPCPhysMemHandleDataSizeRet) { +#ifdef __linux__ + if (!hPhysMem->EnableIpc) + return UR_RESULT_ERROR_INVALID_ARGUMENT; + + // Export the physical memory object as an opaque file descriptor. + ze_external_memory_export_fd_t ExportFd = {}; + ExportFd.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_FD; + ExportFd.pNext = nullptr; + ExportFd.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_FD; + // Initialize to -1 so we can safely detect if the driver opened an fd even + // on a partially-completed call that then returns an error. + ExportFd.fd = -1; + + ze_physical_mem_properties_t Props = {}; + Props.stype = ZE_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES; + Props.pNext = &ExportFd; + + // Use ZE_CALL_NOCHECK so we can close any fd the driver may have opened + // before returning an error, rather than leaking it via an early return. + // Use hPhysMem->Context (the creation context) rather than the caller's + // hContext: zePhysicalMemGetProperties requires the same context that was + // used to create the object. + ze_result_t ZeRes = ZE_CALL_NOCHECK( + zePhysicalMemGetProperties, + (hPhysMem->Context->getZeHandle(), hPhysMem->ZePhysicalMem, &Props)); + if (ZeRes != ZE_RESULT_SUCCESS) { + if (ExportFd.fd >= 0) + close(ExportFd.fd); + return ze2urResult(ZeRes); + } + + if (ExportFd.fd < 0) + return UR_RESULT_ERROR_INVALID_VALUE; + + // dup() the fd so urIPCPutPhysMemHandleExp can close HandleData->Fd without + // closing the original fd exported by the driver, which must remain open for + // the lifetime of the physical_mem object. We own only the dup'd copy. + int DupFd = dup(ExportFd.fd); + if (DupFd < 0) + return UR_RESULT_ERROR_UNKNOWN; + + auto *HandleData = new (std::nothrow) ZeIPCPhysMemHandleData; + if (!HandleData) { + close(DupFd); + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + + // Store the exporting process's PID and dup'd fd. The fd stays open until + // urIPCPutPhysMemHandleExp is called. Cross-process consumers use + // pidfd_getfd(2) to obtain their own duplicate of this fd. + HandleData->Pid = getpid(); + HandleData->Fd = DupFd; + HandleData->Size = hPhysMem->Size; + + *ppIPCPhysMemHandleData = HandleData; + *pIPCPhysMemHandleDataSizeRet = sizeof(ZeIPCPhysMemHandleData); + return UR_RESULT_SUCCESS; +#else + (void)hPhysMem; + (void)ppIPCPhysMemHandleData; + (void)pIPCPhysMemHandleDataSizeRet; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +ur_result_t urIPCPutPhysMemHandleExp(ur_context_handle_t, + const void *pIPCPhysMemHandleData) { +#ifdef __linux__ + auto *HandleData = + static_cast(pIPCPhysMemHandleData); + close(HandleData->Fd); + delete HandleData; + return UR_RESULT_SUCCESS; +#else + (void)pIPCPhysMemHandleData; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +ur_result_t urIPCOpenPhysMemHandleExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const void *pIPCPhysMemHandleData, + size_t ipcPhysMemHandleDataSize, + ur_physical_mem_handle_t *phPhysMem) { +#ifdef __linux__ + if (ipcPhysMemHandleDataSize != sizeof(ZeIPCPhysMemHandleData)) + return UR_RESULT_ERROR_INVALID_VALUE; + + auto *HandleData = + static_cast(pIPCPhysMemHandleData); + + if (HandleData->Fd < 0 || HandleData->Pid <= 0 || HandleData->Size == 0) + return UR_RESULT_ERROR_INVALID_VALUE; + + // Obtain a usable fd in the current process. For same-process opens + // (e.g. conformance tests) dup() suffices. For cross-process opens + // use ur_duplicate_fd() which calls pidfd_getfd(2) (Linux 5.6+) and + // requires the exporting process to be ptrace-accessible. + int ImportFdNum = -1; + if (HandleData->Pid == getpid()) { + ImportFdNum = dup(HandleData->Fd); + if (ImportFdNum < 0) + return UR_RESULT_ERROR_INVALID_VALUE; + } else { + ImportFdNum = ur_duplicate_fd(HandleData->Pid, HandleData->Fd); + if (ImportFdNum < 0) + return errno == EPERM ? UR_RESULT_ERROR_INVALID_ARGUMENT + : errno == ENOTSUP ? UR_RESULT_ERROR_UNSUPPORTED_FEATURE + : UR_RESULT_ERROR_INVALID_VALUE; + } + + ze_external_memory_import_fd_t ImportFd = {}; + ImportFd.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_FD; + ImportFd.pNext = nullptr; + ImportFd.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_FD; + ImportFd.fd = ImportFdNum; + + ZeStruct PhysMemDesc; + PhysMemDesc.pNext = &ImportFd; + PhysMemDesc.flags = 0; + PhysMemDesc.size = HandleData->Size; + + ze_physical_mem_handle_t ZePhysMem; + ze_result_t ZeRes = zePhysicalMemCreate( + hContext->getZeHandle(), hDevice->ZeDevice, &PhysMemDesc, &ZePhysMem); + // Driver has dup'd ImportFdNum internally; close our copy now. + close(ImportFdNum); + + if (ZeRes != ZE_RESULT_SUCCESS) + return ze2urResult(ZeRes); + + try { + *phPhysMem = new ur_physical_mem_handle_t_(ZePhysMem, hContext, hDevice, + HandleData->Size, + /*EnableIpc=*/true); + } catch (const std::bad_alloc &) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysMem); + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + zePhysicalMemDestroy(hContext->getZeHandle(), ZePhysMem); + return UR_RESULT_ERROR_UNKNOWN; + } + return UR_RESULT_SUCCESS; +#else + (void)hContext; + (void)hDevice; + (void)pIPCPhysMemHandleData; + (void)ipcPhysMemHandleDataSize; + (void)phPhysMem; + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +#endif // __linux__ +} + +ur_result_t urIPCClosePhysMemHandleExp(ur_context_handle_t, + ur_physical_mem_handle_t hPhysMem) { + // Delegate to urPhysicalMemRelease so the refcount is respected: if the + // handle has been retained (refcount > 1) it will not be destroyed until + // all references are released. + return ur::level_zero::urPhysicalMemRelease(hPhysMem); +} + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp new file mode 100644 index 0000000000000..9fc2ea8dc55ce --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/physical_mem.hpp @@ -0,0 +1,25 @@ +//===---------- physical_mem.hpp - Level Zero Adapter v2 ----------------===// +// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include "../physical_mem.hpp" + +// Opaque handle data exchanged between processes for physical memory IPC. +// Contains the PID and file descriptor exported by zePhysicalMemGetProperties, +// plus the allocation size required by zePhysicalMemCreate on import. +// Cross-process access uses pidfd_getfd(2) (Linux 5.6+): the consumer obtains +// a duplicate of the spawner's DMA-BUF fd via the spawner's PID. +// Only defined on Linux because pid_t and DMA-BUF fds are Linux concepts. +#ifdef __linux__ +struct ZeIPCPhysMemHandleData { + pid_t Pid; // PID of the exporting process + int Fd; // DMA-BUF fd in the exporting process + size_t Size; +}; +#endif // __linux__ diff --git a/unified-runtime/source/adapters/mock/ur_mockddi.cpp b/unified-runtime/source/adapters/mock/ur_mockddi.cpp index 195702990d65c..ddaa42ac60eec 100644 --- a/unified-runtime/source/adapters/mock/ur_mockddi.cpp +++ b/unified-runtime/source/adapters/mock/ur_mockddi.cpp @@ -9661,6 +9661,200 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return exceptionToResult(std::current_exception()); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_get_phys_mem_handle_exp_params_t params = { + &hContext, &hPhysMem, &ppIPCPhysMemHandleData, + &pIPCPhysMemHandleDataSizeRet}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCGetPhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCGetPhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCGetPhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_put_phys_mem_handle_exp_params_t params = {&hContext, + &pIPCPhysMemHandleData}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCPutPhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCPutPhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCPutPhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_open_phys_mem_handle_exp_params_t params = { + &hContext, &hDevice, &pIPCPhysMemHandleData, &ipcPhysMemHandleDataSize, + &phPhysMem}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCOpenPhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCOpenPhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + *phPhysMem = mock::createDummyHandle(); + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCOpenPhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_close_phys_mem_handle_exp_params_t params = {&hContext, &hPhysMem}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCClosePhysMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; + } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCClosePhysMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCClosePhysMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -13547,6 +13741,14 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnCloseMemHandleExp = driver::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = driver::urIPCGetPhysMemHandleExp; + + pDdiTable->pfnPutPhysMemHandleExp = driver::urIPCPutPhysMemHandleExp; + + pDdiTable->pfnOpenPhysMemHandleExp = driver::urIPCOpenPhysMemHandleExp; + + pDdiTable->pfnClosePhysMemHandleExp = driver::urIPCClosePhysMemHandleExp; + return result; } catch (...) { return exceptionToResult(std::current_exception()); diff --git a/unified-runtime/source/adapters/native_cpu/device.cpp b/unified-runtime/source/adapters/native_cpu/device.cpp index f08de5694a2f3..e74668f197281 100644 --- a/unified-runtime/source/adapters/native_cpu/device.cpp +++ b/unified-runtime/source/adapters/native_cpu/device.cpp @@ -381,6 +381,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, static_cast(0)); case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: return ReturnValue(false); + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_ATOMIC_FENCE_ORDER_CAPABILITIES: { // Currently for Native CPU fences are implemented using OCK // builtins, so we have different capabilities than atomic operations diff --git a/unified-runtime/source/adapters/native_cpu/memory.cpp b/unified-runtime/source/adapters/native_cpu/memory.cpp index 2ac5eccc11cc3..2328dcfc287b8 100644 --- a/unified-runtime/source/adapters/native_cpu/memory.cpp +++ b/unified-runtime/source/adapters/native_cpu/memory.cpp @@ -160,3 +160,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp b/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp index c3b121bfb0cdb..5f1228273b006 100644 --- a/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/native_cpu/ur_interface_loader.cpp @@ -487,6 +487,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/offload/memory.cpp b/unified-runtime/source/adapters/offload/memory.cpp index c0b7a97fb62ef..7d55d14018481 100644 --- a/unified-runtime/source/adapters/offload/memory.cpp +++ b/unified-runtime/source/adapters/offload/memory.cpp @@ -229,3 +229,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/offload/ur_interface_loader.cpp b/unified-runtime/source/adapters/offload/ur_interface_loader.cpp index ce4ab5351a527..f88af93d6a2b2 100644 --- a/unified-runtime/source/adapters/offload/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/offload/ur_interface_loader.cpp @@ -437,6 +437,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/opencl/device.cpp b/unified-runtime/source/adapters/opencl/device.cpp index 9654c6b0f764b..f8d11ce26c3ad 100644 --- a/unified-runtime/source/adapters/opencl/device.cpp +++ b/unified-runtime/source/adapters/opencl/device.cpp @@ -1375,6 +1375,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, } case UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP: return ReturnValue(false); + case UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP: + return ReturnValue(false); case UR_DEVICE_INFO_BFLOAT16_CONVERSIONS_NATIVE: { bool Supported = false; UR_RETURN_ON_FAILURE(hDevice->checkDeviceExtensions( diff --git a/unified-runtime/source/adapters/opencl/memory.cpp b/unified-runtime/source/adapters/opencl/memory.cpp index ca9e88d4f1ab1..1ba798bc1faa7 100644 --- a/unified-runtime/source/adapters/opencl/memory.cpp +++ b/unified-runtime/source/adapters/opencl/memory.cpp @@ -601,3 +601,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp(ur_context_handle_t, void *) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } + +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + ur_context_handle_t, ur_physical_mem_handle_t, void **, size_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCPutPhysMemHandleExp(ur_context_handle_t, const void *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCOpenPhysMemHandleExp(ur_context_handle_t, ur_device_handle_t, const void *, + size_t, ur_physical_mem_handle_t *) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urIPCClosePhysMemHandleExp(ur_context_handle_t, ur_physical_mem_handle_t) { + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} diff --git a/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp b/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp index 3205af2df209e..c9ce08ce199d4 100644 --- a/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/opencl/ur_interface_loader.cpp @@ -458,6 +458,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = urIPCClosePhysMemHandleExp; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp b/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp index 5f8bfca375b61..8bd4bf3ab7575 100644 --- a/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp +++ b/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp @@ -8159,6 +8159,178 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + auto pfnGetPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnGetPhysMemHandleExp; + + if (nullptr == pfnGetPhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_get_phys_mem_handle_exp_params_t params = { + &hContext, &hPhysMem, &ppIPCPhysMemHandleData, + &pIPCPhysMemHandleDataSizeRet}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP, + "urIPCGetPhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCGetPhysMemHandleExp\n"); + + ur_result_t result = pfnGetPhysMemHandleExp( + hContext, hPhysMem, ppIPCPhysMemHandleData, pIPCPhysMemHandleDataSizeRet); + + getContext()->notify_end(UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP, + "urIPCGetPhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_GET_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCGetPhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { + auto pfnPutPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; + + if (nullptr == pfnPutPhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_put_phys_mem_handle_exp_params_t params = {&hContext, + &pIPCPhysMemHandleData}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP, + "urIPCPutPhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCPutPhysMemHandleExp\n"); + + ur_result_t result = pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); + + getContext()->notify_end(UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP, + "urIPCPutPhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_PUT_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCPutPhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + auto pfnOpenPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnOpenPhysMemHandleExp; + + if (nullptr == pfnOpenPhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_open_phys_mem_handle_exp_params_t params = { + &hContext, &hDevice, &pIPCPhysMemHandleData, &ipcPhysMemHandleDataSize, + &phPhysMem}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP, + "urIPCOpenPhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCOpenPhysMemHandleExp\n"); + + ur_result_t result = + pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); + + getContext()->notify_end(UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP, + "urIPCOpenPhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_OPEN_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCOpenPhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + auto pfnClosePhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnClosePhysMemHandleExp; + + if (nullptr == pfnClosePhysMemHandleExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_ipc_close_phys_mem_handle_exp_params_t params = {&hContext, &hPhysMem}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP, + "urIPCClosePhysMemHandleExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urIPCClosePhysMemHandleExp\n"); + + ur_result_t result = pfnClosePhysMemHandleExp(hContext, hPhysMem); + + getContext()->notify_end(UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP, + "urIPCClosePhysMemHandleExp", ¶ms, &result, + instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_IPC_CLOSE_PHYS_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCClosePhysMemHandleExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -11731,6 +11903,22 @@ __urdlllocal ur_result_t UR_APICALL urGetIPCExpProcAddrTable( dditable.pfnCloseMemHandleExp = pDdiTable->pfnCloseMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur_tracing_layer::urIPCCloseMemHandleExp; + dditable.pfnGetPhysMemHandleExp = pDdiTable->pfnGetPhysMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = + ur_tracing_layer::urIPCGetPhysMemHandleExp; + + dditable.pfnPutPhysMemHandleExp = pDdiTable->pfnPutPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = + ur_tracing_layer::urIPCPutPhysMemHandleExp; + + dditable.pfnOpenPhysMemHandleExp = pDdiTable->pfnOpenPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = + ur_tracing_layer::urIPCOpenPhysMemHandleExp; + + dditable.pfnClosePhysMemHandleExp = pDdiTable->pfnClosePhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur_tracing_layer::urIPCClosePhysMemHandleExp; + return result; } /////////////////////////////////////////////////////////////////////////////// diff --git a/unified-runtime/source/loader/layers/validation/ur_valddi.cpp b/unified-runtime/source/loader/layers/validation/ur_valddi.cpp index 077e6b736846a..4368fce591d1f 100644 --- a/unified-runtime/source/loader/layers/validation/ur_valddi.cpp +++ b/unified-runtime/source/loader/layers/validation/ur_valddi.cpp @@ -9003,6 +9003,176 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + auto pfnGetPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnGetPhysMemHandleExp; + + if (nullptr == pfnGetPhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == ppIPCPhysMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == pIPCPhysMemHandleDataSizeRet) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hPhysMem) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hPhysMem)) { + URLOG_CTX_INVALID_REFERENCE(hPhysMem); + } + + ur_result_t result = pfnGetPhysMemHandleExp( + hContext, hPhysMem, ppIPCPhysMemHandleData, pIPCPhysMemHandleDataSizeRet); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { + auto pfnPutPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; + + if (nullptr == pfnPutPhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == pIPCPhysMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + ur_result_t result = pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + auto pfnOpenPhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnOpenPhysMemHandleExp; + + if (nullptr == pfnOpenPhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == phPhysMem) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == pIPCPhysMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); + } + + ur_result_t result = + pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + auto pfnClosePhysMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnClosePhysMemHandleExp; + + if (nullptr == pfnClosePhysMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hPhysMem) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hPhysMem)) { + URLOG_CTX_INVALID_REFERENCE(hPhysMem); + } + + ur_result_t result = pfnClosePhysMemHandleExp(hContext, hPhysMem); + + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -12502,6 +12672,22 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( dditable.pfnCloseMemHandleExp = pDdiTable->pfnCloseMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur_validation_layer::urIPCCloseMemHandleExp; + dditable.pfnGetPhysMemHandleExp = pDdiTable->pfnGetPhysMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = + ur_validation_layer::urIPCGetPhysMemHandleExp; + + dditable.pfnPutPhysMemHandleExp = pDdiTable->pfnPutPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = + ur_validation_layer::urIPCPutPhysMemHandleExp; + + dditable.pfnOpenPhysMemHandleExp = pDdiTable->pfnOpenPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = + ur_validation_layer::urIPCOpenPhysMemHandleExp; + + dditable.pfnClosePhysMemHandleExp = pDdiTable->pfnClosePhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur_validation_layer::urIPCClosePhysMemHandleExp; + return result; } diff --git a/unified-runtime/source/loader/loader.def.in b/unified-runtime/source/loader/loader.def.in index cc6ec03371edb..4a978ad4584c6 100644 --- a/unified-runtime/source/loader/loader.def.in +++ b/unified-runtime/source/loader/loader.def.in @@ -147,9 +147,13 @@ EXPORTS urGraphIsEmptyExp urGraphSetDestructionCallbackExp urIPCCloseMemHandleExp + urIPCClosePhysMemHandleExp urIPCGetMemHandleExp + urIPCGetPhysMemHandleExp urIPCOpenMemHandleExp + urIPCOpenPhysMemHandleExp urIPCPutMemHandleExp + urIPCPutPhysMemHandleExp urKernelCreate urKernelCreateWithNativeHandle urKernelGetGroupInfo @@ -397,9 +401,13 @@ EXPORTS urPrintImageFormat urPrintImageInfo urPrintIpcCloseMemHandleExpParams + urPrintIpcClosePhysMemHandleExpParams urPrintIpcGetMemHandleExpParams + urPrintIpcGetPhysMemHandleExpParams urPrintIpcOpenMemHandleExpParams + urPrintIpcOpenPhysMemHandleExpParams urPrintIpcPutMemHandleExpParams + urPrintIpcPutPhysMemHandleExpParams urPrintKernelArgLocalProperties urPrintKernelArgMemObjProperties urPrintKernelArgPointerProperties diff --git a/unified-runtime/source/loader/loader.map.in b/unified-runtime/source/loader/loader.map.in index 9ffb00211b6dc..0c06c67d8745a 100644 --- a/unified-runtime/source/loader/loader.map.in +++ b/unified-runtime/source/loader/loader.map.in @@ -147,9 +147,13 @@ urGraphIsEmptyExp; urGraphSetDestructionCallbackExp; urIPCCloseMemHandleExp; + urIPCClosePhysMemHandleExp; urIPCGetMemHandleExp; + urIPCGetPhysMemHandleExp; urIPCOpenMemHandleExp; + urIPCOpenPhysMemHandleExp; urIPCPutMemHandleExp; + urIPCPutPhysMemHandleExp; urKernelCreate; urKernelCreateWithNativeHandle; urKernelGetGroupInfo; @@ -397,9 +401,13 @@ urPrintImageFormat; urPrintImageInfo; urPrintIpcCloseMemHandleExpParams; + urPrintIpcClosePhysMemHandleExpParams; urPrintIpcGetMemHandleExpParams; + urPrintIpcGetPhysMemHandleExpParams; urPrintIpcOpenMemHandleExpParams; + urPrintIpcOpenPhysMemHandleExpParams; urPrintIpcPutMemHandleExpParams; + urPrintIpcPutPhysMemHandleExpParams; urPrintKernelArgLocalProperties; urPrintKernelArgMemObjProperties; urPrintKernelArgPointerProperties; diff --git a/unified-runtime/source/loader/ur_ldrddi.cpp b/unified-runtime/source/loader/ur_ldrddi.cpp index a68a92742892e..d522e58ee53e0 100644 --- a/unified-runtime/source/loader/ur_ldrddi.cpp +++ b/unified-runtime/source/loader/ur_ldrddi.cpp @@ -4643,6 +4643,92 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( return pfnCloseMemHandleExp(hContext, pMem); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCGetPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnGetPhysMemHandleExp = dditable->IPCExp.pfnGetPhysMemHandleExp; + if (nullptr == pfnGetPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnGetPhysMemHandleExp(hContext, hPhysMem, ppIPCPhysMemHandleData, + pIPCPhysMemHandleDataSizeRet); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnPutPhysMemHandleExp = dditable->IPCExp.pfnPutPhysMemHandleExp; + if (nullptr == pfnPutPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCOpenPhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnOpenPhysMemHandleExp = dditable->IPCExp.pfnOpenPhysMemHandleExp; + if (nullptr == pfnOpenPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCClosePhysMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnClosePhysMemHandleExp = dditable->IPCExp.pfnClosePhysMemHandleExp; + if (nullptr == pfnClosePhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnClosePhysMemHandleExp(hContext, hPhysMem); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Intercept function for urMemoryExportAllocExportableMemoryExp __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( @@ -6899,6 +6985,11 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetIPCExpProcAddrTable( pDdiTable->pfnPutMemHandleExp = ur_loader::urIPCPutMemHandleExp; pDdiTable->pfnOpenMemHandleExp = ur_loader::urIPCOpenMemHandleExp; pDdiTable->pfnCloseMemHandleExp = ur_loader::urIPCCloseMemHandleExp; + pDdiTable->pfnGetPhysMemHandleExp = ur_loader::urIPCGetPhysMemHandleExp; + pDdiTable->pfnPutPhysMemHandleExp = ur_loader::urIPCPutPhysMemHandleExp; + pDdiTable->pfnOpenPhysMemHandleExp = ur_loader::urIPCOpenPhysMemHandleExp; + pDdiTable->pfnClosePhysMemHandleExp = + ur_loader::urIPCClosePhysMemHandleExp; } else { // return pointers directly to platform's DDIs *pDdiTable = ur_loader::getContext()->platforms.front().dditable.IPCExp; diff --git a/unified-runtime/source/loader/ur_libapi.cpp b/unified-runtime/source/loader/ur_libapi.cpp index 1188978674036..c01853941ecfc 100644 --- a/unified-runtime/source/loader/ur_libapi.cpp +++ b/unified-runtime/source/loader/ur_libapi.cpp @@ -8640,6 +8640,151 @@ ur_result_t UR_APICALL urIPCCloseMemHandleExp( return exceptionToResult(std::current_exception()); } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Gets an inter-process handle for a physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCPhysMemHandleData` +/// + `NULL == pIPCPhysMemHandleDataSizeRet` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_ARGUMENT +/// + `hPhysMem` was not created with the +/// `::UR_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) try { + auto pfnGetPhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnGetPhysMemHandleExp; + if (nullptr == pfnGetPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnGetPhysMemHandleExp(hContext, hPhysMem, ppIPCPhysMemHandleData, + pIPCPhysMemHandleDataSizeRet); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Releases an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) try { + auto pfnPutPhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnPutPhysMemHandleExp; + if (nullptr == pfnPutPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnPutPhysMemHandleExp(hContext, pIPCPhysMemHandleData); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Opens an inter-process physical memory handle to get the +/// corresponding +/// physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPhysMem` +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcPhysMemHandleDataSize is not the same as the size of IPC +/// physical memory handle data +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) try { + auto pfnOpenPhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnOpenPhysMemHandleExp; + if (nullptr == pfnOpenPhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnOpenPhysMemHandleExp(hContext, hDevice, pIPCPhysMemHandleData, + ipcPhysMemHandleDataSize, phPhysMem); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Closes an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) try { + auto pfnClosePhysMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnClosePhysMemHandleExp; + if (nullptr == pfnClosePhysMemHandleExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnClosePhysMemHandleExp(hContext, hPhysMem); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Allocate an exportable memory region and return a pointer to that /// allocation. diff --git a/unified-runtime/source/loader/ur_print.cpp b/unified-runtime/source/loader/ur_print.cpp index 8c56e882cde85..1a748a44673f1 100644 --- a/unified-runtime/source/loader/ur_print.cpp +++ b/unified-runtime/source/loader/ur_print.cpp @@ -2203,6 +2203,38 @@ ur_result_t urPrintIpcCloseMemHandleExpParams( return str_copy(&ss, buffer, buff_size, out_size); } +ur_result_t urPrintIpcGetPhysMemHandleExpParams( + const struct ur_ipc_get_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintIpcPutPhysMemHandleExpParams( + const struct ur_ipc_put_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintIpcOpenPhysMemHandleExpParams( + const struct ur_ipc_open_phys_mem_handle_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintIpcClosePhysMemHandleExpParams( + const struct ur_ipc_close_phys_mem_handle_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + ur_result_t urPrintKernelCreateParams(const struct ur_kernel_create_params_t *params, char *buffer, const size_t buff_size, diff --git a/unified-runtime/source/ur_api.cpp b/unified-runtime/source/ur_api.cpp index d6b7be91aa80d..5dbdb0adeb260 100644 --- a/unified-runtime/source/ur_api.cpp +++ b/unified-runtime/source/ur_api.cpp @@ -7522,6 +7522,125 @@ ur_result_t UR_APICALL urIPCCloseMemHandleExp( return result; } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Gets an inter-process handle for a physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCPhysMemHandleData` +/// + `NULL == pIPCPhysMemHandleDataSizeRet` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_ARGUMENT +/// + `hPhysMem` was not created with the +/// `::UR_PHYSICAL_MEM_FLAG_ENABLE_IPC` flag +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCGetPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the physical memory object + ur_physical_mem_handle_t hPhysMem, + /// [out] a pointer to the IPC physical memory handle data + void **ppIPCPhysMemHandleData, + /// [out] size of the resulting IPC physical memory handle data + size_t *pIPCPhysMemHandleDataSizeRet) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Releases an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCPutPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Opens an inter-process physical memory handle to get the +/// corresponding +/// physical memory object +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPhysMem` +/// + `NULL == pIPCPhysMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcPhysMemHandleDataSize is not the same as the size of IPC +/// physical memory handle data +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCOpenPhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the physical memory was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC physical memory handle data obtained with + /// urIPCGetPhysMemHandleExp + const void *pIPCPhysMemHandleData, + /// [in] size of the IPC physical memory handle data + size_t ipcPhysMemHandleDataSize, + /// [out] pointer to the physical memory handle + ur_physical_mem_handle_t *phPhysMem) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Closes an inter-process physical memory handle +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// + `NULL == hPhysMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urIPCClosePhysMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] physical memory handle opened through urIPCOpenPhysMemHandleExp + ur_physical_mem_handle_t hPhysMem) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + /////////////////////////////////////////////////////////////////////////////// /// @brief Allocate an exportable memory region and return a pointer to that /// allocation. diff --git a/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt b/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt index 65c6c03108f15..dc79f146e5053 100644 --- a/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt +++ b/unified-runtime/test/conformance/virtual_memory/CMakeLists.txt @@ -14,4 +14,8 @@ add_conformance_devices_test(virtual_memory urVirtualMemReserve.cpp urVirtualMemSetAccess.cpp urVirtualMemUnmap.cpp + urIPCGetPhysMemHandleExp.cpp + urIPCPutPhysMemHandleExp.cpp + urIPCOpenPhysMemHandleExp.cpp + urIPCClosePhysMemHandleExp.cpp ) diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCClosePhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCClosePhysMemHandleExp.cpp new file mode 100644 index 0000000000000..4ba5783cdcf08 --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCClosePhysMemHandleExp.cpp @@ -0,0 +1,25 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCClosePhysMemHandleExpTest = urIPCOpenedPhysMemTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCClosePhysMemHandleExpTest); + +TEST_P(urIPCClosePhysMemHandleExpTest, Success) { + ASSERT_SUCCESS(urIPCClosePhysMemHandleExp(context, opened_physical_mem)); + // Prevent TearDown from attempting a second close. + opened_physical_mem = nullptr; +} + +TEST_P(urIPCClosePhysMemHandleExpTest, InvalidNullHandleContext) { + ASSERT_EQ_RESULT(urIPCClosePhysMemHandleExp(nullptr, opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCClosePhysMemHandleExpTest, InvalidNullHandlePhysMem) { + ASSERT_EQ_RESULT(urIPCClosePhysMemHandleExp(context, nullptr), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCGetPhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCGetPhysMemHandleExp.cpp new file mode 100644 index 0000000000000..22bad3ea36f94 --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCGetPhysMemHandleExp.cpp @@ -0,0 +1,99 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCGetPhysMemHandleExpTest = urIPCPhysMemTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCGetPhysMemHandleExpTest); + +TEST_P(urIPCGetPhysMemHandleExpTest, Success) { + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ur_result_t res = urIPCGetPhysMemHandleExp( + context, physical_mem, &ipc_handle_data, &ipc_handle_size); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "IPC physical memory handle export is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(ipc_handle_data, nullptr); + ASSERT_NE(ipc_handle_size, 0U); + ASSERT_SUCCESS(urIPCPutPhysMemHandleExp(context, ipc_handle_data)); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullHandleContext) { + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(nullptr, physical_mem, + &ipc_handle_data, &ipc_handle_size), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullHandlePhysMem) { + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, nullptr, &ipc_handle_data, + &ipc_handle_size), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullPointerIPCHandleData) { + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, physical_mem, nullptr, + &ipc_handle_size), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCGetPhysMemHandleExpTest, InvalidNullPointerIPCHandleDataSize) { + void *ipc_handle_data = nullptr; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, physical_mem, + &ipc_handle_data, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +// Fixture that creates a physical memory object WITHOUT the IPC-export flag. +struct urIPCGetPhysMemHandleExpNoIpcTest : uur::urVirtualMemGranularityTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urVirtualMemGranularityTest::SetUp()); + size = granularity * 256; + + ur_bool_t ipc_support = false; + ASSERT_SUCCESS( + urDeviceGetInfo(device, UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP, + sizeof(ur_bool_t), &ipc_support, nullptr)); + if (!ipc_support) { + GTEST_SKIP() << "IPC physical memory is not supported."; + } + + // Create physical memory without ENABLE_IPC flag. + ur_result_t res = + urPhysicalMemCreate(context, device, size, nullptr, &physical_mem); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "Physical memory creation is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(physical_mem, nullptr); + } + + void TearDown() override { + if (physical_mem) { + EXPECT_SUCCESS(urPhysicalMemRelease(physical_mem)); + } + uur::urVirtualMemGranularityTest::TearDown(); + } + + size_t size = 0; + ur_physical_mem_handle_t physical_mem = nullptr; +}; + +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCGetPhysMemHandleExpNoIpcTest); + +TEST_P(urIPCGetPhysMemHandleExpNoIpcTest, InvalidArgumentNoIpcFlag) { + // Exporting a handle for memory not created with ENABLE_IPC must fail. + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; + ASSERT_EQ_RESULT(urIPCGetPhysMemHandleExp(context, physical_mem, + &ipc_handle_data, &ipc_handle_size), + UR_RESULT_ERROR_INVALID_ARGUMENT); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCOpenPhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCOpenPhysMemHandleExp.cpp new file mode 100644 index 0000000000000..0358f69f1390a --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCOpenPhysMemHandleExp.cpp @@ -0,0 +1,57 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCOpenPhysMemHandleExpTest = urIPCPhysMemHandleTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCOpenPhysMemHandleExpTest); + +TEST_P(urIPCOpenPhysMemHandleExpTest, Success) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_SUCCESS(urIPCOpenPhysMemHandleExp( + context, device, ipc_handle_data, ipc_handle_size, &opened_physical_mem)); + ASSERT_NE(opened_physical_mem, nullptr); + ASSERT_SUCCESS(urIPCClosePhysMemHandleExp(context, opened_physical_mem)); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullHandleContext) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(nullptr, device, ipc_handle_data, + ipc_handle_size, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullHandleDevice) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, nullptr, ipc_handle_data, + ipc_handle_size, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullPointerIPCHandleData) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, device, nullptr, + ipc_handle_size, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidNullPointerPhysMem) { + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, device, ipc_handle_data, + ipc_handle_size, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} + +TEST_P(urIPCOpenPhysMemHandleExpTest, InvalidValue) { + ur_physical_mem_handle_t opened_physical_mem = nullptr; + // Pass a size that differs from the real handle data size to trigger + // UR_RESULT_ERROR_INVALID_VALUE. + ASSERT_EQ_RESULT(urIPCOpenPhysMemHandleExp(context, device, ipc_handle_data, + ipc_handle_size + 1, + &opened_physical_mem), + UR_RESULT_ERROR_INVALID_VALUE); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCPhysMemHandleExpFixtures.hpp b/unified-runtime/test/conformance/virtual_memory/urIPCPhysMemHandleExpFixtures.hpp new file mode 100644 index 0000000000000..93430e0dacc8e --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCPhysMemHandleExpFixtures.hpp @@ -0,0 +1,92 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once + +#include + +// Fixture that creates a physical memory object with the IPC-export flag. +struct urIPCPhysMemTest : uur::urVirtualMemGranularityTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urVirtualMemGranularityTest::SetUp()); + size = granularity * 256; + + ur_bool_t ipc_support = false; + ASSERT_SUCCESS( + urDeviceGetInfo(device, UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP, + sizeof(ur_bool_t), &ipc_support, nullptr)); + if (!ipc_support) { + GTEST_SKIP() << "IPC physical memory is not supported."; + } + + ur_physical_mem_properties_t properties{ + UR_STRUCTURE_TYPE_PHYSICAL_MEM_PROPERTIES, nullptr, + UR_PHYSICAL_MEM_FLAG_ENABLE_IPC}; + ur_result_t res = + urPhysicalMemCreate(context, device, size, &properties, &physical_mem); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "IPC physical memory is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(physical_mem, nullptr); + } + + void TearDown() override { + if (physical_mem) { + EXPECT_SUCCESS(urPhysicalMemRelease(physical_mem)); + } + uur::urVirtualMemGranularityTest::TearDown(); + } + + size_t size = 0; + ur_physical_mem_handle_t physical_mem = nullptr; +}; + +// Fixture that also acquires an IPC handle for the physical memory object. +struct urIPCPhysMemHandleTest : urIPCPhysMemTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urIPCPhysMemTest::SetUp()); + ur_result_t res = urIPCGetPhysMemHandleExp( + context, physical_mem, &ipc_handle_data, &ipc_handle_size); + if (res == UR_RESULT_ERROR_UNSUPPORTED_FEATURE) { + GTEST_SKIP() << "IPC physical memory handle export is not supported."; + } + ASSERT_SUCCESS(res); + ASSERT_NE(ipc_handle_data, nullptr); + ASSERT_NE(ipc_handle_size, 0U); + } + + void TearDown() override { + if (ipc_handle_data) { + EXPECT_SUCCESS(urIPCPutPhysMemHandleExp(context, ipc_handle_data)); + ipc_handle_data = nullptr; + } + urIPCPhysMemTest::TearDown(); + } + + void *ipc_handle_data = nullptr; + size_t ipc_handle_size = 0; +}; + +// Fixture that also opens the IPC handle to produce a second physical_mem. +struct urIPCOpenedPhysMemTest : urIPCPhysMemHandleTest { + void SetUp() override { + UUR_RETURN_ON_FATAL_FAILURE(urIPCPhysMemHandleTest::SetUp()); + ASSERT_SUCCESS(urIPCOpenPhysMemHandleExp(context, device, ipc_handle_data, + ipc_handle_size, + &opened_physical_mem)); + ASSERT_NE(opened_physical_mem, nullptr); + } + + void TearDown() override { + if (opened_physical_mem) { + EXPECT_SUCCESS(urIPCClosePhysMemHandleExp(context, opened_physical_mem)); + opened_physical_mem = nullptr; + } + urIPCPhysMemHandleTest::TearDown(); + } + + ur_physical_mem_handle_t opened_physical_mem = nullptr; +}; diff --git a/unified-runtime/test/conformance/virtual_memory/urIPCPutPhysMemHandleExp.cpp b/unified-runtime/test/conformance/virtual_memory/urIPCPutPhysMemHandleExp.cpp new file mode 100644 index 0000000000000..7a824b7ab6efb --- /dev/null +++ b/unified-runtime/test/conformance/virtual_memory/urIPCPutPhysMemHandleExp.cpp @@ -0,0 +1,25 @@ +// Part of the LLVM Project, under the Apache License v2.0 with LLVM +// Exceptions. See https://llvm.org/LICENSE.txt for license information. +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "urIPCPhysMemHandleExpFixtures.hpp" + +using urIPCPutPhysMemHandleExpTest = urIPCPhysMemHandleTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE(urIPCPutPhysMemHandleExpTest); + +TEST_P(urIPCPutPhysMemHandleExpTest, Success) { + ASSERT_SUCCESS(urIPCPutPhysMemHandleExp(context, ipc_handle_data)); + // Prevent TearDown from attempting a second put. + ipc_handle_data = nullptr; +} + +TEST_P(urIPCPutPhysMemHandleExpTest, InvalidNullHandleContext) { + ASSERT_EQ_RESULT(urIPCPutPhysMemHandleExp(nullptr, ipc_handle_data), + UR_RESULT_ERROR_INVALID_NULL_HANDLE); +} + +TEST_P(urIPCPutPhysMemHandleExpTest, InvalidNullPointerIPCHandleData) { + ASSERT_EQ_RESULT(urIPCPutPhysMemHandleExp(context, nullptr), + UR_RESULT_ERROR_INVALID_NULL_POINTER); +} diff --git a/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp b/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp index f7c9538d79294..77ef56462f14c 100644 --- a/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp +++ b/unified-runtime/test/conformance/virtual_memory/urPhysicalMemCreate.cpp @@ -55,7 +55,7 @@ using urPhysicalMemCreateWithFlagsParamTest = uur::urPhysicalMemTestWithParam; UUR_DEVICE_TEST_SUITE_WITH_PARAM( urPhysicalMemCreateWithFlagsParamTest, - ::testing::Values(UR_PHYSICAL_MEM_FLAG_TBD), + ::testing::Values(UR_PHYSICAL_MEM_FLAG_ENABLE_IPC), uur::deviceTestWithParamPrinter); TEST_P(urPhysicalMemCreateWithFlagsParamTest, Success) { diff --git a/unified-runtime/tools/urinfo/urinfo.hpp b/unified-runtime/tools/urinfo/urinfo.hpp index d5b15e5aaa00b..fe99cf3ef5f00 100644 --- a/unified-runtime/tools/urinfo/urinfo.hpp +++ b/unified-runtime/tools/urinfo/urinfo.hpp @@ -466,6 +466,9 @@ inline void printDeviceInfos(ur_device_handle_t hDevice, std::cout << prefix; printDeviceInfo(hDevice, UR_DEVICE_INFO_IPC_MEMORY_SUPPORT_EXP); std::cout << prefix; + printDeviceInfo(hDevice, + UR_DEVICE_INFO_IPC_PHYSICAL_MEMORY_SUPPORT_EXP); + std::cout << prefix; printDeviceInfo(hDevice, UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_SUPPORT_EXP); std::cout << prefix;