[UR] Add urUSMContextMemcpyExp API and basic l0 implementation.

Author: aarongreig

unified-runtime/include/ur_api.h

@@ -453,6 +453,8 @@ typedef enum ur_function_t {
   UR_FUNCTION_USM_POOL_TRIM_TO_EXP = 261,
   /// Enumerator for ::urUSMPoolGetInfoExp
   UR_FUNCTION_USM_POOL_GET_INFO_EXP = 262,
+  /// Enumerator for ::urUSMContextMemcpyExp
+  UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP = 264,
   /// @cond
   UR_FUNCTION_FORCE_UINT32 = 0x7fffffff
   /// @endcond
@@ -2320,6 +2322,8 @@ typedef enum ur_device_info_t {
   /// [::ur_bool_t] returns true if the device supports enqueueing of
   /// allocations and frees.
   UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_EXP = 0x2050,
+  /// [::ur_bool_t] returns true if the device supports ::urUSMContextMemcpy
+  UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP = 0x7000,
   /// @cond
   UR_DEVICE_INFO_FORCE_UINT32 = 0x7fffffff
   /// @endcond
@@ -2345,7 +2349,7 @@ typedef enum ur_device_info_t {
 ///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
 ///         + `NULL == hDevice`
 ///     - ::UR_RESULT_ERROR_INVALID_ENUMERATION
-///         + `::UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_EXP < propName`
+///         + `::UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP < propName`
 ///     - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION
 ///         + If `propName` is not supported by the adapter.
 ///     - ::UR_RESULT_ERROR_INVALID_SIZE
@@ -12153,6 +12157,39 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramLinkExp(
     /// [out][alloc] pointer to handle of program object created.
     ur_program_handle_t *phProgram);
 
+#if !defined(__GNUC__)
+#pragma endregion
+#endif
+// Intel 'oneAPI' Unified Runtime Experimental APIs for USM Context Memcpy
+#if !defined(__GNUC__)
+#pragma region usm_context_memcpy_(experimental)
+#endif
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Enable access to peer device memory
+///
+/// @returns
+///     - ::UR_RESULT_SUCCESS
+///     - ::UR_RESULT_ERROR_UNINITIALIZED
+///     - ::UR_RESULT_ERROR_DEVICE_LOST
+///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
+///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
+///         + `NULL == hContext`
+///     - ::UR_RESULT_ERROR_INVALID_NULL_POINTER
+///         + `NULL == pDst`
+///         + `NULL == pSrc`
+///     - ::UR_RESULT_SUCCESS
+///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
+UR_APIEXPORT ur_result_t UR_APICALL urUSMContextMemcpyExp(
+    /// [in] Context associated with the device(s) that own the allocations
+    /// `pSrc` and `pDst`.
+    ur_context_handle_t hContext,
+    /// [in] Destination pointer to copy to.
+    void *pDst,
+    /// [in] Source pointer to copy from.
+    const void *pSrc,
+    /// [in] Size in bytes to be copied.
+    size_t size);
+
 #if !defined(__GNUC__)
 #pragma endregion
 #endif
@@ -14282,6 +14319,17 @@ typedef struct ur_usm_pitched_alloc_exp_params_t {
   size_t **ppResultPitch;
 } ur_usm_pitched_alloc_exp_params_t;
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for urUSMContextMemcpyExp
+/// @details Each entry is a pointer to the parameter passed to the function;
+///     allowing the callback the ability to modify the parameter's value
+typedef struct ur_usm_context_memcpy_exp_params_t {
+  ur_context_handle_t *phContext;
+  void **ppDst;
+  const void **ppSrc;
+  size_t *psize;
+} ur_usm_context_memcpy_exp_params_t;
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Function parameters for urUSMImportExp
 /// @details Each entry is a pointer to the parameter passed to the function;

unified-runtime/include/ur_api_funcs.def

@@ -157,6 +157,7 @@ _UR_API(urUSMPoolSetDevicePoolExp)
 _UR_API(urUSMPoolGetDevicePoolExp)
 _UR_API(urUSMPoolTrimToExp)
 _UR_API(urUSMPitchedAllocExp)
+_UR_API(urUSMContextMemcpyExp)
 _UR_API(urUSMImportExp)
 _UR_API(urUSMReleaseExp)
 _UR_API(urBindlessImagesUnsampledImageHandleDestroyExp)

unified-runtime/include/ur_ddi.h

@@ -1338,6 +1338,11 @@ typedef ur_result_t(UR_APICALL *ur_pfnUSMPitchedAllocExp_t)(
     ur_context_handle_t, ur_device_handle_t, const ur_usm_desc_t *,
     ur_usm_pool_handle_t, size_t, size_t, size_t, void **, size_t *);
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function-pointer for urUSMContextMemcpyExp
+typedef ur_result_t(UR_APICALL *ur_pfnUSMContextMemcpyExp_t)(
+    ur_context_handle_t, void *, const void *, size_t);
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Function-pointer for urUSMImportExp
 typedef ur_result_t(UR_APICALL *ur_pfnUSMImportExp_t)(ur_context_handle_t,
@@ -1360,6 +1365,7 @@ typedef struct ur_usm_exp_dditable_t {
   ur_pfnUSMPoolGetDevicePoolExp_t pfnPoolGetDevicePoolExp;
   ur_pfnUSMPoolTrimToExp_t pfnPoolTrimToExp;
   ur_pfnUSMPitchedAllocExp_t pfnPitchedAllocExp;
+  ur_pfnUSMContextMemcpyExp_t pfnContextMemcpyExp;
   ur_pfnUSMImportExp_t pfnImportExp;
   ur_pfnUSMReleaseExp_t pfnReleaseExp;
 } ur_usm_exp_dditable_t;

unified-runtime/include/ur_print.h

@@ -2897,6 +2897,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintUsmPitchedAllocExpParams(
     const struct ur_usm_pitched_alloc_exp_params_t *params, char *buffer,
     const size_t buff_size, size_t *out_size);
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print ur_usm_context_memcpy_exp_params_t struct
+/// @returns
+///     - ::UR_RESULT_SUCCESS
+///     - ::UR_RESULT_ERROR_INVALID_SIZE
+///         - `buff_size < out_size`
+UR_APIEXPORT ur_result_t UR_APICALL urPrintUsmContextMemcpyExpParams(
+    const struct ur_usm_context_memcpy_exp_params_t *params, char *buffer,
+    const size_t buff_size, size_t *out_size);
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Print ur_usm_import_exp_params_t struct
 /// @returns

unified-runtime/include/ur_print.hpp

@@ -1222,6 +1222,9 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_function_t value) {
   case UR_FUNCTION_USM_POOL_GET_INFO_EXP:
     os << "UR_FUNCTION_USM_POOL_GET_INFO_EXP";
     break;
+  case UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP:
+    os << "UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP";
+    break;
   default:
     os << "unknown enumerator";
     break;
@@ -3032,6 +3035,9 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_device_info_t value) {
   case UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_EXP:
     os << "UR_DEVICE_INFO_ASYNC_USM_ALLOCATIONS_EXP";
     break;
+  case UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP:
+    os << "UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP";
+    break;
   default:
     os << "unknown enumerator";
     break;
@@ -5036,6 +5042,19 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr,
 
     os << ")";
   } break;
+  case UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP: {
+    const ur_bool_t *tptr = (const ur_bool_t *)ptr;
+    if (sizeof(ur_bool_t) > size) {
+      os << "invalid size (is: " << size
+         << ", expected: >=" << sizeof(ur_bool_t) << ")";
+      return UR_RESULT_ERROR_INVALID_SIZE;
+    }
+    os << (const void *)(tptr) << " (";
+
+    os << *tptr;
+
+    os << ")";
+  } break;
   default:
     os << "unknown enumerator";
     return UR_RESULT_ERROR_INVALID_ENUMERATION;
@@ -17459,6 +17478,36 @@ inline std::ostream &operator<<(
   return os;
 }
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print operator for the ur_usm_context_memcpy_exp_params_t type
+/// @returns
+///     std::ostream &
+inline std::ostream &operator<<(
+    std::ostream &os,
+    [[maybe_unused]] const struct ur_usm_context_memcpy_exp_params_t *params) {
+
+  os << ".hContext = ";
+
+  ur::details::printPtr(os, *(params->phContext));
+
+  os << ", ";
+  os << ".pDst = ";
+
+  ur::details::printPtr(os, *(params->ppDst));
+
+  os << ", ";
+  os << ".pSrc = ";
+
+  ur::details::printPtr(os, *(params->ppSrc));
+
+  os << ", ";
+  os << ".size = ";
+
+  os << *(params->psize);
+
+  return os;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Print operator for the ur_usm_import_exp_params_t type
 /// @returns
@@ -20708,6 +20757,9 @@ inline ur_result_t UR_APICALL printFunctionParams(std::ostream &os,
   case UR_FUNCTION_USM_PITCHED_ALLOC_EXP: {
     os << (const struct ur_usm_pitched_alloc_exp_params_t *)params;
   } break;
+  case UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP: {
+    os << (const struct ur_usm_context_memcpy_exp_params_t *)params;
+  } break;
   case UR_FUNCTION_USM_IMPORT_EXP: {
     os << (const struct ur_usm_import_exp_params_t *)params;
   } break;

unified-runtime/scripts/core/EXP-USM-CONTEXT-MEMCPY.rst

@@ -0,0 +1,63 @@
+<%
+    OneApi=tags['$OneApi']
+    x=tags['$x']
+    X=x.upper()
+%>
+
+.. _experimental-usm-context-memcpy:
+
+================================================================================
+USM Context Memcpy
+================================================================================
+
+.. warning::
+
+    Experimental features:
+
+    *   May be replaced, updated, or removed at any time.
+    *   Do not require maintaining API/ABI stability of their own additions over
+        time.
+    *   Do not require conformance testing of their own additions.
+
+
+Motivation
+--------------------------------------------------------------------------------
+
+In order to support device globals there's a need for a blocking USM write
+operation that doesn't need a queue. This is to facilitate fast initialization
+of the device global memory via native APIs that enable this kind of operation.
+
+API
+--------------------------------------------------------------------------------
+
+Enums
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ${x}_device_info_t
+    * ${X}_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP
+
+Functions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* ${x}USMContextMemcpyExp
+
+Changelog
+--------------------------------------------------------------------------------
+
++-----------+---------------------------+
+| Revision  | Changes                   |
++===========+===========================+
+| 1.0       | Initial Draft             |
++-----------+---------------------------+
+
+
+Support
+--------------------------------------------------------------------------------
+
+Adapters which support this experimental feature *must* return true for the new
+``${X}_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP`` device info query.
+
+
+Contributors
+--------------------------------------------------------------------------------
+
+* Aaron Greig `[email protected] <[email protected]>`

unified-runtime/scripts/core/exp-usm-context-memcpy.yml

@@ -0,0 +1,45 @@
+#
+# Copyright (C) 2025 Intel Corporation
+#
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# See YaML.md for syntax definition
+#
+--- #--------------------------------------------------------------------------
+type: header
+desc: "Intel $OneApi Unified Runtime Experimental APIs for USM Context Memcpy"
+ordinal: "99"
+--- #--------------------------------------------------------------------------
+type: enum
+extend: true
+typed_etors: true
+desc: "Extension enums to $x_device_info_t to support $xUSMContextMemcpy"
+name: $x_device_info_t
+etors:
+    - name: USM_CONTEXT_MEMCPY_SUPPORT_EXP
+      value: "0x7000"
+      desc: "[$x_bool_t] returns true if the device supports $xUSMContextMemcpyExp"
+--- #--------------------------------------------------------------------------
+type: function
+desc: "Perform a synchronous, blocking memcpy operation between USM allocations."
+class: $xUSM
+name: ContextMemcpyExp
+ordinal: "0"
+params:
+    - type: $x_context_handle_t
+      name: hContext
+      desc: "[in] Context associated with the device(s) that own the allocations `pSrc` and `pDst`."
+    - type: void*
+      name: pDst
+      desc: "[in] Destination pointer to copy to."
+    - type: const void*
+      name: pSrc
+      desc: "[in] Source pointer to copy from."
+    - type: size_t
+      name: size
+      desc: "[in] Size in bytes to be copied."
+returns:
+    - $X_RESULT_SUCCESS
+    - $X_RESULT_ERROR_ADAPTER_SPECIFIC

unified-runtime/scripts/core/registry.yml

@@ -637,6 +637,9 @@ etors:
 - name: USM_POOL_GET_INFO_EXP
   desc: Enumerator for $xUSMPoolGetInfoExp
   value: '262'
+- name: USM_CONTEXT_MEMCPY_EXP
+  desc: Enumerator for $xUSMContextMemcpyExp
+  value: '264'
 ---
 type: enum
 desc: Defines structure types

unified-runtime/source/adapters/cuda/device.cpp

@@ -1118,6 +1118,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
   }
   case UR_DEVICE_INFO_LOW_POWER_EVENTS_EXP:
     return ReturnValue(false);
+  case UR_DEVICE_INFO_USM_CONTEXT_MEMCPY_SUPPORT_EXP:
+    return ReturnValue(false);
   default:
     break;
   }

unified-runtime/source/adapters/cuda/ur_interface_loader.cpp

@@ -369,6 +369,7 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetUSMExpProcAddrTable(
   pDdiTable->pfnPoolSetDevicePoolExp = urUSMPoolSetDevicePoolExp;
   pDdiTable->pfnPoolGetDevicePoolExp = urUSMPoolGetDevicePoolExp;
   pDdiTable->pfnPoolTrimToExp = urUSMPoolTrimToExp;
+  pDdiTable->pfnContextMemcpyExp = urUSMContextMemcpyExp;
   return UR_RESULT_SUCCESS;
 }