Merge branch 'main' into georgi/hip-max-version-for-usmmemcpy2d-workaround

Author: Georgi Mirazchiyski

CMakeLists.txt

@@ -137,7 +137,7 @@ if(UR_ENABLE_TRACING)
 
     if (UR_BUILD_XPTI_LIBS)
         # fetch xpti proxy library for the tracing layer
-        FetchContentSparse_Declare(xpti https://github.com/intel/llvm.git "sycl-nightly/20230703" "xpti")
+        FetchContentSparse_Declare(xpti https://github.com/intel/llvm.git "nightly-2024-10-22" "xpti")
         FetchContent_MakeAvailable(xpti)
 
         # set -fPIC for xpti since we are linking it with a shared library
@@ -149,7 +149,7 @@ if(UR_ENABLE_TRACING)
         set(XPTI_DIR ${xpti_SOURCE_DIR})
         set(XPTI_ENABLE_TESTS OFF CACHE INTERNAL "Turn off xptifw tests")
 
-        FetchContentSparse_Declare(xptifw https://github.com/intel/llvm.git "sycl-nightly/20230703" "xptifw")
+        FetchContentSparse_Declare(xptifw https://github.com/intel/llvm.git "nightly-2024-10-22" "xptifw")
 
         FetchContent_MakeAvailable(xptifw)
 

examples/collector/collector.cpp

@@ -125,8 +125,7 @@ XPTI_CALLBACK_API void xptiTraceInit(unsigned int major_version,
         return;
     }
     if (std::string_view(stream_name) != UR_STREAM_NAME) {
-        std::cout << "Invalid stream name: " << stream_name << ". Expected "
-                  << UR_STREAM_NAME << ". Aborting." << std::endl;
+        // we expect ur.call, but this can also be xpti.framework.
         return;
     }
 

include/ur_api.h

@@ -1639,6 +1639,9 @@ typedef enum ur_device_info_t {
     UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT = 118,                    ///< [::ur_bool_t] return true if the device supports the
                                                                      ///< `EnqueueDeviceGlobalVariableWrite` and
                                                                      ///< `EnqueueDeviceGlobalVariableRead` entry points.
+    UR_DEVICE_INFO_USM_POOL_SUPPORT = 119,                           ///< [::ur_bool_t] return true if the device supports USM pooling. Pertains
+                                                                     ///< to the `USMPool` entry points and usage of the `pool` parameter of the
+                                                                     ///< USM alloc entry points.
     UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP = 0x1000,              ///< [::ur_bool_t] Returns true if the device supports the use of
                                                                      ///< command-buffers.
     UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP = 0x1001,  ///< [::ur_device_command_buffer_update_capability_flags_t] Command-buffer
@@ -3503,7 +3506,6 @@ typedef struct ur_usm_pool_limits_desc_t {
 /// @brief USM allocate host memory
 ///
 /// @details
-///     - This function must support memory pooling.
 ///     - If pUSMDesc is not NULL and pUSMDesc->pool is not NULL the allocation
 ///       will be served from a specified memory pool.
 ///     - Otherwise, the behavior is implementation-defined.
@@ -3536,6 +3538,8 @@ typedef struct ur_usm_pool_limits_desc_t {
 ///         + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE.
 ///     - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
 ///     - ::UR_RESULT_ERROR_OUT_OF_RESOURCES
+///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
+///         + If any device associated with `hContext` reports `false` for ::UR_DEVICE_INFO_USM_POOL_SUPPORT
 UR_APIEXPORT ur_result_t UR_APICALL
 urUSMHostAlloc(
     ur_context_handle_t hContext,  ///< [in] handle of the context object
@@ -3549,7 +3553,6 @@ urUSMHostAlloc(
 /// @brief USM allocate device memory
 ///
 /// @details
-///     - This function must support memory pooling.
 ///     - If pUSMDesc is not NULL and pUSMDesc->pool is not NULL the allocation
 ///       will be served from a specified memory pool.
 ///     - Otherwise, the behavior is implementation-defined.
@@ -3583,6 +3586,8 @@ urUSMHostAlloc(
 ///         + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE.
 ///     - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
 ///     - ::UR_RESULT_ERROR_OUT_OF_RESOURCES
+///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
+///         + If any device associated with `hContext` reports `false` for ::UR_DEVICE_INFO_USM_POOL_SUPPORT
 UR_APIEXPORT ur_result_t UR_APICALL
 urUSMDeviceAlloc(
     ur_context_handle_t hContext,  ///< [in] handle of the context object
@@ -3597,7 +3602,6 @@ urUSMDeviceAlloc(
 /// @brief USM allocate shared memory
 ///
 /// @details
-///     - This function must support memory pooling.
 ///     - If pUSMDesc is not NULL and pUSMDesc->pool is not NULL the allocation
 ///       will be served from a specified memory pool.
 ///     - Otherwise, the behavior is implementation-defined.
@@ -3632,6 +3636,8 @@ urUSMDeviceAlloc(
 ///         + If `UR_DEVICE_INFO_USM_SINGLE_SHARED_SUPPORT` and `UR_DEVICE_INFO_USM_CROSS_SHARED_SUPPORT` are both false.
 ///     - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
 ///     - ::UR_RESULT_ERROR_OUT_OF_RESOURCES
+///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
+///         + If any device associated with `hContext` reports `false` for ::UR_DEVICE_INFO_USM_POOL_SUPPORT
 UR_APIEXPORT ur_result_t UR_APICALL
 urUSMSharedAlloc(
     ur_context_handle_t hContext,  ///< [in] handle of the context object
@@ -3713,6 +3719,8 @@ urUSMGetMemAllocInfo(
 ///         + `::UR_USM_POOL_FLAGS_MASK & pPoolDesc->flags`
 ///     - ::UR_RESULT_ERROR_INVALID_VALUE
 ///     - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
+///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
+///         + If any device associated with `hContext` reports `false` for ::UR_DEVICE_INFO_USM_POOL_SUPPORT
 UR_APIEXPORT ur_result_t UR_APICALL
 urUSMPoolCreate(
     ur_context_handle_t hContext,  ///< [in] handle of the context object
@@ -3731,6 +3739,7 @@ urUSMPoolCreate(
 ///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
 ///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
 ///         + `NULL == pPool`
+///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
 UR_APIEXPORT ur_result_t UR_APICALL
 urUSMPoolRetain(
     ur_usm_pool_handle_t pPool ///< [in][retain] pointer to USM memory pool
@@ -3753,6 +3762,7 @@ urUSMPoolRetain(
 ///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
 ///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
 ///         + `NULL == pPool`
+///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
 UR_APIEXPORT ur_result_t UR_APICALL
 urUSMPoolRelease(
     ur_usm_pool_handle_t pPool ///< [in][release] pointer to USM memory pool
@@ -3794,6 +3804,7 @@ typedef enum ur_usm_pool_info_t {
 ///         + `pPropValue == NULL && pPropSizeRet == NULL`
 ///     - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
 ///     - ::UR_RESULT_ERROR_OUT_OF_RESOURCES
+///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
 UR_APIEXPORT ur_result_t UR_APICALL
 urUSMPoolGetInfo(
     ur_usm_pool_handle_t hPool,  ///< [in] handle of the USM memory pool

include/ur_print.hpp

@@ -2550,6 +2550,9 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_device_info_t value) {
     case UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT:
         os << "UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT";
         break;
+    case UR_DEVICE_INFO_USM_POOL_SUPPORT:
+        os << "UR_DEVICE_INFO_USM_POOL_SUPPORT";
+        break;
     case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP:
         os << "UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP";
         break;
@@ -4052,6 +4055,18 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, ur_device_info
 
         os << ")";
     } break;
+    case UR_DEVICE_INFO_USM_POOL_SUPPORT: {
+        const ur_bool_t *tptr = (const ur_bool_t *)ptr;
+        if (sizeof(ur_bool_t) > size) {
+            os << "invalid size (is: " << size << ", expected: >=" << sizeof(ur_bool_t) << ")";
+            return UR_RESULT_ERROR_INVALID_SIZE;
+        }
+        os << (const void *)(tptr) << " (";
+
+        os << *tptr;
+
+        os << ")";
+    } break;
     case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP: {
         const ur_bool_t *tptr = (const ur_bool_t *)ptr;
         if (sizeof(ur_bool_t) > size) {

scripts/core/device.yml

@@ -441,6 +441,8 @@ etors:
       desc: "[$x_device_handle_t] The composite device containing this component device."
     - name: GLOBAL_VARIABLE_SUPPORT
       desc: "[$x_bool_t] return true if the device supports the `EnqueueDeviceGlobalVariableWrite` and `EnqueueDeviceGlobalVariableRead` entry points."
+    - name: USM_POOL_SUPPORT
+      desc: "[$x_bool_t] return true if the device supports USM pooling. Pertains to the `USMPool` entry points and usage of the `pool` parameter of the USM alloc entry points."
 --- #--------------------------------------------------------------------------
 type: function
 desc: "Retrieves various information about device"

scripts/core/usm.yml

@@ -229,7 +229,6 @@ class: $xUSM
 name: HostAlloc
 ordinal: "0"
 details:
-  - "This function must support memory pooling."
   - "If pUSMDesc is not NULL and pUSMDesc->pool is not NULL the allocation will be served from a specified memory pool."
   - "Otherwise, the behavior is implementation-defined."
   - "Allocations served from different memory pools must be isolated and must not reside on the same page."
@@ -264,14 +263,15 @@ returns:
       - "`size` is greater than $X_DEVICE_INFO_MAX_MEM_ALLOC_SIZE."
     - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY
     - $X_RESULT_ERROR_OUT_OF_RESOURCES
+    - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:
+        - "If any device associated with `hContext` reports `false` for $X_DEVICE_INFO_USM_POOL_SUPPORT"
 --- #--------------------------------------------------------------------------
 type: function
 desc: "USM allocate device memory"
 class: $xUSM
 name: DeviceAlloc
 ordinal: "0"
 details:
-  - "This function must support memory pooling."
   - "If pUSMDesc is not NULL and pUSMDesc->pool is not NULL the allocation will be served from a specified memory pool."
   - "Otherwise, the behavior is implementation-defined."
   - "Allocations served from different memory pools must be isolated and must not reside on the same page."
@@ -309,14 +309,15 @@ returns:
       - "`size` is greater than $X_DEVICE_INFO_MAX_MEM_ALLOC_SIZE."
     - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY
     - $X_RESULT_ERROR_OUT_OF_RESOURCES
+    - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:
+        - "If any device associated with `hContext` reports `false` for $X_DEVICE_INFO_USM_POOL_SUPPORT"
 --- #--------------------------------------------------------------------------
 type: function
 desc: "USM allocate shared memory"
 class: $xUSM
 name: SharedAlloc
 ordinal: "0"
 details:
-  - "This function must support memory pooling."
   - "If pUSMDesc is not NULL and pUSMDesc->pool is not NULL the allocation will be served from a specified memory pool."
   - "Otherwise, the behavior is implementation-defined."
   - "Allocations served from different memory pools must be isolated and must not reside on the same page."
@@ -355,6 +356,8 @@ returns:
       - "If `UR_DEVICE_INFO_USM_SINGLE_SHARED_SUPPORT` and `UR_DEVICE_INFO_USM_CROSS_SHARED_SUPPORT` are both false."
     - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY
     - $X_RESULT_ERROR_OUT_OF_RESOURCES
+    - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:
+        - "If any device associated with `hContext` reports `false` for $X_DEVICE_INFO_USM_POOL_SUPPORT"
 --- #--------------------------------------------------------------------------
 type: function
 desc: "Free the USM memory object"
@@ -424,6 +427,8 @@ returns:
     - $X_RESULT_ERROR_INVALID_NULL_HANDLE
     - $X_RESULT_ERROR_INVALID_VALUE
     - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY
+    - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:
+        - "If any device associated with `hContext` reports `false` for $X_DEVICE_INFO_USM_POOL_SUPPORT"
 --- #--------------------------------------------------------------------------
 type: function
 desc: "Get a reference to the pool handle. Increment its reference count"
@@ -436,6 +441,7 @@ params:
       desc: "[in][retain] pointer to USM memory pool"
 returns:
     - $X_RESULT_ERROR_INVALID_NULL_HANDLE
+    - $X_RESULT_ERROR_UNSUPPORTED_FEATURE
 --- #--------------------------------------------------------------------------
 type: function
 desc: "Decrement the pool's reference count and delete the pool if the reference count becomes zero."
@@ -452,6 +458,7 @@ params:
       desc: "[in][release] pointer to USM memory pool"
 returns:
     - $X_RESULT_ERROR_INVALID_NULL_HANDLE
+    - $X_RESULT_ERROR_UNSUPPORTED_FEATURE
 --- #--------------------------------------------------------------------------
 type: enum
 desc: "Get USM memory pool information"
@@ -499,3 +506,4 @@ returns:
         - "`pPropValue == NULL && pPropSizeRet == NULL`"
     - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY
     - $X_RESULT_ERROR_OUT_OF_RESOURCES
+    - $X_RESULT_ERROR_UNSUPPORTED_FEATURE

source/adapters/cuda/device.cpp

@@ -1076,6 +1076,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
     return ReturnValue(static_cast<ur_bool_t>(false));
   case UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT:
     return ReturnValue(static_cast<ur_bool_t>(true));
+  case UR_DEVICE_INFO_USM_POOL_SUPPORT:
+    return ReturnValue(static_cast<ur_bool_t>(true));
   case UR_DEVICE_INFO_COMPONENT_DEVICES:
   case UR_DEVICE_INFO_COMPOSITE_DEVICE:
   case UR_DEVICE_INFO_MAX_READ_WRITE_IMAGE_ARGS:

source/adapters/hip/device.cpp

@@ -889,6 +889,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
 
   case UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT:
     return ReturnValue(ur_bool_t{false});
+  case UR_DEVICE_INFO_USM_POOL_SUPPORT:
+    return ReturnValue(ur_bool_t{true});
   // TODO: Investigate if this information is available on HIP.
   case UR_DEVICE_INFO_COMPONENT_DEVICES:
   case UR_DEVICE_INFO_COMPOSITE_DEVICE:

source/adapters/level_zero/command_buffer.cpp

@@ -784,19 +784,10 @@ ur_result_t urCommandBufferAppendUSMMemcpyExp(
   std::ignore = Event;
   std::ignore = Command;
 
-  bool PreferCopyEngine = !IsDevicePointer(CommandBuffer->Context, Src) ||
-                          !IsDevicePointer(CommandBuffer->Context, Dst);
-  // For better performance, Copy Engines are not preferred given Shared
-  // pointers on DG2.
-  if (CommandBuffer->Device->isDG2() &&
-      (IsSharedPointer(CommandBuffer->Context, Src) ||
-       IsSharedPointer(CommandBuffer->Context, Dst))) {
-    PreferCopyEngine = false;
-  }
-  PreferCopyEngine |= UseCopyEngineForD2DCopy;
-
   return enqueueCommandBufferMemCopyHelper(
-      UR_COMMAND_USM_MEMCPY, CommandBuffer, Dst, Src, Size, PreferCopyEngine,
+      UR_COMMAND_USM_MEMCPY, CommandBuffer, Dst, Src, Size,
+      PreferCopyEngineUsage(CommandBuffer->Device, CommandBuffer->Context, Src,
+                            Dst),
       NumSyncPointsInWaitList, SyncPointWaitList, SyncPoint);
 }
 

source/adapters/level_zero/device.cpp

@@ -1053,9 +1053,7 @@ ur_result_t urDeviceGetInfo(
   case UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP:
     return ReturnValue(false);
   case UR_DEVICE_INFO_BINDLESS_IMAGES_SUPPORT_EXP: {
-    bool DeviceIsDG2OrNewer =
-        Device->ZeDeviceIpVersionExt->ipVersion >= 0x030dc000;
-    return ReturnValue(DeviceIsDG2OrNewer &&
+    return ReturnValue(Device->isIntelDG2OrNewer() &&
                        Device->ZeDeviceImageProperties->maxImageDims1D > 0 &&
                        Device->ZeDeviceImageProperties->maxImageDims2D > 0 &&
                        Device->ZeDeviceImageProperties->maxImageDims3D > 0);
@@ -1065,15 +1063,11 @@ ur_result_t urDeviceGetInfo(
     return ReturnValue(false);
   }
   case UR_DEVICE_INFO_BINDLESS_IMAGES_1D_USM_SUPPORT_EXP: {
-    bool DeviceIsDG2OrNewer =
-        Device->ZeDeviceIpVersionExt->ipVersion >= 0x030dc000;
-    return ReturnValue(DeviceIsDG2OrNewer &&
+    return ReturnValue(Device->isIntelDG2OrNewer() &&
                        Device->ZeDeviceImageProperties->maxImageDims1D > 0);
   }
   case UR_DEVICE_INFO_BINDLESS_IMAGES_2D_USM_SUPPORT_EXP: {
-    bool DeviceIsDG2OrNewer =
-        Device->ZeDeviceIpVersionExt->ipVersion >= 0x030dc000;
-    return ReturnValue(DeviceIsDG2OrNewer &&
+    return ReturnValue(Device->isIntelDG2OrNewer() &&
                        Device->ZeDeviceImageProperties->maxImageDims2D > 0);
   }
   case UR_DEVICE_INFO_IMAGE_PITCH_ALIGN_EXP:
@@ -1155,6 +1149,8 @@ ur_result_t urDeviceGetInfo(
     return ReturnValue(false);
   case UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT:
     return ReturnValue(true);
+  case UR_DEVICE_INFO_USM_POOL_SUPPORT:
+    return ReturnValue(true);
   default:
     logger::error("Unsupported ParamName in urGetDeviceInfo");
     logger::error("ParamNameParamName={}(0x{})", ParamName,
@@ -1409,13 +1405,35 @@ ur_result_t urDeviceRelease(ur_device_handle_t Device) {
 }
 } // namespace ur::level_zero
 
-// Whether immediate commandlists will be used for kernel launches and copies.
-// The default is standard commandlists. Setting 1 or 2 specifies use of
-// immediate commandlists. Note: when immediate commandlists are used then
-// device-only events must be either AllHostVisible or OnDemandHostVisibleProxy.
-// (See env var UR_L0_DEVICE_SCOPE_EVENTS).
-
-// Get value of immediate commandlists env var setting or -1 if unset
+/**
+ * @brief Determines the mode of immediate command lists to be used.
+ *
+ * This function checks environment variables and device properties to decide
+ * the mode of immediate command lists. The mode can be influenced by the
+ * following environment variables:
+ * - `UR_L0_USE_IMMEDIATE_COMMANDLISTS`
+ * - `SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS`
+ *
+ * If neither environment variable is set, the function defaults to using the
+ * device's properties to determine the mode.
+ *
+ * @return The mode of immediate command lists, which can be one of the
+ * following:
+ * - `NotUsed`: Immediate command lists are not used.
+ * - `PerQueue`: Immediate command lists are used per queue.
+ * - `PerThreadPerQueue`: Immediate command lists are used per thread per queue.
+ *
+ * The decision process is as follows:
+ * 1. If the environment variables are not set, the function checks if the
+ * device is Intel DG2 or newer and if the driver version is supported. If both
+ *    conditions are met, or if the device is PVC, it returns `PerQueue`.
+ *    Otherwise, it returns `NotUsed`.
+ * 2. If the environment variable is set, it returns the corresponding mode:
+ *    - `0`: `NotUsed`
+ *    - `1`: `PerQueue`
+ *    - `2`: `PerThreadPerQueue`
+ *    - Any other value: `NotUsed`
+ */
 ur_device_handle_t_::ImmCmdlistMode
 ur_device_handle_t_::useImmediateCommandLists() {
   // If immediate commandlist setting is not explicitly set, then use the device
@@ -1433,9 +1451,10 @@ ur_device_handle_t_::useImmediateCommandLists() {
   }();
 
   if (ImmediateCommandlistsSetting == -1) {
+    bool isDG2OrNewer = this->isIntelDG2OrNewer();
     bool isDG2SupportedDriver =
         this->Platform->isDriverVersionNewerOrSimilar(1, 5, 30820);
-    if ((isDG2SupportedDriver && isDG2()) || isPVC()) {
+    if ((isDG2SupportedDriver && isDG2OrNewer) || isPVC()) {
       return PerQueue;
     } else {
       return NotUsed;