Merge branch 'main' into sanitizer-device-global

Author: zhaomaosu

include/ur_api.h

@@ -220,6 +220,7 @@ typedef enum ur_function_t {
     UR_FUNCTION_COMMAND_BUFFER_RELEASE_COMMAND_EXP = 217,                      ///< Enumerator for ::urCommandBufferReleaseCommandExp
     UR_FUNCTION_COMMAND_BUFFER_GET_INFO_EXP = 218,                             ///< Enumerator for ::urCommandBufferGetInfoExp
     UR_FUNCTION_COMMAND_BUFFER_COMMAND_GET_INFO_EXP = 219,                     ///< Enumerator for ::urCommandBufferCommandGetInfoExp
+    UR_FUNCTION_DEVICE_GET_SELECTED = 220,                                     ///< Enumerator for ::urDeviceGetSelected
     /// @cond
     UR_FUNCTION_FORCE_UINT32 = 0x7fffffff
     /// @endcond
@@ -1387,6 +1388,46 @@ urDeviceGet(
                                     ///< pNumDevices will be updated with the total number of devices available.
 );
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Retrieves devices within a platform selected by ONEAPI_DEVICE_SELECTOR
+///
+/// @details
+///     - Multiple calls to this function will return identical device handles,
+///       in the same order.
+///     - The number and order of handles returned from this function will be
+///       affected by environment variables that filter or select which devices
+///       are exposed through this API.
+///     - A reference is taken for each returned device and must be released
+///       with a subsequent call to ::urDeviceRelease.
+///     - The application may call this function from simultaneous threads, the
+///       implementation must be thread-safe.
+///
+/// @returns
+///     - ::UR_RESULT_SUCCESS
+///     - ::UR_RESULT_ERROR_UNINITIALIZED
+///     - ::UR_RESULT_ERROR_DEVICE_LOST
+///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
+///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
+///         + `NULL == hPlatform`
+///     - ::UR_RESULT_ERROR_INVALID_ENUMERATION
+///         + `::UR_DEVICE_TYPE_VPU < DeviceType`
+///     - ::UR_RESULT_ERROR_INVALID_VALUE
+UR_APIEXPORT ur_result_t UR_APICALL
+urDeviceGetSelected(
+    ur_platform_handle_t hPlatform, ///< [in] handle of the platform instance
+    ur_device_type_t DeviceType,    ///< [in] the type of the devices.
+    uint32_t NumEntries,            ///< [in] the number of devices to be added to phDevices.
+                                    ///< If phDevices in not NULL then NumEntries should be greater than zero,
+                                    ///< otherwise ::UR_RESULT_ERROR_INVALID_VALUE,
+                                    ///< will be returned.
+    ur_device_handle_t *phDevices,  ///< [out][optional][range(0, NumEntries)] array of handle of devices.
+                                    ///< If NumEntries is less than the number of devices available, then only
+                                    ///< that number of devices will be retrieved.
+    uint32_t *pNumDevices           ///< [out][optional] pointer to the number of devices.
+                                    ///< pNumDevices will be updated with the total number of selected devices
+                                    ///< available for the given platform.
+);
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Supported device info
 typedef enum ur_device_info_t {
@@ -11154,6 +11195,18 @@ typedef struct ur_device_get_params_t {
     uint32_t **ppNumDevices;
 } ur_device_get_params_t;
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for urDeviceGetSelected
+/// @details Each entry is a pointer to the parameter passed to the function;
+///     allowing the callback the ability to modify the parameter's value
+typedef struct ur_device_get_selected_params_t {
+    ur_platform_handle_t *phPlatform;
+    ur_device_type_t *pDeviceType;
+    uint32_t *pNumEntries;
+    ur_device_handle_t **pphDevices;
+    uint32_t **ppNumDevices;
+} ur_device_get_selected_params_t;
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Function parameters for urDeviceGetInfo
 /// @details Each entry is a pointer to the parameter passed to the function;

include/ur_print.h

@@ -2450,6 +2450,14 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintVirtualMemGetInfoParams(const struct
 ///         - `buff_size < out_size`
 UR_APIEXPORT ur_result_t UR_APICALL urPrintDeviceGetParams(const struct ur_device_get_params_t *params, char *buffer, const size_t buff_size, size_t *out_size);
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print ur_device_get_selected_params_t struct
+/// @returns
+///     - ::UR_RESULT_SUCCESS
+///     - ::UR_RESULT_ERROR_INVALID_SIZE
+///         - `buff_size < out_size`
+UR_APIEXPORT ur_result_t UR_APICALL urPrintDeviceGetSelectedParams(const struct ur_device_get_selected_params_t *params, char *buffer, const size_t buff_size, size_t *out_size);
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Print ur_device_get_info_params_t struct
 /// @returns

include/ur_print.hpp

@@ -909,6 +909,9 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_function_t value) {
     case UR_FUNCTION_COMMAND_BUFFER_COMMAND_GET_INFO_EXP:
         os << "UR_FUNCTION_COMMAND_BUFFER_COMMAND_GET_INFO_EXP";
         break;
+    case UR_FUNCTION_DEVICE_GET_SELECTED:
+        os << "UR_FUNCTION_DEVICE_GET_SELECTED";
+        break;
     default:
         os << "unknown enumerator";
         break;
@@ -16292,6 +16295,48 @@ inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct
     return os;
 }
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print operator for the ur_device_get_selected_params_t type
+/// @returns
+///     std::ostream &
+inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_device_get_selected_params_t *params) {
+
+    os << ".hPlatform = ";
+
+    ur::details::printPtr(os,
+                          *(params->phPlatform));
+
+    os << ", ";
+    os << ".DeviceType = ";
+
+    os << *(params->pDeviceType);
+
+    os << ", ";
+    os << ".NumEntries = ";
+
+    os << *(params->pNumEntries);
+
+    os << ", ";
+    os << ".phDevices = {";
+    for (size_t i = 0; *(params->pphDevices) != NULL && i < *params->pNumEntries; ++i) {
+        if (i != 0) {
+            os << ", ";
+        }
+
+        ur::details::printPtr(os,
+                              (*(params->pphDevices))[i]);
+    }
+    os << "}";
+
+    os << ", ";
+    os << ".pNumDevices = ";
+
+    ur::details::printPtr(os,
+                          *(params->ppNumDevices));
+
+    return os;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Print operator for the ur_device_get_info_params_t type
 /// @returns
@@ -17090,6 +17135,9 @@ inline ur_result_t UR_APICALL printFunctionParams(std::ostream &os, ur_function_
     case UR_FUNCTION_DEVICE_GET: {
         os << (const struct ur_device_get_params_t *)params;
     } break;
+    case UR_FUNCTION_DEVICE_GET_SELECTED: {
+        os << (const struct ur_device_get_selected_params_t *)params;
+    } break;
     case UR_FUNCTION_DEVICE_GET_INFO: {
         os << (const struct ur_device_get_info_params_t *)params;
     } break;

scripts/core/device.yml

@@ -150,6 +150,45 @@ returns:
         - "`NumEntries > 0 && phDevices == NULL`"
     - $X_RESULT_ERROR_INVALID_VALUE
 --- #--------------------------------------------------------------------------
+type: function
+desc: "Retrieves devices within a platform selected by ONEAPI_DEVICE_SELECTOR"
+class: $xDevice
+loader_only: True
+name: GetSelected
+decl: static
+ordinal: "0"
+details:
+    - "Multiple calls to this function will return identical device handles, in the same order."
+    - "The number and order of handles returned from this function will be affected by environment variables that filter or select which devices are exposed through this API."
+    - "A reference is taken for each returned device and must be released with a subsequent call to $xDeviceRelease."
+    - "The application may call this function from simultaneous threads, the implementation must be thread-safe."
+params:
+    - type: $x_platform_handle_t
+      name: hPlatform
+      desc: "[in] handle of the platform instance"
+    - type: "$x_device_type_t"
+      name: DeviceType
+      desc: |
+            [in] the type of the devices.
+    - type: "uint32_t"
+      name: NumEntries
+      desc: |
+            [in] the number of devices to be added to phDevices.
+            If phDevices in not NULL then NumEntries should be greater than zero, otherwise $X_RESULT_ERROR_INVALID_VALUE,
+            will be returned.
+    - type: "$x_device_handle_t*"
+      name: phDevices
+      desc: |
+            [out][optional][range(0, NumEntries)] array of handle of devices.
+            If NumEntries is less than the number of devices available, then only that number of devices will be retrieved.
+    - type: "uint32_t*"
+      name: pNumDevices
+      desc: |
+            [out][optional] pointer to the number of devices.
+            pNumDevices will be updated with the total number of selected devices available for the given platform.
+returns:
+    - $X_RESULT_ERROR_INVALID_VALUE
+--- #--------------------------------------------------------------------------
 type: enum
 desc: "Supported device info"
 class: $xDevice

scripts/core/registry.yml

@@ -574,6 +574,9 @@ etors:
 - name: COMMAND_BUFFER_COMMAND_GET_INFO_EXP
   desc: Enumerator for $xCommandBufferCommandGetInfoExp
   value: '219'
+- name: DEVICE_GET_SELECTED
+  desc: Enumerator for $xDeviceGetSelected
+  value: '220'
 ---
 type: enum
 desc: Defines structure types

source/adapters/level_zero/context.cpp

@@ -395,27 +395,29 @@ ur_result_t ur_context_handle_t_::finalize() {
 
   if (!DisableEventsCaching) {
     std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
-    for (auto &EventCache : EventCaches) {
-      for (auto &Event : EventCache) {
+    for (auto EventCache : EventCaches) {
+      for (auto Event : *EventCache) {
         auto ZeResult = ZE_CALL_NOCHECK(zeEventDestroy, (Event->ZeEvent));
         // Gracefully handle the case that L0 was already unloaded.
         if (ZeResult && ZeResult != ZE_RESULT_ERROR_UNINITIALIZED)
           return ze2urResult(ZeResult);
         delete Event;
       }
-      EventCache.clear();
+      EventCache->clear();
+      delete EventCache;
     }
   }
   {
     std::scoped_lock<ur_mutex> Lock(ZeEventPoolCacheMutex);
-    for (auto &ZePoolCache : ZeEventPoolCache) {
-      for (auto &ZePool : ZePoolCache) {
+    for (auto ZePoolCache : ZeEventPoolCache) {
+      for (auto ZePool : *ZePoolCache) {
         auto ZeResult = ZE_CALL_NOCHECK(zeEventPoolDestroy, (ZePool));
         // Gracefully handle the case that L0 was already unloaded.
         if (ZeResult && ZeResult != ZE_RESULT_ERROR_UNINITIALIZED)
           return ze2urResult(ZeResult);
       }
-      ZePoolCache.clear();
+      ZePoolCache->clear();
+      delete ZePoolCache;
     }
   }
 

source/adapters/level_zero/context.hpp

@@ -141,7 +141,7 @@ struct ur_context_handle_t_ : _ur_object {
   // head.
   //
   // Cache of event pools to which host-visible events are added to.
-  std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{4};
+  std::vector<std::list<ze_event_pool_handle_t> *> ZeEventPoolCache;
   std::vector<std::unordered_map<ze_device_handle_t,
                                  std::list<ze_event_pool_handle_t> *>>
       ZeEventPoolCacheDeviceMap{4};
@@ -165,7 +165,7 @@ struct ur_context_handle_t_ : _ur_object {
   ur_mutex EventCacheMutex;
 
   // Caches for events.
-  std::vector<std::list<ur_event_handle_t>> EventCaches{4};
+  std::vector<std::list<ur_event_handle_t> *> EventCaches;
   std::vector<
       std::unordered_map<ur_device_handle_t, std::list<ur_event_handle_t> *>>
       EventCachesDeviceMap{4};
@@ -207,31 +207,44 @@ struct ur_context_handle_t_ : _ur_object {
 
   auto getZeEventPoolCache(bool HostVisible, bool WithProfiling,
                            ze_device_handle_t ZeDevice) {
+    // Adding 4 initial global caches for provided scope and profiling modes:
+    // Host Scope, Device Scope, with Profiling, without Profiling.
+    if (ZeEventPoolCache.empty()) {
+      for (int i = 0; i < 4; i++) {
+        std::list<ze_event_pool_handle_t> *deviceZeEventPoolCache =
+            new std::list<ze_event_pool_handle_t>;
+        ZeEventPoolCache.push_back(deviceZeEventPoolCache);
+      }
+    }
     if (HostVisible) {
       if (ZeDevice) {
         auto ZeEventPoolCacheMap = WithProfiling
                                        ? &ZeEventPoolCacheDeviceMap[0]
                                        : &ZeEventPoolCacheDeviceMap[1];
         if (ZeEventPoolCacheMap->find(ZeDevice) == ZeEventPoolCacheMap->end()) {
-          ZeEventPoolCache.emplace_back();
-          (*ZeEventPoolCacheMap)[ZeDevice] = &ZeEventPoolCache.back();
+          std::list<ze_event_pool_handle_t> *deviceZeEventPoolCache =
+              new std::list<ze_event_pool_handle_t>;
+          ZeEventPoolCache.push_back(deviceZeEventPoolCache);
+          (*ZeEventPoolCacheMap)[ZeDevice] = deviceZeEventPoolCache;
         }
         return (*ZeEventPoolCacheMap)[ZeDevice];
       } else {
-        return WithProfiling ? &ZeEventPoolCache[0] : &ZeEventPoolCache[1];
+        return WithProfiling ? ZeEventPoolCache[0] : ZeEventPoolCache[1];
       }
     } else {
       if (ZeDevice) {
         auto ZeEventPoolCacheMap = WithProfiling
                                        ? &ZeEventPoolCacheDeviceMap[2]
                                        : &ZeEventPoolCacheDeviceMap[3];
         if (ZeEventPoolCacheMap->find(ZeDevice) == ZeEventPoolCacheMap->end()) {
-          ZeEventPoolCache.emplace_back();
-          (*ZeEventPoolCacheMap)[ZeDevice] = &ZeEventPoolCache.back();
+          std::list<ze_event_pool_handle_t> *deviceZeEventPoolCache =
+              new std::list<ze_event_pool_handle_t>;
+          ZeEventPoolCache.push_back(deviceZeEventPoolCache);
+          (*ZeEventPoolCacheMap)[ZeDevice] = deviceZeEventPoolCache;
         }
         return (*ZeEventPoolCacheMap)[ZeDevice];
       } else {
-        return WithProfiling ? &ZeEventPoolCache[2] : &ZeEventPoolCache[3];
+        return WithProfiling ? ZeEventPoolCache[2] : ZeEventPoolCache[3];
       }
     }
   }
@@ -274,29 +287,42 @@ struct ur_context_handle_t_ : _ur_object {
   // Get the cache of events for a provided scope and profiling mode.
   auto getEventCache(bool HostVisible, bool WithProfiling,
                      ur_device_handle_t Device) {
+    // Adding 4 initial global caches for provided scope and profiling modes:
+    // Host Scope, Device Scope, with Profiling, without Profiling.
+    if (EventCaches.empty()) {
+      for (int i = 0; i < 4; i++) {
+        std::list<ur_event_handle_t> *deviceEventCache =
+            new std::list<ur_event_handle_t>;
+        EventCaches.push_back(deviceEventCache);
+      }
+    }
     if (HostVisible) {
       if (Device) {
         auto EventCachesMap =
             WithProfiling ? &EventCachesDeviceMap[0] : &EventCachesDeviceMap[1];
         if (EventCachesMap->find(Device) == EventCachesMap->end()) {
-          EventCaches.emplace_back();
-          (*EventCachesMap)[Device] = &EventCaches.back();
+          std::list<ur_event_handle_t> *deviceEventCache =
+              new std::list<ur_event_handle_t>;
+          EventCaches.push_back(deviceEventCache);
+          (*EventCachesMap)[Device] = deviceEventCache;
         }
         return (*EventCachesMap)[Device];
       } else {
-        return WithProfiling ? &EventCaches[0] : &EventCaches[1];
+        return WithProfiling ? EventCaches[0] : EventCaches[1];
       }
     } else {
       if (Device) {
         auto EventCachesMap =
             WithProfiling ? &EventCachesDeviceMap[2] : &EventCachesDeviceMap[3];
         if (EventCachesMap->find(Device) == EventCachesMap->end()) {
-          EventCaches.emplace_back();
-          (*EventCachesMap)[Device] = &EventCaches.back();
+          std::list<ur_event_handle_t> *deviceEventCache =
+              new std::list<ur_event_handle_t>;
+          EventCaches.push_back(deviceEventCache);
+          (*EventCachesMap)[Device] = deviceEventCache;
         }
         return (*EventCachesMap)[Device];
       } else {
-        return WithProfiling ? &EventCaches[2] : &EventCaches[3];
+        return WithProfiling ? EventCaches[2] : EventCaches[3];
       }
     }
   }

source/adapters/level_zero/event.cpp

@@ -399,11 +399,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetInfo(
     auto UrQueue = Event->UrQueue;
     if (UrQueue) {
       // Lock automatically releases when this goes out of scope.
-      std::scoped_lock<ur_shared_mutex> lock(UrQueue->Mutex);
-      const auto &OpenCommandList = UrQueue->eventOpenCommandList(Event);
-      if (OpenCommandList != UrQueue->CommandListMap.end()) {
-        UR_CALL(UrQueue->executeOpenCommandList(
-            OpenCommandList->second.isCopy(UrQueue)));
+      std::unique_lock<ur_shared_mutex> Lock(UrQueue->Mutex, std::try_to_lock);
+      // If we fail to acquire the lock, it's possible that the queue might
+      // already be waiting for this event in synchronize().
+      if (Lock.owns_lock()) {
+        const auto &OpenCommandList = UrQueue->eventOpenCommandList(Event);
+        if (OpenCommandList != UrQueue->CommandListMap.end()) {
+          UR_CALL(UrQueue->executeOpenCommandList(
+              OpenCommandList->second.isCopy(UrQueue)));
+        }
       }
     }