Merge branch 'main' into detect-memory-leak

Author: zhaomaosu

source/adapters/hip/device.cpp

@@ -549,6 +549,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
     // native asserts are in progress
     std::string SupportedExtensions = "";
     SupportedExtensions += "pi_ext_intel_devicelib_assert ";
+    SupportedExtensions += "ur_exp_usm_p2p ";
 
     int RuntimeVersion = 0;
     UR_CHECK_ERROR(hipRuntimeGetVersion(&RuntimeVersion));

source/adapters/level_zero/command_buffer.cpp

@@ -30,25 +30,40 @@ struct ur_exp_command_buffer_handle_t_ : public _ur_object {
       ze_command_list_handle_t CommandList,
       ze_command_list_handle_t CommandListTranslated,
       ze_command_list_handle_t CommandListResetEvents,
-      ze_command_list_handle_t CopyCommandList,
-      ZeStruct<ze_command_list_desc_t> ZeDesc,
-      ZeStruct<ze_command_list_desc_t> ZeCopyDesc,
+      ze_command_list_handle_t CopyCommandList, ur_event_handle_t SignalEvent,
+      ur_event_handle_t WaitEvent, ur_event_handle_t AllResetEvent,
       const ur_exp_command_buffer_desc_t *Desc, const bool IsInOrderCmdList);
 
   ~ur_exp_command_buffer_handle_t_();
 
-  void RegisterSyncPoint(ur_exp_command_buffer_sync_point_t SyncPoint,
-                         ur_event_handle_t Event) {
-    SyncPoints[SyncPoint] = Event;
-    NextSyncPoint++;
-  }
+  void registerSyncPoint(ur_exp_command_buffer_sync_point_t SyncPoint,
+                         ur_event_handle_t Event);
 
-  ur_exp_command_buffer_sync_point_t GetNextSyncPoint() const {
+  ur_exp_command_buffer_sync_point_t getNextSyncPoint() const {
     return NextSyncPoint;
   }
 
   // Indicates if a copy engine is available for use
-  bool UseCopyEngine() const { return ZeCopyCommandList != nullptr; }
+  bool useCopyEngine() const { return ZeCopyCommandList != nullptr; }
+
+  /**
+   * Obtains a fence for a specific L0 queue. If there is already an available
+   * fence for this queue, it will be reused.
+   * @param[in] ZeCommandQueue The L0 queue associated with the fence.
+   * @param[out] ZeFence The fence.
+   * @return UR_RESULT_SUCCESS or an error code on failure
+   */
+  ur_result_t getFenceForQueue(ze_command_queue_handle_t &ZeCommandQueue,
+                               ze_fence_handle_t &ZeFence);
+
+  /**
+   * Chooses which command list to use when appending a command to this command
+   * buffer.
+   * @param[in] PreferCopyEngine If true, will try to choose a copy engine
+   * command-list. Will choose a compute command-list otherwise.
+   * @return The chosen command list.
+   */
+  ze_command_list_handle_t chooseCommandList(bool PreferCopyEngine);
 
   // UR context associated with this command-buffer
   ur_context_handle_t Context;
@@ -61,12 +76,17 @@ struct ur_exp_command_buffer_handle_t_ : public _ur_object {
   ze_command_list_handle_t ZeComputeCommandListTranslated;
   // Level Zero command list handle
   ze_command_list_handle_t ZeCommandListResetEvents;
-  // Level Zero command list descriptor
-  ZeStruct<ze_command_list_desc_t> ZeCommandListDesc;
   // Level Zero Copy command list handle
   ze_command_list_handle_t ZeCopyCommandList;
-  // Level Zero Copy command list descriptor
-  ZeStruct<ze_command_list_desc_t> ZeCopyCommandListDesc;
+  // Event which will signals the most recent execution of the command-buffer
+  // has finished
+  ur_event_handle_t SignalEvent = nullptr;
+  // Event which a command-buffer waits on until the wait-list dependencies
+  // passed to a command-buffer enqueue have been satisfied.
+  ur_event_handle_t WaitEvent = nullptr;
+  // Event which a command-buffer waits on until the main command-list event
+  // have been reset.
+  ur_event_handle_t AllResetEvent = nullptr;
   // This flag is must be set to false if at least one copy command has been
   // added to `ZeCopyCommandList`
   bool MCopyCommandListEmpty = true;
@@ -77,26 +97,15 @@ struct ur_exp_command_buffer_handle_t_ : public _ur_object {
   // Must be an element in ZeFencesMap, so is not required to be destroyed
   // itself.
   ze_fence_handle_t ZeActiveFence;
-  // Queue properties from command-buffer descriptor
-  // TODO: Do we need these?
-  ur_queue_properties_t QueueProperties;
   // Map of sync_points to ur_events
   std::unordered_map<ur_exp_command_buffer_sync_point_t, ur_event_handle_t>
       SyncPoints;
   // Next sync_point value (may need to consider ways to reuse values if 32-bits
   // is not enough)
   ur_exp_command_buffer_sync_point_t NextSyncPoint;
-  // List of Level Zero events associated to submitted commands.
+  // List of Level Zero events associated with submitted commands.
   std::vector<ze_event_handle_t> ZeEventsList;
-  // Event which will signals the most recent execution of the command-buffer
-  // has finished
-  ur_event_handle_t SignalEvent = nullptr;
-  // Event which a command-buffer waits on until the wait-list dependencies
-  // passed to a command-buffer enqueue have been satisfied.
-  ur_event_handle_t WaitEvent = nullptr;
-  // Event which a command-buffer waits on until the main command-list event
-  // have been reset.
-  ur_event_handle_t AllResetEvent = nullptr;
+
   // Indicates if command-buffer commands can be updated after it is closed.
   bool IsUpdatable = false;
   // Indicates if command buffer was finalized.

source/adapters/level_zero/command_buffer.hpp

@@ -247,6 +247,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(
     SupportedExtensions += ("ur_exp_command_buffer ");
     // Return supported for the UR multi-device compile experimental feature
     SupportedExtensions += ("ur_exp_multi_device_compile ");
+    SupportedExtensions += ("ur_exp_usm_p2p ");
 
     return ReturnValue(SupportedExtensions.c_str());
   }
@@ -803,7 +804,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(
     return ReturnValue(uint32_t{Device->ZeDeviceProperties->numThreadsPerEU});
   case UR_DEVICE_INFO_MAX_MEMORY_BANDWIDTH:
     // currently not supported in level zero runtime
-    return UR_RESULT_ERROR_INVALID_VALUE;
+    return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION;
   case UR_DEVICE_INFO_BFLOAT16: {
     // bfloat16 math functions are not yet supported on Intel GPUs.
     return ReturnValue(ur_bool_t{false});
@@ -996,7 +997,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(
     logger::error("Unsupported ParamName in urGetDeviceInfo");
     logger::error("ParamNameParamName={}(0x{})", ParamName,
                   logger::toHex(ParamName));
-    return UR_RESULT_ERROR_INVALID_VALUE;
+    return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION;
   }
 
   return UR_RESULT_SUCCESS;
@@ -1554,8 +1555,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetGlobalTimestamps(
 ) {
   const uint64_t &ZeTimerResolution =
       Device->ZeDeviceProperties->timerResolution;
-  const uint64_t TimestampMaxCount =
-      ((1ULL << Device->ZeDeviceProperties->kernelTimestampValidBits) - 1ULL);
+  const uint64_t TimestampMaxCount = Device->getTimestampMask();
   uint64_t DeviceClockCount, Dummy;
 
   ZE2UR_CALL(zeDeviceGetGlobalTimestamps,

source/adapters/level_zero/device.cpp

@@ -198,6 +198,12 @@ struct ur_device_handle_t_ : _ur_object {
                .ZeIndex >= 0;
   }
 
+  uint64_t getTimestampMask() {
+    auto ValidBits = ZeDeviceProperties->kernelTimestampValidBits;
+    assert(ValidBits <= 64);
+    return ValidBits == 64 ? ~0ULL : (1ULL << ValidBits) - 1ULL;
+  }
+
   // Cache of the immutable device properties.
   ZeCache<ZeStruct<ze_device_properties_t>> ZeDeviceProperties;
   ZeCache<ZeStruct<ze_device_compute_properties_t>> ZeDeviceComputeProperties;

source/adapters/level_zero/device.hpp

@@ -494,8 +494,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetProfilingInfo(
                                   : Event->Context->Devices[0];
 
   uint64_t ZeTimerResolution = Device->ZeDeviceProperties->timerResolution;
-  const uint64_t TimestampMaxValue =
-      ((1ULL << Device->ZeDeviceProperties->kernelTimestampValidBits) - 1ULL);
+  const uint64_t TimestampMaxValue = Device->getTimestampMask();
 
   UrReturnHelper ReturnValue(PropValueSize, PropValue, PropValueSizeRet);
 

source/adapters/level_zero/event.cpp

@@ -130,8 +130,10 @@ ur_result_t ur_queue_handle_legacy_t_::enqueueKernelLaunch(
         *OutEvent ///< [in,out][optional] return an event object that identifies
                   ///< this particular kernel execution instance.
 ) {
-  auto Queue = this;
+  UR_ASSERT(WorkDim > 0, UR_RESULT_ERROR_INVALID_WORK_DIMENSION);
+  UR_ASSERT(WorkDim < 4, UR_RESULT_ERROR_INVALID_WORK_DIMENSION);
 
+  auto Queue = this;
   ze_kernel_handle_t ZeKernel{};
   UR_CALL(getZeKernel(Queue, Kernel, &ZeKernel));
 
@@ -337,6 +339,9 @@ ur_result_t ur_queue_handle_legacy_t_::enqueueCooperativeKernelLaunchExp(
         *OutEvent ///< [in,out][optional] return an event object that identifies
                   ///< this particular kernel execution instance.
 ) {
+  UR_ASSERT(WorkDim > 0, UR_RESULT_ERROR_INVALID_WORK_DIMENSION);
+  UR_ASSERT(WorkDim < 4, UR_RESULT_ERROR_INVALID_WORK_DIMENSION);
+
   auto Queue = this;
   auto ZeDevice = Queue->Device->ZeDevice;
 

source/adapters/level_zero/kernel.cpp

@@ -1550,8 +1550,7 @@ ur_result_t ur_queue_handle_legacy_t_::active_barriers::clear() {
 
 void ur_queue_handle_legacy_t_::clearEndTimeRecordings() {
   uint64_t ZeTimerResolution = Device->ZeDeviceProperties->timerResolution;
-  const uint64_t TimestampMaxValue =
-      ((1ULL << Device->ZeDeviceProperties->kernelTimestampValidBits) - 1ULL);
+  const uint64_t TimestampMaxValue = Device->getTimestampMask();
 
   for (auto Entry : EndTimeRecordings) {
     auto &Event = Entry.first;

source/adapters/level_zero/queue.cpp

@@ -17,9 +17,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp(
   std::ignore = commandDevice;
   std::ignore = peerDevice;
 
-  logger::error(logger::LegacyMessage("[UR][L0] {} function not implemented!"),
-                "{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  // L0 has peer devices enabled by default
+  return UR_RESULT_SUCCESS;
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp(
@@ -28,9 +27,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp(
   std::ignore = commandDevice;
   std::ignore = peerDevice;
 
-  logger::error(logger::LegacyMessage("[UR][L0] {} function not implemented!"),
-                "{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  // L0 has peer devices enabled by default
+  return UR_RESULT_SUCCESS;
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp(

source/adapters/level_zero/usm_p2p.cpp

@@ -1,7 +1,3 @@
 urDeviceCreateWithNativeHandleTest.SuccessWithUnOwnedNativeHandle
 {{OPT}}urDeviceGetGlobalTimestampTest.SuccessSynchronizedTime
 urDeviceGetInfoTest.Success/UR_DEVICE_INFO_GLOBAL_MEM_FREE
-urDeviceGetInfoTest.Success/UR_DEVICE_INFO_MAX_MEMORY_BANDWIDTH
-urDeviceGetInfoTest.Success/UR_DEVICE_INFO_ASYNC_BARRIER
-urDeviceGetInfoTest.Success/UR_DEVICE_INFO_HOST_PIPE_READ_WRITE_SUPPORTED
-urDeviceGetInfoTest.Success/UR_DEVICE_INFO_MAX_REGISTERS_PER_WORK_GROUP