[L0 v2] use single command list for all operations

Author: igchor

source/adapters/level_zero/v2/command_list_cache.cpp

@@ -13,6 +13,21 @@
 
 #include "../device.hpp"
 
+typedef struct _zex_intel_queue_copy_operations_offload_hint_exp_desc_t {
+  ze_structure_type_t stype;
+  const void *pNext;
+  ze_bool_t copyOffloadEnabled;
+} zex_intel_queue_copy_operations_offload_hint_exp_desc_t;
+
+#define ZEX_INTEL_STRUCTURE_TYPE_QUEUE_COPY_OPERATIONS_OFFLOAD_HINT_EXP_PROPERTIES \
+  (ze_structure_type_t)0x0003001B
+
+template <>
+ze_structure_type_t
+getZeStructureType<zex_intel_queue_copy_operations_offload_hint_exp_desc_t>() {
+  return ZEX_INTEL_STRUCTURE_TYPE_QUEUE_COPY_OPERATIONS_OFFLOAD_HINT_EXP_PROPERTIES;
+}
+
 bool v2::immediate_command_list_descriptor_t::operator==(
     const immediate_command_list_descriptor_t &rhs) const {
   return ZeDevice == rhs.ZeDevice && IsInOrder == rhs.IsInOrder &&
@@ -45,6 +60,10 @@ command_list_cache_t::command_list_cache_t(ze_context_handle_t ZeContext)
 
 raii::ze_command_list_handle_t
 command_list_cache_t::createCommandList(const command_list_descriptor_t &desc) {
+  ZeStruct<zex_intel_queue_copy_operations_offload_hint_exp_desc_t> offloadDesc;
+  offloadDesc.copyOffloadEnabled =
+      std::visit([](auto &&arg) { return arg.CopyOffloadEnabled; }, desc);
+
   if (auto ImmCmdDesc =
           std::get_if<immediate_command_list_descriptor_t>(&desc)) {
     ze_command_list_handle_t ZeCommandList;
@@ -58,6 +77,7 @@ command_list_cache_t::createCommandList(const command_list_descriptor_t &desc) {
       QueueDesc.flags |= ZE_COMMAND_QUEUE_FLAG_EXPLICIT_ONLY;
       QueueDesc.index = ImmCmdDesc->Index.value();
     }
+    QueueDesc.pNext = &offloadDesc;
     ZE2UR_CALL_THROWS(
         zeCommandListCreateImmediate,
         (ZeContext, ImmCmdDesc->ZeDevice, &QueueDesc, &ZeCommandList));
@@ -68,6 +88,7 @@ command_list_cache_t::createCommandList(const command_list_descriptor_t &desc) {
     CmdListDesc.flags =
         RegCmdDesc.IsInOrder ? ZE_COMMAND_LIST_FLAG_IN_ORDER : 0;
     CmdListDesc.commandQueueGroupOrdinal = RegCmdDesc.Ordinal;
+    CmdListDesc.pNext = &offloadDesc;
 
     ze_command_list_handle_t ZeCommandList;
     ZE2UR_CALL_THROWS(zeCommandListCreate, (ZeContext, RegCmdDesc.ZeDevice,
@@ -78,13 +99,14 @@ command_list_cache_t::createCommandList(const command_list_descriptor_t &desc) {
 
 raii::command_list_unique_handle command_list_cache_t::getImmediateCommandList(
     ze_device_handle_t ZeDevice, bool IsInOrder, uint32_t Ordinal,
-    ze_command_queue_mode_t Mode, ze_command_queue_priority_t Priority,
-    std::optional<uint32_t> Index) {
+    bool CopyOffloadEnable, ze_command_queue_mode_t Mode,
+    ze_command_queue_priority_t Priority, std::optional<uint32_t> Index) {
   TRACK_SCOPE_LATENCY("command_list_cache_t::getImmediateCommandList");
 
   immediate_command_list_descriptor_t Desc;
   Desc.ZeDevice = ZeDevice;
   Desc.Ordinal = Ordinal;
+  Desc.CopyOffloadEnabled = CopyOffloadEnable;
   Desc.IsInOrder = IsInOrder;
   Desc.Mode = Mode;
   Desc.Priority = Priority;
@@ -99,13 +121,15 @@ raii::command_list_unique_handle command_list_cache_t::getImmediateCommandList(
 
 raii::command_list_unique_handle
 command_list_cache_t::getRegularCommandList(ze_device_handle_t ZeDevice,
-                                            bool IsInOrder, uint32_t Ordinal) {
+                                            bool IsInOrder, uint32_t Ordinal,
+                                            bool CopyOffloadEnable) {
   TRACK_SCOPE_LATENCY("command_list_cache_t::getRegularCommandList");
 
   regular_command_list_descriptor_t Desc;
   Desc.ZeDevice = ZeDevice;
   Desc.IsInOrder = IsInOrder;
   Desc.Ordinal = Ordinal;
+  Desc.CopyOffloadEnabled = CopyOffloadEnable;
 
   auto [CommandList, _] = getCommandList(Desc).release();
 

source/adapters/level_zero/v2/command_list_cache.hpp

@@ -30,6 +30,7 @@ struct immediate_command_list_descriptor_t {
   ze_device_handle_t ZeDevice;
   bool IsInOrder;
   uint32_t Ordinal;
+  bool CopyOffloadEnabled;
   ze_command_queue_mode_t Mode;
   ze_command_queue_priority_t Priority;
   std::optional<uint32_t> Index;
@@ -40,6 +41,7 @@ struct regular_command_list_descriptor_t {
   ze_device_handle_t ZeDevice;
   bool IsInOrder;
   uint32_t Ordinal;
+  bool CopyOffloadEnabled;
   bool operator==(const regular_command_list_descriptor_t &rhs) const;
 };
 
@@ -56,12 +58,13 @@ struct command_list_cache_t {
 
   raii::command_list_unique_handle
   getImmediateCommandList(ze_device_handle_t ZeDevice, bool IsInOrder,
-                          uint32_t Ordinal, ze_command_queue_mode_t Mode,
+                          uint32_t Ordinal, bool CopyOffloadEnable,
+                          ze_command_queue_mode_t Mode,
                           ze_command_queue_priority_t Priority,
                           std::optional<uint32_t> Index = std::nullopt);
   raii::command_list_unique_handle
   getRegularCommandList(ze_device_handle_t ZeDevice, bool IsInOrder,
-                        uint32_t Ordinal);
+                        uint32_t Ordinal, bool CopyOffloadEnable);
 
   // For testing purposes
   size_t getNumImmediateCommandLists();

source/adapters/level_zero/v2/memory.cpp

@@ -157,7 +157,7 @@ static ur_result_t synchronousZeCopy(ur_context_handle_t hContext,
       hDevice
           ->QueueGroup[ur_device_handle_t_::queue_group_info_t::type::Compute]
           .ZeOrdinal,
-      ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS, ZE_COMMAND_QUEUE_PRIORITY_NORMAL,
+      true, ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS, ZE_COMMAND_QUEUE_PRIORITY_NORMAL,
       std::nullopt);
 
   ZE2UR_CALL(zeCommandListAppendMemoryCopy,