Merge pull request #1483 from nrspruit/fix_inorder_lists_reuse

[L0] Fix regular in order command list reuse given inorder queue

Author: aarongreig

source/adapters/level_zero/context.cpp

@@ -712,6 +712,11 @@ ur_result_t ur_context_handle_t_::getAvailableCommandList(
 
     for (auto ZeCommandListIt = ZeCommandListCache.begin();
          ZeCommandListIt != ZeCommandListCache.end(); ++ZeCommandListIt) {
+      // If this is an InOrder Queue, then only allow lists which are in order.
+      if (Queue->Device->useDriverInOrderLists() && Queue->isInOrderQueue() &&
+          !(ZeCommandListIt->second.InOrderList)) {
+        continue;
+      }
       auto &ZeCommandList = ZeCommandListIt->first;
       auto it = Queue->CommandListMap.find(ZeCommandList);
       if (it != Queue->CommandListMap.end()) {
@@ -766,6 +771,12 @@ ur_result_t ur_context_handle_t_::getAvailableCommandList(
     if (UseCopyEngine != it->second.isCopy(Queue))
       continue;
 
+    // If this is an InOrder Queue, then only allow lists which are in order.
+    if (Queue->Device->useDriverInOrderLists() && Queue->isInOrderQueue() &&
+        !(it->second.IsInOrderList)) {
+      continue;
+    }
+
     ze_result_t ZeResult =
         ZE_CALL_NOCHECK(zeFenceQueryStatus, (it->second.ZeFence));
     if (ZeResult == ZE_RESULT_SUCCESS) {

source/adapters/level_zero/context.hpp

@@ -27,6 +27,11 @@
 
 #include <umf_helpers.hpp>
 
+struct l0_command_list_cache_info {
+  ZeStruct<ze_command_queue_desc_t> ZeQueueDesc;
+  bool InOrderList = false;
+};
+
 struct ur_context_handle_t_ : _ur_object {
   ur_context_handle_t_(ze_context_handle_t ZeContext, uint32_t NumDevices,
                        const ur_device_handle_t *Devs, bool OwnZeContext)
@@ -87,11 +92,11 @@ struct ur_context_handle_t_ : _ur_object {
   //
   std::unordered_map<ze_device_handle_t,
                      std::list<std::pair<ze_command_list_handle_t,
-                                         ZeStruct<ze_command_queue_desc_t>>>>
+                                         l0_command_list_cache_info>>>
       ZeComputeCommandListCache;
   std::unordered_map<ze_device_handle_t,
                      std::list<std::pair<ze_command_list_handle_t,
-                                         ZeStruct<ze_command_queue_desc_t>>>>
+                                         l0_command_list_cache_info>>>
       ZeCopyCommandListCache;
 
   // Store USM pool for USM shared and device allocations. There is 1 memory

source/adapters/level_zero/queue.cpp

@@ -449,7 +449,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueRelease(
                         ->ZeCopyCommandListCache[Queue->Device->ZeDevice]
                   : Queue->Context
                         ->ZeComputeCommandListCache[Queue->Device->ZeDevice];
-          ZeCommandListCache.push_back({it->first, it->second.ZeQueueDesc});
+          struct l0_command_list_cache_info ListInfo;
+          ListInfo.ZeQueueDesc = it->second.ZeQueueDesc;
+          ListInfo.InOrderList = it->second.IsInOrderList;
+          ZeCommandListCache.push_back({it->first, ListInfo});
         } else {
           // A non-reusable comamnd list that came from a make_queue call is
           // destroyed since it cannot be recycled.
@@ -1708,8 +1711,10 @@ ur_result_t ur_queue_handle_t_::resetCommandList(
         UseCopyEngine
             ? this->Context->ZeCopyCommandListCache[this->Device->ZeDevice]
             : this->Context->ZeComputeCommandListCache[this->Device->ZeDevice];
-    ZeCommandListCache.push_back(
-        {CommandList->first, CommandList->second.ZeQueueDesc});
+    struct l0_command_list_cache_info ListInfo;
+    ListInfo.ZeQueueDesc = CommandList->second.ZeQueueDesc;
+    ListInfo.InOrderList = CommandList->second.IsInOrderList;
+    ZeCommandListCache.push_back({CommandList->first, ListInfo});
   }
 
   return UR_RESULT_SUCCESS;
@@ -1870,8 +1875,10 @@ ur_result_t ur_queue_handle_t_::createCommandList(
   ZeStruct<ze_command_list_desc_t> ZeCommandListDesc;
   ZeCommandListDesc.commandQueueGroupOrdinal = QueueGroupOrdinal;
 
+  bool IsInOrderList = false;
   if (Device->useDriverInOrderLists() && isInOrderQueue()) {
     ZeCommandListDesc.flags = ZE_COMMAND_LIST_FLAG_IN_ORDER;
+    IsInOrderList = true;
   }
 
   ZE2UR_CALL(zeCommandListCreate, (Context->ZeContext, Device->ZeDevice,
@@ -1882,7 +1889,8 @@ ur_result_t ur_queue_handle_t_::createCommandList(
   ZeQueueDesc.ordinal = QueueGroupOrdinal;
   std::tie(CommandList, std::ignore) = CommandListMap.insert(
       std::pair<ze_command_list_handle_t, ur_command_list_info_t>(
-          ZeCommandList, {ZeFence, false, false, ZeCommandQueue, ZeQueueDesc}));
+          ZeCommandList,
+          {ZeFence, false, false, ZeCommandQueue, ZeQueueDesc, IsInOrderList}));
 
   UR_CALL(insertStartBarrierIfDiscardEventsMode(CommandList));
   UR_CALL(insertActiveBarriers(CommandList, UseCopyEngine));
@@ -2011,7 +2019,7 @@ ur_command_list_ptr_t &ur_queue_handle_t_::ur_queue_group_t::getImmCmdList() {
                   ->ZeComputeCommandListCache[Queue->Device->ZeDevice];
     for (auto ZeCommandListIt = ZeCommandListCache.begin();
          ZeCommandListIt != ZeCommandListCache.end(); ++ZeCommandListIt) {
-      const auto &Desc = (*ZeCommandListIt).second;
+      const auto &Desc = (*ZeCommandListIt).second.ZeQueueDesc;
       if (Desc.index == ZeCommandQueueDesc.index &&
           Desc.flags == ZeCommandQueueDesc.flags &&
           Desc.mode == ZeCommandQueueDesc.mode &&

source/adapters/level_zero/queue.hpp

@@ -65,6 +65,8 @@ struct ur_command_list_info_t {
   // the make_queue API the descriptor is unavailable so a dummy descriptor is
   // used and then this entry is marked as not eligible for recycling.
   ZeStruct<ze_command_queue_desc_t> ZeQueueDesc;
+  // Indicates if this is an inorder list
+  bool IsInOrderList{false};
   bool CanReuse{true};
 
   // Helper functions to tell if this is a copy command-list.