[UR][L0] Event cleanup in urEnqueueKernelLaunch

When using internal events we do not clean up after execution. In the
case of repeated call to urEnqueueKernelLaunch we eventually return
UR_RESULT_ERROR_OUT_OF_RESOURCES. The cleanup is done when we call zeEventPoolCreate.

Signed-off-by: Zhang, Winston <[email protected]>

Author: winstonzhang-intel

unified-runtime/source/adapters/level_zero/context.cpp

@@ -406,7 +406,7 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
     ze_event_pool_handle_t &Pool, size_t &Index, bool HostVisible,
     bool ProfilingEnabled, ur_device_handle_t Device,
     bool CounterBasedEventEnabled, bool UsingImmCmdList,
-    bool InterruptBasedEventEnabled) {
+    bool InterruptBasedEventEnabled, ur_queue_handle_t Queue, bool IsInternal) {
 
   ze_device_handle_t ZeDevice = nullptr;
   if (Device) {
@@ -461,8 +461,25 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
                     });
     }
 
-    ZE2UR_CALL(zeEventPoolCreate, (ZeContext, &ZeEventPoolDesc,
-                                   ZeDevices.size(), &ZeDevices[0], &Pool));
+    ze_result_t Result = ZE_CALL_NOCHECK(
+        zeEventPoolCreate,
+        (ZeContext, &ZeEventPoolDesc, ZeDevices.size(), &ZeDevices[0], &Pool));
+    if (IsInternal && ze2urResult(Result) == UR_RESULT_ERROR_OUT_OF_RESOURCES &&
+        Queue) {
+      if (!Queue->isInOrderQueue()) {
+        if (Queue->UsingImmCmdLists) {
+          UR_CALL(CleanupEventsInImmCmdLists(Queue, true /*QueueLocked*/,
+                                             false /*QueueSynced*/,
+                                             nullptr /*CompletedEvent*/));
+        } else {
+          UR_CALL(resetCommandLists(Queue));
+        }
+        ZE2UR_CALL(zeEventPoolCreate, (ZeContext, &ZeEventPoolDesc,
+                                       ZeDevices.size(), &ZeDevices[0], &Pool));
+      }
+    } else if (ze2urResult(Result) != UR_RESULT_SUCCESS) {
+      return ze2urResult(Result);
+    }
     Index = 0;
     NumEventsAvailableInEventPool[Pool] = MaxNumEventsPerPool - 1;
     NumEventsUnreleasedInEventPool[Pool] = 1;
@@ -546,8 +563,25 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
                     });
     }
 
-    ZE2UR_CALL(zeEventPoolCreate, (ZeContext, &ZeEventPoolDesc,
-                                   ZeDevices.size(), &ZeDevices[0], ZePool));
+    ze_result_t Result = ZE_CALL_NOCHECK(
+        zeEventPoolCreate,
+        (ZeContext, &ZeEventPoolDesc, ZeDevices.size(), &ZeDevices[0], &Pool));
+    if (IsInternal && ze2urResult(Result) == UR_RESULT_ERROR_OUT_OF_RESOURCES &&
+        Queue) {
+      if (!Queue->isInOrderQueue()) {
+        if (Queue->UsingImmCmdLists) {
+          UR_CALL(CleanupEventsInImmCmdLists(Queue, true /*QueueLocked*/,
+                                             false /*QueueSynced*/,
+                                             nullptr /*CompletedEvent*/));
+        } else {
+          UR_CALL(resetCommandLists(Queue));
+        }
+        ZE2UR_CALL(zeEventPoolCreate, (ZeContext, &ZeEventPoolDesc,
+                                       ZeDevices.size(), &ZeDevices[0], &Pool));
+      }
+    } else if (ze2urResult(Result) != UR_RESULT_SUCCESS) {
+      return ze2urResult(Result);
+    }
     NumEventsAvailableInEventPool[*ZePool] = MaxNumEventsPerPool - 1;
     NumEventsUnreleasedInEventPool[*ZePool] = 1;
   } else {

unified-runtime/source/adapters/level_zero/context.hpp

@@ -202,13 +202,12 @@ struct ur_context_handle_t_ : ur_object {
   // pool then create new one. The HostVisible parameter tells if we need a
   // slot for a host-visible event. The ProfilingEnabled tells is we need a
   // slot for an event with profiling capabilities.
-  ur_result_t getFreeSlotInExistingOrNewPool(ze_event_pool_handle_t &, size_t &,
-                                             bool HostVisible,
-                                             bool ProfilingEnabled,
-                                             ur_device_handle_t Device,
-                                             bool CounterBasedEventEnabled,
-                                             bool UsingImmCmdList,
-                                             bool InterruptBasedEventEnabled);
+  ur_result_t getFreeSlotInExistingOrNewPool(
+      ze_event_pool_handle_t &, size_t &, bool HostVisible,
+      bool ProfilingEnabled, ur_device_handle_t Device,
+      bool CounterBasedEventEnabled, bool UsingImmCmdList,
+      bool InterruptBasedEventEnabled, ur_queue_handle_t Queue,
+      bool IsInternal);
 
   // Get ur_event_handle_t from cache.
   ur_event_handle_t getEventFromContextCache(bool HostVisible,

unified-runtime/source/adapters/level_zero/event.cpp

@@ -1349,16 +1349,18 @@ ur_result_t CleanupCompletedEvent(ur_event_handle_t Event, bool QueueLocked,
 // The "HostVisible" argument specifies if event needs to be allocated from
 // a host-visible pool.
 //
-ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
-                        bool IsMultiDevice, bool HostVisible,
-                        ur_event_handle_t *RetEvent,
-                        bool CounterBasedEventEnabled,
-                        bool ForceDisableProfiling,
-                        bool InterruptBasedEventEnabled) {
+ur_result_t
+EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
+            bool IsMultiDevice, bool HostVisible, ur_event_handle_t *RetEvent,
+            bool CounterBasedEventEnabled, bool ForceDisableProfiling,
+            bool InterruptBasedEventEnabled, std::optional<bool> IsInternal) {
   bool ProfilingEnabled =
       ForceDisableProfiling ? false : (!Queue || Queue->isProfilingEnabled());
   bool UsingImmediateCommandlists = !Queue || Queue->UsingImmCmdLists;
 
+  // Handle optional IsInternal parameter - default to false if not provided
+  bool isInternalValue = IsInternal.value_or(false);
+
   ur_device_handle_t Device = nullptr;
 
   if (!IsMultiDevice && Queue) {
@@ -1380,7 +1382,7 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   if (auto Res = Context->getFreeSlotInExistingOrNewPool(
           ZeEventPool, Index, HostVisible, ProfilingEnabled, Device,
           CounterBasedEventEnabled, UsingImmediateCommandlists,
-          InterruptBasedEventEnabled))
+          InterruptBasedEventEnabled, Queue, isInternalValue))
     return Res;
 
   ZeStruct<ze_event_desc_t> ZeEventDesc;

unified-runtime/source/adapters/level_zero/event.hpp

@@ -37,7 +37,8 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
                         ur_event_handle_t *RetEvent,
                         bool CounterBasedEventEnabled,
                         bool ForceDisableProfiling,
-                        bool InterruptBasedEventEnabled);
+                        bool InterruptBasedEventEnabled,
+                        std::optional<bool> IsInternal = std::nullopt);
 } // extern "C"
 
 // This is an experimental option that allows to disable caching of events in

unified-runtime/source/adapters/level_zero/queue.cpp

@@ -1904,7 +1904,7 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue,
     UR_CALL(EventCreate(
         Queue->Context, Queue, IsMultiDevice, HostVisible.value(), Event,
         Queue->CounterBasedEventsEnabled, false /*ForceDisableProfiling*/,
-        Queue->InterruptBasedEventsEnabled));
+        Queue->InterruptBasedEventsEnabled, IsInternal));
 
   (*Event)->UrQueue = Queue;
   (*Event)->CommandType = CommandType;