[SYCL][Graph][L0] Enable Copy engine support with in-order command lists (#16830)

Currently, copy engine is only available to use for certain commands in
the out-of-order command-buffer path. This PR adds required
synchronization between the 2 command lists, when both are in-order.
Hence, it enables using the copy engine with in-order path.

For the immediateAppend path, this only works for now if the environment
variable `UR_L0_USE_DRIVER_COUNTER_BASED_EVENTS=0` is set.

Author: Konrad Kusiak

sycl/test-e2e/Graph/ValidUsage/linear_graph_copy.cpp

@@ -5,7 +5,10 @@
 // Extra run to check for immediate-command-list in Level Zero
 // RUN: %if level_zero %{env SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
 //
-
+// Extra runs to test in-order command lists path
+// RUN: %if level_zero %{env UR_L0_USE_DRIVER_INORDER_LISTS=1 SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=0 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
+// RUN: %if level_zero %{env UR_L0_USE_DRIVER_INORDER_LISTS=1 SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 UR_L0_USE_DRIVER_COUNTER_BASED_EVENTS=0 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
+//
 // Tests that the optimization to use the L0 Copy Engine for memory commands
 // does not interfere with the linear graph optimization
 

unified-runtime/source/adapters/level_zero/command_buffer.cpp

@@ -10,6 +10,7 @@
 #include "command_buffer.hpp"
 #include "helpers/kernel_helpers.hpp"
 #include "logger/ur_logger.hpp"
+#include "ur_api.h"
 #include "ur_interface_loader.hpp"
 #include "ur_level_zero.hpp"
 
@@ -170,6 +171,67 @@ ur_result_t getEventsFromSyncPoints(
   return UR_RESULT_SUCCESS;
 }
 
+/**
+ * If necessary, it creates a signal event and appends it to the previous
+ * command list (copy or compute), to indicate when it's finished executing.
+ * @param[in] CommandBuffer The CommandBuffer where the command is appended.
+ * @param[in] ZeCommandList the CommandList that's currently in use.
+ * @param[out] WaitEventList The list of event for the future command list to
+ * wait on before execution.
+ * @return UR_RESULT_SUCCESS or an error code on failure
+ */
+ur_result_t createSyncPointBetweenCopyAndCompute(
+    ur_exp_command_buffer_handle_t CommandBuffer,
+    ze_command_list_handle_t ZeCommandList,
+    std::vector<ze_event_handle_t> &WaitEventList) {
+
+  if (!CommandBuffer->ZeCopyCommandList) {
+    return UR_RESULT_SUCCESS;
+  }
+
+  bool IsCopy{ZeCommandList == CommandBuffer->ZeCopyCommandList};
+
+  // Skip synchronization for the first node in a graph or if the current
+  // command list matches the previous one.
+  if (!CommandBuffer->MWasPrevCopyCommandList.has_value()) {
+    CommandBuffer->MWasPrevCopyCommandList = IsCopy;
+    return UR_RESULT_SUCCESS;
+  } else if (IsCopy == CommandBuffer->MWasPrevCopyCommandList) {
+    return UR_RESULT_SUCCESS;
+  }
+
+  /*
+   * If the current CommandList differs from the previously used one, we must
+   * append a signal event to the previous CommandList to track when
+   * its execution is complete.
+   */
+  ur_event_handle_t SignalPrevCommandEvent = nullptr;
+  UR_CALL(EventCreate(CommandBuffer->Context, nullptr /*Queue*/,
+                      false /*IsMultiDevice*/, false, &SignalPrevCommandEvent,
+                      false /*CounterBasedEventEnabled*/,
+                      !CommandBuffer->IsProfilingEnabled,
+                      false /*InterruptBasedEventEnabled*/));
+
+  // Determine which command list to signal.
+  auto CommandListToSignal = (!IsCopy && CommandBuffer->MWasPrevCopyCommandList)
+                                 ? CommandBuffer->ZeCopyCommandList
+                                 : CommandBuffer->ZeComputeCommandList;
+  CommandBuffer->MWasPrevCopyCommandList = IsCopy;
+
+  ZE2UR_CALL(zeCommandListAppendSignalEvent,
+             (CommandListToSignal, SignalPrevCommandEvent->ZeEvent));
+
+  // Add the event to the dependencies for future command list to wait on.
+  WaitEventList.push_back(SignalPrevCommandEvent->ZeEvent);
+
+  // Get sync point and register the event with it.
+  ur_exp_command_buffer_sync_point_t SyncPoint =
+      CommandBuffer->getNextSyncPoint();
+  CommandBuffer->registerSyncPoint(SyncPoint, SignalPrevCommandEvent);
+
+  return UR_RESULT_SUCCESS;
+}
+
 /**
  * If needed, creates a sync point for a given command and returns the L0
  * events associated with the sync point.
@@ -190,7 +252,7 @@ ur_result_t getEventsFromSyncPoints(
  */
 ur_result_t createSyncPointAndGetZeEvents(
     ur_command_t CommandType, ur_exp_command_buffer_handle_t CommandBuffer,
-    uint32_t NumSyncPointsInWaitList,
+    ze_command_list_handle_t ZeCommandList, uint32_t NumSyncPointsInWaitList,
     const ur_exp_command_buffer_sync_point_t *SyncPointWaitList,
     bool HostVisible, ur_exp_command_buffer_sync_point_t *RetSyncPoint,
     std::vector<ze_event_handle_t> &ZeEventList,
@@ -199,6 +261,11 @@ ur_result_t createSyncPointAndGetZeEvents(
   ZeLaunchEvent = nullptr;
 
   if (CommandBuffer->IsInOrderCmdList) {
+    UR_CALL(createSyncPointBetweenCopyAndCompute(CommandBuffer, ZeCommandList,
+                                                 ZeEventList));
+    if (!ZeEventList.empty()) {
+      NumSyncPointsInWaitList = ZeEventList.size();
+    }
     return UR_RESULT_SUCCESS;
   }
 
@@ -225,24 +292,24 @@ ur_result_t createSyncPointAndGetZeEvents(
   return UR_RESULT_SUCCESS;
 }
 
-// Shared by all memory read/write/copy PI interfaces.
-// Helper function for common code when enqueuing memory operations to a command
-// buffer.
+// Shared by all memory read/write/copy UR interfaces.
+// Helper function for common code when enqueuing memory operations to a
+// command buffer.
 ur_result_t enqueueCommandBufferMemCopyHelper(
     ur_command_t CommandType, ur_exp_command_buffer_handle_t CommandBuffer,
     void *Dst, const void *Src, size_t Size, bool PreferCopyEngine,
     uint32_t NumSyncPointsInWaitList,
     const ur_exp_command_buffer_sync_point_t *SyncPointWaitList,
     ur_exp_command_buffer_sync_point_t *RetSyncPoint) {
 
+  ze_command_list_handle_t ZeCommandList =
+      CommandBuffer->chooseCommandList(PreferCopyEngine);
+
   std::vector<ze_event_handle_t> ZeEventList;
   ze_event_handle_t ZeLaunchEvent = nullptr;
   UR_CALL(createSyncPointAndGetZeEvents(
-      CommandType, CommandBuffer, NumSyncPointsInWaitList, SyncPointWaitList,
-      false, RetSyncPoint, ZeEventList, ZeLaunchEvent));
-
-  ze_command_list_handle_t ZeCommandList =
-      CommandBuffer->chooseCommandList(PreferCopyEngine);
+      CommandType, CommandBuffer, ZeCommandList, NumSyncPointsInWaitList,
+      SyncPointWaitList, false, RetSyncPoint, ZeEventList, ZeLaunchEvent));
 
   ZE2UR_CALL(zeCommandListAppendMemoryCopy,
              (ZeCommandList, Dst, Src, Size, ZeLaunchEvent, ZeEventList.size(),
@@ -293,14 +360,14 @@ ur_result_t enqueueCommandBufferMemCopyRectHelper(
   const ze_copy_region_t ZeDstRegion = {DstOriginX, DstOriginY, DstOriginZ,
                                         Width,      Height,     Depth};
 
+  ze_command_list_handle_t ZeCommandList =
+      CommandBuffer->chooseCommandList(PreferCopyEngine);
+
   std::vector<ze_event_handle_t> ZeEventList;
   ze_event_handle_t ZeLaunchEvent = nullptr;
   UR_CALL(createSyncPointAndGetZeEvents(
-      CommandType, CommandBuffer, NumSyncPointsInWaitList, SyncPointWaitList,
-      false, RetSyncPoint, ZeEventList, ZeLaunchEvent));
-
-  ze_command_list_handle_t ZeCommandList =
-      CommandBuffer->chooseCommandList(PreferCopyEngine);
+      CommandType, CommandBuffer, ZeCommandList, NumSyncPointsInWaitList,
+      SyncPointWaitList, false, RetSyncPoint, ZeEventList, ZeLaunchEvent));
 
   ZE2UR_CALL(zeCommandListAppendMemoryCopyRegion,
              (ZeCommandList, Dst, &ZeDstRegion, DstPitch, DstSlicePitch, Src,
@@ -321,19 +388,19 @@ ur_result_t enqueueCommandBufferFillHelper(
   UR_ASSERT((PatternSize > 0) && ((PatternSize & (PatternSize - 1)) == 0),
             UR_RESULT_ERROR_INVALID_VALUE);
 
-  std::vector<ze_event_handle_t> ZeEventList;
-  ze_event_handle_t ZeLaunchEvent = nullptr;
-  UR_CALL(createSyncPointAndGetZeEvents(
-      CommandType, CommandBuffer, NumSyncPointsInWaitList, SyncPointWaitList,
-      true, RetSyncPoint, ZeEventList, ZeLaunchEvent));
-
   bool PreferCopyEngine;
   UR_CALL(
       preferCopyEngineForFill(CommandBuffer, PatternSize, PreferCopyEngine));
 
   ze_command_list_handle_t ZeCommandList =
       CommandBuffer->chooseCommandList(PreferCopyEngine);
 
+  std::vector<ze_event_handle_t> ZeEventList;
+  ze_event_handle_t ZeLaunchEvent = nullptr;
+  UR_CALL(createSyncPointAndGetZeEvents(
+      CommandType, CommandBuffer, ZeCommandList, NumSyncPointsInWaitList,
+      SyncPointWaitList, true, RetSyncPoint, ZeEventList, ZeLaunchEvent));
+
   ZE2UR_CALL(zeCommandListAppendMemoryFill,
              (ZeCommandList, Ptr, Pattern, PatternSize, Size, ZeLaunchEvent,
               ZeEventList.size(), getPointerFromVector(ZeEventList)));
@@ -477,12 +544,12 @@ void ur_exp_command_buffer_handle_t_::registerSyncPoint(
 
 ze_command_list_handle_t
 ur_exp_command_buffer_handle_t_::chooseCommandList(bool PreferCopyEngine) {
-  if (PreferCopyEngine && this->useCopyEngine() && !this->IsInOrderCmdList) {
+  if (PreferCopyEngine && useCopyEngine()) {
     // We indicate that ZeCopyCommandList contains commands to be submitted.
-    this->MCopyCommandListEmpty = false;
-    return this->ZeCopyCommandList;
+    MCopyCommandListEmpty = false;
+    return ZeCopyCommandList;
   }
-  return this->ZeComputeCommandList;
+  return ZeComputeCommandList;
 }
 
 ur_result_t ur_exp_command_buffer_handle_t_::getFenceForQueue(
@@ -646,7 +713,7 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
   // the current implementation only uses the main copy engine and does not use
   // the link engine even if available.
   if (Device->hasMainCopyEngine()) {
-    UR_CALL(createMainCommandList(Context, Device, false, false, true,
+    UR_CALL(createMainCommandList(Context, Device, IsInOrder, false, true,
                                   ZeCopyCommandList));
   }
 
@@ -812,18 +879,18 @@ finalizeWaitEventPath(ur_exp_command_buffer_handle_t CommandBuffer) {
              (CommandBuffer->ZeCommandListResetEvents,
               CommandBuffer->ExecutionFinishedEvent->ZeEvent));
 
+  // Reset the L0 events we use for command-buffer sync-points to the
+  // non-signaled state. This is required for multiple submissions.
+  for (auto &Event : CommandBuffer->ZeEventsList) {
+    ZE2UR_CALL(zeCommandListAppendEventReset,
+               (CommandBuffer->ZeCommandListResetEvents, Event));
+  }
+
   if (CommandBuffer->IsInOrderCmdList) {
     ZE2UR_CALL(zeCommandListAppendSignalEvent,
                (CommandBuffer->ZeComputeCommandList,
                 CommandBuffer->ExecutionFinishedEvent->ZeEvent));
   } else {
-    // Reset the L0 events we use for command-buffer sync-points to the
-    // non-signaled state. This is required for multiple submissions.
-    for (auto &Event : CommandBuffer->ZeEventsList) {
-      ZE2UR_CALL(zeCommandListAppendEventReset,
-                 (CommandBuffer->ZeCommandListResetEvents, Event));
-    }
-
     // Wait for all the user added commands to complete, and signal the
     // command-buffer signal-event when they are done.
     ZE2UR_CALL(zeCommandListAppendBarrier,
@@ -1073,7 +1140,8 @@ ur_result_t urCommandBufferAppendKernelLaunchExp(
   std::vector<ze_event_handle_t> ZeEventList;
   ze_event_handle_t ZeLaunchEvent = nullptr;
   UR_CALL(createSyncPointAndGetZeEvents(
-      UR_COMMAND_KERNEL_LAUNCH, CommandBuffer, NumSyncPointsInWaitList,
+      UR_COMMAND_KERNEL_LAUNCH, CommandBuffer,
+      CommandBuffer->ZeComputeCommandList, NumSyncPointsInWaitList,
       SyncPointWaitList, false, RetSyncPoint, ZeEventList, ZeLaunchEvent));
 
   ZE2UR_CALL(zeCommandListAppendLaunchKernel,
@@ -1306,29 +1374,25 @@ ur_result_t urCommandBufferAppendUSMPrefetchExp(
   std::ignore = Command;
   std::ignore = Flags;
 
-  if (CommandBuffer->IsInOrderCmdList) {
-    // Add the prefetch command to the command-buffer.
-    // Note that L0 does not handle migration flags.
-    ZE2UR_CALL(zeCommandListAppendMemoryPrefetch,
-               (CommandBuffer->ZeComputeCommandList, Mem, Size));
-  } else {
-    std::vector<ze_event_handle_t> ZeEventList;
-    ze_event_handle_t ZeLaunchEvent = nullptr;
-    UR_CALL(createSyncPointAndGetZeEvents(
-        UR_COMMAND_USM_PREFETCH, CommandBuffer, NumSyncPointsInWaitList,
-        SyncPointWaitList, true, RetSyncPoint, ZeEventList, ZeLaunchEvent));
-
-    if (NumSyncPointsInWaitList) {
-      ZE2UR_CALL(zeCommandListAppendWaitOnEvents,
-                 (CommandBuffer->ZeComputeCommandList, NumSyncPointsInWaitList,
-                  ZeEventList.data()));
-    }
+  std::vector<ze_event_handle_t> ZeEventList;
+  ze_event_handle_t ZeLaunchEvent = nullptr;
+  UR_CALL(createSyncPointAndGetZeEvents(
+      UR_COMMAND_USM_PREFETCH, CommandBuffer,
+      CommandBuffer->ZeComputeCommandList, NumSyncPointsInWaitList,
+      SyncPointWaitList, true, RetSyncPoint, ZeEventList, ZeLaunchEvent));
+
+  if (NumSyncPointsInWaitList) {
+    ZE2UR_CALL(zeCommandListAppendWaitOnEvents,
+               (CommandBuffer->ZeComputeCommandList, NumSyncPointsInWaitList,
+                ZeEventList.data()));
+  }
 
-    // Add the prefetch command to the command-buffer.
-    // Note that L0 does not handle migration flags.
-    ZE2UR_CALL(zeCommandListAppendMemoryPrefetch,
-               (CommandBuffer->ZeComputeCommandList, Mem, Size));
+  // Add the prefetch command to the command-buffer.
+  // Note that L0 does not handle migration flags.
+  ZE2UR_CALL(zeCommandListAppendMemoryPrefetch,
+             (CommandBuffer->ZeComputeCommandList, Mem, Size));
 
+  if (!CommandBuffer->IsInOrderCmdList) {
     // Level Zero does not have a completion "event" with the prefetch API,
     // so manually add command to signal our event.
     ZE2UR_CALL(zeCommandListAppendSignalEvent,
@@ -1376,27 +1440,24 @@ ur_result_t urCommandBufferAppendUSMAdviseExp(
 
   ze_memory_advice_t ZeAdvice = static_cast<ze_memory_advice_t>(Value);
 
-  if (CommandBuffer->IsInOrderCmdList) {
-    ZE2UR_CALL(zeCommandListAppendMemAdvise,
-               (CommandBuffer->ZeComputeCommandList,
-                CommandBuffer->Device->ZeDevice, Mem, Size, ZeAdvice));
-  } else {
-    std::vector<ze_event_handle_t> ZeEventList;
-    ze_event_handle_t ZeLaunchEvent = nullptr;
-    UR_CALL(createSyncPointAndGetZeEvents(
-        UR_COMMAND_USM_ADVISE, CommandBuffer, NumSyncPointsInWaitList,
-        SyncPointWaitList, true, RetSyncPoint, ZeEventList, ZeLaunchEvent));
-
-    if (NumSyncPointsInWaitList) {
-      ZE2UR_CALL(zeCommandListAppendWaitOnEvents,
-                 (CommandBuffer->ZeComputeCommandList, NumSyncPointsInWaitList,
-                  ZeEventList.data()));
-    }
+  std::vector<ze_event_handle_t> ZeEventList;
+  ze_event_handle_t ZeLaunchEvent = nullptr;
+  UR_CALL(createSyncPointAndGetZeEvents(
+      UR_COMMAND_USM_ADVISE, CommandBuffer, CommandBuffer->ZeComputeCommandList,
+      NumSyncPointsInWaitList, SyncPointWaitList, true, RetSyncPoint,
+      ZeEventList, ZeLaunchEvent));
 
-    ZE2UR_CALL(zeCommandListAppendMemAdvise,
-               (CommandBuffer->ZeComputeCommandList,
-                CommandBuffer->Device->ZeDevice, Mem, Size, ZeAdvice));
+  if (NumSyncPointsInWaitList) {
+    ZE2UR_CALL(zeCommandListAppendWaitOnEvents,
+               (CommandBuffer->ZeComputeCommandList, NumSyncPointsInWaitList,
+                ZeEventList.data()));
+  }
+
+  ZE2UR_CALL(zeCommandListAppendMemAdvise,
+             (CommandBuffer->ZeComputeCommandList,
+              CommandBuffer->Device->ZeDevice, Mem, Size, ZeAdvice));
 
+  if (!CommandBuffer->IsInOrderCmdList) {
     // Level Zero does not have a completion "event" with the advise API,
     // so manually add command to signal our event.
     ZE2UR_CALL(zeCommandListAppendSignalEvent,

unified-runtime/source/adapters/level_zero/command_buffer.hpp

@@ -9,6 +9,7 @@
 //===----------------------------------------------------------------------===//
 #pragma once
 
+#include <optional>
 #include <ur/ur.hpp>
 #include <ur_api.h>
 #include <ze_api.h>
@@ -110,6 +111,8 @@ struct ur_exp_command_buffer_handle_t_ : public _ur_object {
   // This flag must be set to false if at least one copy command has been
   // added to `ZeCopyCommandList`
   bool MCopyCommandListEmpty = true;
+  // This flag tracks if the previous node submission was of a copy type.
+  std::optional<bool> MWasPrevCopyCommandList;
   // [WaitEvent Path only] Level Zero fences for each queue the command-buffer
   // has been enqueued to. These should be destroyed when the command-buffer is
   // released.