[SYCL][Graph][UR] Rename cmd-buf enqueue entry-point (#16984)

**Same PR as was closed in #16747 due
to CI issue #16982 which has since
been resolved.**

The API to enqueue a closed command-buffer to a queue is defined in the
YAML as a part of the command-buffer class, but it should be part of the
enqueue class like other enqueue API extensions.

This PR updates the YAML and regenerates UR code, making the associated
changes to adapters and CTS.

Closes UR issue
oneapi-src/unified-runtime#2600

Author: Ewan Crawford

sycl/doc/design/CommandGraph.md

@@ -47,7 +47,7 @@ with the following entry-points:
 | `urCommandBufferAppendMemBufferWriteRectExp` | Append a rectangular memory write command to a command-buffer object. |
 | `urCommandBufferAppendMemBufferReadRectExp`  | Append a rectangular memory read command to a command-buffer object. |
 | `urCommandBufferAppendMemBufferFillExp`      | Append a memory fill command to a command-buffer object. |
-| `urCommandBufferEnqueueExp`                  | Submit command-buffer to a command-queue for execution. |
+| `urEnqueueCommandBufferExp`                  | Submit command-buffer to a command-queue for execution. |
 | `urCommandBufferUpdateKernelLaunchExp`       | Updates the parameters of a previous kernel launch command. |
 
 See the [UR EXP-COMMAND-BUFFER](https://oneapi-src.github.io/unified-runtime/core/EXP-COMMAND-BUFFER.html)
@@ -393,7 +393,7 @@ we need 2 of these helper command-lists:
 `CopyCommandList`
 
 #### Wait event Path Implementation Details
-The UR `urCommandBufferEnqueueExp` interface for submitting a command-buffer
+The UR `urEnqueueCommandBufferExp` interface for submitting a command-buffer
 takes a list of events to wait on, and returns an event representing the
 completion of that specific submission of the command-buffer.
 
@@ -474,7 +474,7 @@ the *SignalEvent*, is added (when the command-buffer is finalized). In an
 additional command-list (*signal command-list*), a barrier waiting for this
 event is also added. This barrier signals, in turn, the UR return event that
 has be defined by the runtime layer when calling the
-`urCommandBufferEnqueueExp` function.
+`urEnqueueCommandBufferExp` function.
 
 2) Manage the profiling. If a command-buffer is about to be submitted to a
 queue with the profiling property enabled, an extra command that copies
@@ -484,7 +484,7 @@ information that corresponds to the current submission of the command-buffer.
 
 ![L0 command-buffer diagram](images/L0_UR_command-buffer-v5.jpg)
 
-For a call to `urCommandBufferEnqueueExp` with an `event_list` *EL*,
+For a call to `urEnqueueCommandBufferExp` with an `event_list` *EL*,
 command-buffer *CB*, and return event *RE* our implementation has to submit three
 new command-lists for the above approach to work. Two before the command-list
 with extra commands associated with *CB*, and the other after *CB*. These new
@@ -666,7 +666,7 @@ adapter where there is matching support for each function in the list.
 | urCommandBufferAppendMemBufferFillExp | clCommandFillBufferKHR | Yes |
 | urCommandBufferAppendUSMPrefetchExp |  | No |
 | urCommandBufferAppendUSMAdviseExp |  | No |
-| urCommandBufferEnqueueExp | clEnqueueCommandBufferKHR | Yes |
+| urEnqueueCommandBufferExp | clEnqueueCommandBufferKHR | Yes |
 |  | clCommandBarrierWithWaitListKHR | No |
 |  | clCommandCopyImageKHR | No |
 |  | clCommandCopyImageToBufferKHR | No |

sycl/source/detail/graph_impl.cpp

@@ -1054,8 +1054,8 @@ exec_graph_impl::enqueue(const std::shared_ptr<sycl::detail::queue_impl> &Queue,
         ur_result_t Res =
             Queue->getAdapter()
                 ->call_nocheck<
-                    sycl::detail::UrApiKind::urCommandBufferEnqueueExp>(
-                    CommandBuffer, Queue->getHandleRef(), 0, nullptr, &UREvent);
+                    sycl::detail::UrApiKind::urEnqueueCommandBufferExp>(
+                    Queue->getHandleRef(), CommandBuffer, 0, nullptr, &UREvent);
         NewEvent->setHandle(UREvent);
         if (Res == UR_RESULT_ERROR_INVALID_QUEUE_PROPERTIES) {
           throw sycl::exception(

sycl/source/detail/scheduler/commands.cpp

@@ -3567,8 +3567,8 @@ ur_result_t ExecCGCommand::enqueueImpQueue() {
       MEvent->setHostEnqueueTime();
     if (auto Result =
             MQueue->getAdapter()
-                ->call_nocheck<UrApiKind::urCommandBufferEnqueueExp>(
-                    CmdBufferCG->MCommandBuffer, MQueue->getHandleRef(),
+                ->call_nocheck<UrApiKind::urEnqueueCommandBufferExp>(
+                    MQueue->getHandleRef(), CmdBufferCG->MCommandBuffer,
                     RawEvents.size(),
                     RawEvents.empty() ? nullptr : &RawEvents[0], Event);
         Result != UR_RESULT_SUCCESS)

unified-runtime/include/ur_api.h

@@ -417,8 +417,8 @@ typedef enum ur_function_t {
   UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_PREFETCH_EXP = 240,
   /// Enumerator for ::urCommandBufferAppendUSMAdviseExp
   UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_ADVISE_EXP = 241,
-  /// Enumerator for ::urCommandBufferEnqueueExp
-  UR_FUNCTION_COMMAND_BUFFER_ENQUEUE_EXP = 242,
+  /// Enumerator for ::urEnqueueCommandBufferExp
+  UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP = 242,
   /// Enumerator for ::urCommandBufferUpdateSignalEventExp
   UR_FUNCTION_COMMAND_BUFFER_UPDATE_SIGNAL_EVENT_EXP = 243,
   /// Enumerator for ::urCommandBufferUpdateWaitEventsExp
@@ -7095,8 +7095,8 @@ typedef enum ur_command_t {
   UR_COMMAND_READ_HOST_PIPE = 25,
   /// Event created by ::urEnqueueWriteHostPipe
   UR_COMMAND_WRITE_HOST_PIPE = 26,
-  /// Event created by ::urCommandBufferEnqueueExp
-  UR_COMMAND_COMMAND_BUFFER_ENQUEUE_EXP = 0x1000,
+  /// Event created by ::urEnqueueCommandBufferExp
+  UR_COMMAND_ENQUEUE_COMMAND_BUFFER_EXP = 0x1000,
   /// Event created by ::urBindlessImagesWaitExternalSemaphoreExp
   UR_COMMAND_EXTERNAL_SEMAPHORE_WAIT_EXP = 0x2000,
   /// Event created by ::urBindlessImagesSignalExternalSemaphoreExp
@@ -11488,8 +11488,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp(
 ///     - ::UR_RESULT_ERROR_DEVICE_LOST
 ///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
 ///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
-///         + `NULL == hCommandBuffer`
 ///         + `NULL == hQueue`
+///         + `NULL == hCommandBuffer`
 ///     - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP
 ///     - ::UR_RESULT_ERROR_INVALID_QUEUE
 ///     - ::UR_RESULT_ERROR_INVALID_EVENT
@@ -11499,11 +11499,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp(
 ///         + If event objects in phEventWaitList are not valid events.
 ///     - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
 ///     - ::UR_RESULT_ERROR_OUT_OF_RESOURCES
-UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferEnqueueExp(
-    /// [in] Handle of the command-buffer object.
-    ur_exp_command_buffer_handle_t hCommandBuffer,
+UR_APIEXPORT ur_result_t UR_APICALL urEnqueueCommandBufferExp(
     /// [in] The queue to submit this command-buffer for execution.
     ur_queue_handle_t hQueue,
+    /// [in] Handle of the command-buffer object.
+    ur_exp_command_buffer_handle_t hCommandBuffer,
     /// [in] Size of the event wait list.
     uint32_t numEventsInWaitList,
     /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of
@@ -14052,6 +14052,18 @@ typedef struct ur_enqueue_usm_free_exp_params_t {
   ur_event_handle_t **pphEvent;
 } ur_enqueue_usm_free_exp_params_t;
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function parameters for urEnqueueCommandBufferExp
+/// @details Each entry is a pointer to the parameter passed to the function;
+///     allowing the callback the ability to modify the parameter's value
+typedef struct ur_enqueue_command_buffer_exp_params_t {
+  ur_queue_handle_t *phQueue;
+  ur_exp_command_buffer_handle_t *phCommandBuffer;
+  uint32_t *pnumEventsInWaitList;
+  const ur_event_handle_t **pphEventWaitList;
+  ur_event_handle_t **pphEvent;
+} ur_enqueue_command_buffer_exp_params_t;
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Function parameters for urEnqueueCooperativeKernelLaunchExp
 /// @details Each entry is a pointer to the parameter passed to the function;
@@ -14815,18 +14827,6 @@ typedef struct ur_command_buffer_append_usm_advise_exp_params_t {
   ur_exp_command_buffer_command_handle_t **pphCommand;
 } ur_command_buffer_append_usm_advise_exp_params_t;
 
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function parameters for urCommandBufferEnqueueExp
-/// @details Each entry is a pointer to the parameter passed to the function;
-///     allowing the callback the ability to modify the parameter's value
-typedef struct ur_command_buffer_enqueue_exp_params_t {
-  ur_exp_command_buffer_handle_t *phCommandBuffer;
-  ur_queue_handle_t *phQueue;
-  uint32_t *pnumEventsInWaitList;
-  const ur_event_handle_t **pphEventWaitList;
-  ur_event_handle_t **pphEvent;
-} ur_command_buffer_enqueue_exp_params_t;
-
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Function parameters for urCommandBufferUpdateKernelLaunchExp
 /// @details Each entry is a pointer to the parameter passed to the function;

unified-runtime/include/ur_api_funcs.def

@@ -136,6 +136,7 @@ _UR_API(urEnqueueUSMDeviceAllocExp)
 _UR_API(urEnqueueUSMSharedAllocExp)
 _UR_API(urEnqueueUSMHostAllocExp)
 _UR_API(urEnqueueUSMFreeExp)
+_UR_API(urEnqueueCommandBufferExp)
 _UR_API(urEnqueueCooperativeKernelLaunchExp)
 _UR_API(urEnqueueTimestampRecordingExp)
 _UR_API(urEnqueueNativeCommandExp)
@@ -193,7 +194,6 @@ _UR_API(urCommandBufferAppendMemBufferReadRectExp)
 _UR_API(urCommandBufferAppendMemBufferFillExp)
 _UR_API(urCommandBufferAppendUSMPrefetchExp)
 _UR_API(urCommandBufferAppendUSMAdviseExp)
-_UR_API(urCommandBufferEnqueueExp)
 _UR_API(urCommandBufferUpdateKernelLaunchExp)
 _UR_API(urCommandBufferUpdateSignalEventExp)
 _UR_API(urCommandBufferUpdateWaitEventsExp)

unified-runtime/include/ur_ddi.h

@@ -1144,6 +1144,12 @@ typedef ur_result_t(UR_APICALL *ur_pfnEnqueueUSMFreeExp_t)(
     ur_queue_handle_t, ur_usm_pool_handle_t, void *, uint32_t,
     const ur_event_handle_t *, ur_event_handle_t *);
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Function-pointer for urEnqueueCommandBufferExp
+typedef ur_result_t(UR_APICALL *ur_pfnEnqueueCommandBufferExp_t)(
+    ur_queue_handle_t, ur_exp_command_buffer_handle_t, uint32_t,
+    const ur_event_handle_t *, ur_event_handle_t *);
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Function-pointer for urEnqueueCooperativeKernelLaunchExp
 typedef ur_result_t(UR_APICALL *ur_pfnEnqueueCooperativeKernelLaunchExp_t)(
@@ -1173,6 +1179,7 @@ typedef struct ur_enqueue_exp_dditable_t {
   ur_pfnEnqueueUSMSharedAllocExp_t pfnUSMSharedAllocExp;
   ur_pfnEnqueueUSMHostAllocExp_t pfnUSMHostAllocExp;
   ur_pfnEnqueueUSMFreeExp_t pfnUSMFreeExp;
+  ur_pfnEnqueueCommandBufferExp_t pfnCommandBufferExp;
   ur_pfnEnqueueCooperativeKernelLaunchExp_t pfnCooperativeKernelLaunchExp;
   ur_pfnEnqueueTimestampRecordingExp_t pfnTimestampRecordingExp;
   ur_pfnEnqueueNativeCommandExp_t pfnNativeCommandExp;
@@ -1673,12 +1680,6 @@ typedef ur_result_t(UR_APICALL *ur_pfnCommandBufferAppendUSMAdviseExp_t)(
     const ur_event_handle_t *, ur_exp_command_buffer_sync_point_t *,
     ur_event_handle_t *, ur_exp_command_buffer_command_handle_t *);
 
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Function-pointer for urCommandBufferEnqueueExp
-typedef ur_result_t(UR_APICALL *ur_pfnCommandBufferEnqueueExp_t)(
-    ur_exp_command_buffer_handle_t, ur_queue_handle_t, uint32_t,
-    const ur_event_handle_t *, ur_event_handle_t *);
-
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Function-pointer for urCommandBufferUpdateKernelLaunchExp
 typedef ur_result_t(UR_APICALL *ur_pfnCommandBufferUpdateKernelLaunchExp_t)(
@@ -1722,7 +1723,6 @@ typedef struct ur_command_buffer_exp_dditable_t {
   ur_pfnCommandBufferAppendMemBufferFillExp_t pfnAppendMemBufferFillExp;
   ur_pfnCommandBufferAppendUSMPrefetchExp_t pfnAppendUSMPrefetchExp;
   ur_pfnCommandBufferAppendUSMAdviseExp_t pfnAppendUSMAdviseExp;
-  ur_pfnCommandBufferEnqueueExp_t pfnEnqueueExp;
   ur_pfnCommandBufferUpdateKernelLaunchExp_t pfnUpdateKernelLaunchExp;
   ur_pfnCommandBufferUpdateSignalEventExp_t pfnUpdateSignalEventExp;
   ur_pfnCommandBufferUpdateWaitEventsExp_t pfnUpdateWaitEventsExp;

unified-runtime/include/ur_print.h

@@ -2686,6 +2686,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintEnqueueUsmFreeExpParams(
     const struct ur_enqueue_usm_free_exp_params_t *params, char *buffer,
     const size_t buff_size, size_t *out_size);
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print ur_enqueue_command_buffer_exp_params_t struct
+/// @returns
+///     - ::UR_RESULT_SUCCESS
+///     - ::UR_RESULT_ERROR_INVALID_SIZE
+///         - `buff_size < out_size`
+UR_APIEXPORT ur_result_t UR_APICALL urPrintEnqueueCommandBufferExpParams(
+    const struct ur_enqueue_command_buffer_exp_params_t *params, char *buffer,
+    const size_t buff_size, size_t *out_size);
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Print ur_enqueue_cooperative_kernel_launch_exp_params_t struct
 /// @returns
@@ -3301,16 +3311,6 @@ urPrintCommandBufferAppendUsmAdviseExpParams(
     const struct ur_command_buffer_append_usm_advise_exp_params_t *params,
     char *buffer, const size_t buff_size, size_t *out_size);
 
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print ur_command_buffer_enqueue_exp_params_t struct
-/// @returns
-///     - ::UR_RESULT_SUCCESS
-///     - ::UR_RESULT_ERROR_INVALID_SIZE
-///         - `buff_size < out_size`
-UR_APIEXPORT ur_result_t UR_APICALL urPrintCommandBufferEnqueueExpParams(
-    const struct ur_command_buffer_enqueue_exp_params_t *params, char *buffer,
-    const size_t buff_size, size_t *out_size);
-
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Print ur_command_buffer_update_kernel_launch_exp_params_t struct
 /// @returns

unified-runtime/include/ur_print.hpp

@@ -1168,8 +1168,8 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_function_t value) {
   case UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_ADVISE_EXP:
     os << "UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_ADVISE_EXP";
     break;
-  case UR_FUNCTION_COMMAND_BUFFER_ENQUEUE_EXP:
-    os << "UR_FUNCTION_COMMAND_BUFFER_ENQUEUE_EXP";
+  case UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP:
+    os << "UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP";
     break;
   case UR_FUNCTION_COMMAND_BUFFER_UPDATE_SIGNAL_EVENT_EXP:
     os << "UR_FUNCTION_COMMAND_BUFFER_UPDATE_SIGNAL_EVENT_EXP";
@@ -10093,8 +10093,8 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_command_t value) {
   case UR_COMMAND_WRITE_HOST_PIPE:
     os << "UR_COMMAND_WRITE_HOST_PIPE";
     break;
-  case UR_COMMAND_COMMAND_BUFFER_ENQUEUE_EXP:
-    os << "UR_COMMAND_COMMAND_BUFFER_ENQUEUE_EXP";
+  case UR_COMMAND_ENQUEUE_COMMAND_BUFFER_EXP:
+    os << "UR_COMMAND_ENQUEUE_COMMAND_BUFFER_EXP";
     break;
   case UR_COMMAND_EXTERNAL_SEMAPHORE_WAIT_EXP:
     os << "UR_COMMAND_EXTERNAL_SEMAPHORE_WAIT_EXP";
@@ -16727,6 +16727,53 @@ inline std::ostream &operator<<(
   return os;
 }
 
+///////////////////////////////////////////////////////////////////////////////
+/// @brief Print operator for the ur_enqueue_command_buffer_exp_params_t type
+/// @returns
+///     std::ostream &
+inline std::ostream &
+operator<<(std::ostream &os,
+           [[maybe_unused]] const struct ur_enqueue_command_buffer_exp_params_t
+               *params) {
+
+  os << ".hQueue = ";
+
+  ur::details::printPtr(os, *(params->phQueue));
+
+  os << ", ";
+  os << ".hCommandBuffer = ";
+
+  ur::details::printPtr(os, *(params->phCommandBuffer));
+
+  os << ", ";
+  os << ".numEventsInWaitList = ";
+
+  os << *(params->pnumEventsInWaitList);
+
+  os << ", ";
+  os << ".phEventWaitList = ";
+  ur::details::printPtr(
+      os, reinterpret_cast<const void *>(*(params->pphEventWaitList)));
+  if (*(params->pphEventWaitList) != NULL) {
+    os << " {";
+    for (size_t i = 0; i < *params->pnumEventsInWaitList; ++i) {
+      if (i != 0) {
+        os << ", ";
+      }
+
+      ur::details::printPtr(os, (*(params->pphEventWaitList))[i]);
+    }
+    os << "}";
+  }
+
+  os << ", ";
+  os << ".phEvent = ";
+
+  ur::details::printPtr(os, *(params->pphEvent));
+
+  return os;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Print operator for the
 /// ur_enqueue_cooperative_kernel_launch_exp_params_t type
@@ -19381,53 +19428,6 @@ operator<<(std::ostream &os, [[maybe_unused]] const struct
   return os;
 }
 
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ur_command_buffer_enqueue_exp_params_t type
-/// @returns
-///     std::ostream &
-inline std::ostream &
-operator<<(std::ostream &os,
-           [[maybe_unused]] const struct ur_command_buffer_enqueue_exp_params_t
-               *params) {
-
-  os << ".hCommandBuffer = ";
-
-  ur::details::printPtr(os, *(params->phCommandBuffer));
-
-  os << ", ";
-  os << ".hQueue = ";
-
-  ur::details::printPtr(os, *(params->phQueue));
-
-  os << ", ";
-  os << ".numEventsInWaitList = ";
-
-  os << *(params->pnumEventsInWaitList);
-
-  os << ", ";
-  os << ".phEventWaitList = ";
-  ur::details::printPtr(
-      os, reinterpret_cast<const void *>(*(params->pphEventWaitList)));
-  if (*(params->pphEventWaitList) != NULL) {
-    os << " {";
-    for (size_t i = 0; i < *params->pnumEventsInWaitList; ++i) {
-      if (i != 0) {
-        os << ", ";
-      }
-
-      ur::details::printPtr(os, (*(params->pphEventWaitList))[i]);
-    }
-    os << "}";
-  }
-
-  os << ", ";
-  os << ".phEvent = ";
-
-  ur::details::printPtr(os, *(params->pphEvent));
-
-  return os;
-}
-
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Print operator for the
 /// ur_command_buffer_update_kernel_launch_exp_params_t type
@@ -20664,6 +20664,9 @@ inline ur_result_t UR_APICALL printFunctionParams(std::ostream &os,
   case UR_FUNCTION_ENQUEUE_USM_FREE_EXP: {
     os << (const struct ur_enqueue_usm_free_exp_params_t *)params;
   } break;
+  case UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP: {
+    os << (const struct ur_enqueue_command_buffer_exp_params_t *)params;
+  } break;
   case UR_FUNCTION_ENQUEUE_COOPERATIVE_KERNEL_LAUNCH_EXP: {
     os << (const struct ur_enqueue_cooperative_kernel_launch_exp_params_t *)
             params;
@@ -20863,9 +20866,6 @@ inline ur_result_t UR_APICALL printFunctionParams(std::ostream &os,
     os << (const struct ur_command_buffer_append_usm_advise_exp_params_t *)
             params;
   } break;
-  case UR_FUNCTION_COMMAND_BUFFER_ENQUEUE_EXP: {
-    os << (const struct ur_command_buffer_enqueue_exp_params_t *)params;
-  } break;
   case UR_FUNCTION_COMMAND_BUFFER_UPDATE_KERNEL_LAUNCH_EXP: {
     os << (const struct ur_command_buffer_update_kernel_launch_exp_params_t *)
             params;