[SYCL] Simplify secondary queue usage (#18642)

Following KhronosGroup/SYCL-Docs#811, the SYCL
2020 specification will not mandate the use nor exception checking of
secondary queues. This allows us to relax the interfaces taking a
secondary queue to fully ignore it. This commit drops the passing of the
secondary queue throughout the pipeline.

---------

Signed-off-by: Larsen, Steffen <[email protected]>

Author: steffenlarsen

sycl/include/sycl/ext/oneapi/experimental/enqueue_functions.hpp

@@ -109,7 +109,7 @@ event submit_with_event_impl(const queue &Q, PropertiesT Props,
                              CommandGroupFunc &&CGF,
                              const sycl::detail::code_location &CodeLoc) {
   return Q.submit_with_event<__SYCL_USE_FALLBACK_ASSERT>(
-      Props, detail::type_erased_cgfo_ty{CGF}, nullptr, CodeLoc);
+      Props, detail::type_erased_cgfo_ty{CGF}, CodeLoc);
 }
 } // namespace detail
 

sycl/include/sycl/queue.hpp

@@ -66,7 +66,7 @@ auto get_native(const SyclObjectT &Obj)
 namespace detail {
 class queue_impl;
 
-inline event submitAssertCapture(const queue &, event &, queue *,
+inline event submitAssertCapture(const queue &, event &,
                                  const detail::code_location &);
 
 // Function to postprocess submitted command
@@ -87,8 +87,10 @@ class __SYCL_EXPORT SubmissionInfo {
   sycl::detail::optional<SubmitPostProcessF> &PostProcessorFunc();
   const sycl::detail::optional<SubmitPostProcessF> &PostProcessorFunc() const;
 
+#ifndef __INTEL_PREVIEW_BREAKING_CHANGES
   std::shared_ptr<detail::queue_impl> &SecondaryQueue();
   const std::shared_ptr<detail::queue_impl> &SecondaryQueue() const;
+#endif
 
   ext::oneapi::experimental::event_mode_enum &EventMode();
   const ext::oneapi::experimental::event_mode_enum &EventMode() const;
@@ -434,21 +436,18 @@ class __SYCL_EXPORT queue : public detail::OwnerLessBase<queue> {
   /// Submits a command group function object to the queue, in order to be
   /// scheduled for execution on the device.
   ///
-  /// On a kernel error, this command group function object is then scheduled
-  /// for execution on a secondary queue.
-  ///
   /// \param CGF is a function object containing command group.
-  /// \param SecondaryQueue is a fallback SYCL queue.
+  /// \param SecondaryQueue is a fallback SYCL queue. (unused)
   /// \param CodeLoc is the code location of the submit call (default argument)
   /// \return a SYCL event object, which corresponds to the queue the command
   /// group is being enqueued on.
   template <typename T>
   std::enable_if_t<std::is_invocable_r_v<void, T, handler &>, event> submit(
-      T CGF, queue &SecondaryQueue,
+      T CGF, [[maybe_unused]] queue &SecondaryQueue,
       const detail::code_location &CodeLoc = detail::code_location::current()) {
     return submit_with_event<__SYCL_USE_FALLBACK_ASSERT>(
         sycl::ext::oneapi::experimental::empty_properties_t{},
-        detail::type_erased_cgfo_ty{CGF}, &SecondaryQueue, CodeLoc);
+        detail::type_erased_cgfo_ty{CGF}, CodeLoc);
   }
 
   /// Prevents any commands submitted afterward to this queue from executing
@@ -3582,7 +3581,7 @@ class __SYCL_EXPORT queue : public detail::OwnerLessBase<queue> {
       -> backend_return_t<BackendName, SyclObjectT>;
 
 #if __SYCL_USE_FALLBACK_ASSERT
-  friend event detail::submitAssertCapture(const queue &, event &, queue *,
+  friend event detail::submitAssertCapture(const queue &, event &,
                                            const detail::code_location &);
 #endif
 
@@ -3686,47 +3685,6 @@ class __SYCL_EXPORT queue : public detail::OwnerLessBase<queue> {
                                  const detail::code_location &CodeLoc,
                                  bool IsTopCodeLoc) const;
 
-  /// Submits a command group function object to the queue, in order to be
-  /// scheduled for execution on the device.
-  ///
-  /// \param Props is a property list with submission properties.
-  /// \param CGF is a function object containing command group.
-  /// \param SecondaryQueuePtr is a pointer to the secondary queue.
-  /// \param CodeLoc is the code location of the submit call (default argument)
-  /// \return a SYCL event object for the submitted command group.
-  //
-  // UseFallBackAssert as template param vs `#if` in function body is necessary
-  // to prevent ODR-violation between TUs built with different fallback assert
-  // modes.
-  template <bool UseFallbackAssert, typename PropertiesT>
-  event submit_with_event(PropertiesT Props,
-                          const detail::type_erased_cgfo_ty &CGF,
-                          queue *SecondaryQueuePtr,
-                          const detail::code_location &CodeLoc =
-                              detail::code_location::current()) const {
-    detail::tls_code_loc_t TlsCodeLocCapture(CodeLoc);
-    detail::v1::SubmissionInfo SI{};
-    ProcessSubmitProperties(Props, SI);
-    if (SecondaryQueuePtr)
-      SI.SecondaryQueue() = detail::getSyclObjImpl(*SecondaryQueuePtr);
-    if constexpr (UseFallbackAssert)
-      SI.PostProcessorFunc() =
-          [this, &SecondaryQueuePtr,
-           &TlsCodeLocCapture](bool IsKernel, bool KernelUsesAssert, event &E) {
-            if (IsKernel && !device_has(aspect::ext_oneapi_native_assert) &&
-                KernelUsesAssert && !device_has(aspect::accelerator)) {
-              // __devicelib_assert_fail isn't supported by Device-side Runtime
-              // Linking against fallback impl of __devicelib_assert_fail is
-              // performed by program manager class
-              // Fallback assert isn't supported for FPGA
-              submitAssertCapture(*this, E, SecondaryQueuePtr,
-                                  TlsCodeLocCapture.query());
-            }
-          };
-    return submit_with_event_impl(CGF, SI, TlsCodeLocCapture.query(),
-                                  TlsCodeLocCapture.isToplevel());
-  }
-
   /// Submits a command group function object to the queue, in order to be
   /// scheduled for execution on the device.
   ///
@@ -3756,7 +3714,7 @@ class __SYCL_EXPORT queue : public detail::OwnerLessBase<queue> {
           // Linking against fallback impl of __devicelib_assert_fail is
           // performed by program manager class
           // Fallback assert isn't supported for FPGA
-          submitAssertCapture(*this, E, nullptr, TlsCodeLocCapture.query());
+          submitAssertCapture(*this, E, TlsCodeLocCapture.query());
         }
       };
     return submit_with_event_impl(CGF, SI, TlsCodeLocCapture.query(),
@@ -3955,15 +3913,13 @@ class AssertInfoCopier;
  * Submit copy task for assert failure flag and host-task to check the flag
  * \param Event kernel's event to depend on i.e. the event represents the
  *              kernel to check for assertion failure
- * \param SecondaryQueue secondary queue for submit process, null if not used
  * \returns host tasks event
  *
  * This method doesn't belong to queue class to overcome msvc behaviour due to
  * which it gets compiled and exported without any integration header and, thus,
  * with no proper KernelInfo instance.
  */
 event submitAssertCapture(const queue &Self, event &Event,
-                          queue *SecondaryQueue,
                           const detail::code_location &CodeLoc) {
   buffer<detail::AssertHappened, 1> Buffer{1};
 
@@ -4019,10 +3975,10 @@ event submitAssertCapture(const queue &Self, event &Event,
 
   CopierEv = Self.submit_with_event<true>(
       sycl::ext::oneapi::experimental::empty_properties_t{}, CopierCGF,
-      SecondaryQueue, CodeLoc);
+      CodeLoc);
   CheckerEv = Self.submit_with_event<true>(
       sycl::ext::oneapi::experimental::empty_properties_t{}, CheckerCGF,
-      SecondaryQueue, CodeLoc);
+      CodeLoc);
 
   return CheckerEv;
 }

sycl/source/detail/handler_impl.hpp

@@ -31,10 +31,8 @@ enum class HandlerSubmissionState : std::uint8_t {
 
 class handler_impl {
 public:
-  handler_impl(queue_impl &Queue, queue_impl *SubmissionSecondaryQueue,
-               bool EventNeeded)
-      : MSubmissionSecondaryQueue(SubmissionSecondaryQueue),
-        MEventNeeded(EventNeeded), MQueueOrGraph{Queue} {};
+  handler_impl(queue_impl &Queue, bool EventNeeded)
+      : MEventNeeded(EventNeeded), MQueueOrGraph{Queue} {};
 
   handler_impl(ext::oneapi::experimental::detail::graph_impl &Graph)
       : MQueueOrGraph{Graph} {}
@@ -65,10 +63,6 @@ class handler_impl {
   /// Registers mutually exclusive submission states.
   HandlerSubmissionState MSubmissionState = HandlerSubmissionState::NO_STATE;
 
-  /// Pointer to the secondary queue implementation. Nullptr if no
-  /// secondary queue fallback was given in the associated submission.
-  queue_impl *MSubmissionSecondaryQueue = nullptr;
-
   /// Bool stores information about whether the event resulting from the
   /// corresponding work is required.
   bool MEventNeeded = true;

sycl/source/detail/queue_impl.cpp

@@ -304,14 +304,14 @@ void queue_impl::addEvent(const detail::EventImplPtr &EventImpl) {
 
 detail::EventImplPtr
 queue_impl::submit_impl(const detail::type_erased_cgfo_ty &CGF,
-                        queue_impl *SecondaryQueue, bool CallerNeedsEvent,
-                        const detail::code_location &Loc, bool IsTopCodeLoc,
+                        bool CallerNeedsEvent, const detail::code_location &Loc,
+                        bool IsTopCodeLoc,
                         const v1::SubmissionInfo &SubmitInfo) {
 #ifdef __INTEL_PREVIEW_BREAKING_CHANGES
-  detail::handler_impl HandlerImplVal(*this, SecondaryQueue, CallerNeedsEvent);
+  detail::handler_impl HandlerImplVal(*this, CallerNeedsEvent);
   handler Handler(HandlerImplVal);
 #else
-  handler Handler(shared_from_this(), SecondaryQueue, CallerNeedsEvent);
+  handler Handler(shared_from_this(), CallerNeedsEvent);
 #endif
   detail::handler_impl &HandlerImpl = *detail::getSyclObjImpl(Handler);
 
@@ -390,8 +390,8 @@ queue_impl::submit_impl(const detail::type_erased_cgfo_ty &CGF,
       Stream->generateFlushCommand(ServiceCGH);
     };
     detail::type_erased_cgfo_ty CGF{L};
-    detail::EventImplPtr FlushEvent = submit_impl(
-        CGF, SecondaryQueue, /*CallerNeedsEvent*/ true, Loc, IsTopCodeLoc, {});
+    detail::EventImplPtr FlushEvent =
+        submit_impl(CGF, /*CallerNeedsEvent*/ true, Loc, IsTopCodeLoc, {});
     if (EventImpl)
       EventImpl->attachEventToCompleteWeak(FlushEvent);
     if (!isInOrder()) {
@@ -403,18 +403,6 @@ queue_impl::submit_impl(const detail::type_erased_cgfo_ty &CGF,
   return EventImpl;
 }
 
-#ifndef __INTEL_PREVIEW_BREAKING_CHANGES
-detail::EventImplPtr
-queue_impl::submit_impl(const detail::type_erased_cgfo_ty &CGF,
-                        const std::shared_ptr<queue_impl> & /*PrimaryQueue*/,
-                        const std::shared_ptr<queue_impl> &SecondaryQueue,
-                        bool CallerNeedsEvent, const detail::code_location &Loc,
-                        bool IsTopCodeLoc, const SubmissionInfo &SubmitInfo) {
-  return submit_impl(CGF, SecondaryQueue.get(), CallerNeedsEvent, Loc,
-                     IsTopCodeLoc, SubmitInfo);
-}
-#endif
-
 template <typename HandlerFuncT>
 event queue_impl::submitWithHandler(const std::vector<event> &DepEvents,
                                     bool CallerNeedsEvent,

sycl/source/detail/queue_impl.hpp

@@ -66,7 +66,9 @@ enum QueueOrder { Ordered, OOO };
 // Implementation of the submission information storage.
 struct SubmissionInfoImpl {
   optional<detail::SubmitPostProcessF> MPostProcessorFunc = std::nullopt;
+#ifndef __INTEL_PREVIEW_BREAKING_CHANGES
   std::shared_ptr<detail::queue_impl> MSecondaryQueue = nullptr;
+#endif
   ext::oneapi::experimental::event_mode_enum MEventMode =
       ext::oneapi::experimental::event_mode_enum::none;
 };
@@ -331,22 +333,16 @@ class queue_impl : public std::enable_shared_from_this<queue_impl> {
   /// Submits a command group function object to the queue, in order to be
   /// scheduled for execution on the device.
   ///
-  /// On a kernel error, this command group function object is then scheduled
-  /// for execution on a secondary queue.
-  ///
   /// \param CGF is a function object containing command group.
-  /// \param SecondQueue is a shared_ptr to the secondary queue.
   /// \param Loc is the code location of the submit call (default argument)
   /// \param StoreAdditionalInfo makes additional info be stored in event_impl
   /// \return a SYCL event object, which corresponds to the queue the command
   /// group is being enqueued on.
   event submit(const detail::type_erased_cgfo_ty &CGF,
-               const std::shared_ptr<queue_impl> &SecondQueue,
                const detail::code_location &Loc, bool IsTopCodeLoc,
                const SubmitPostProcessF *PostProcess = nullptr) {
     event ResEvent;
     v1::SubmissionInfo SI{};
-    SI.SecondaryQueue() = SecondQueue;
     if (PostProcess)
       SI.PostProcessorFunc() = *PostProcess;
     return submit_with_event(CGF, SI, Loc, IsTopCodeLoc);
@@ -364,18 +360,16 @@ class queue_impl : public std::enable_shared_from_this<queue_impl> {
                           const v1::SubmissionInfo &SubmitInfo,
                           const detail::code_location &Loc, bool IsTopCodeLoc) {
 
-    detail::EventImplPtr ResEvent =
-        submit_impl(CGF, SubmitInfo.SecondaryQueue().get(),
-                    /*CallerNeedsEvent=*/true, Loc, IsTopCodeLoc, SubmitInfo);
+    detail::EventImplPtr ResEvent = submit_impl(CGF, /*CallerNeedsEvent=*/true,
+                                                Loc, IsTopCodeLoc, SubmitInfo);
     return createSyclObjFromImpl<event>(ResEvent);
   }
 
   void submit_without_event(const detail::type_erased_cgfo_ty &CGF,
                             const v1::SubmissionInfo &SubmitInfo,
                             const detail::code_location &Loc,
                             bool IsTopCodeLoc) {
-    submit_impl(CGF, SubmitInfo.SecondaryQueue().get(),
-                /*CallerNeedsEvent=*/false, Loc, IsTopCodeLoc, SubmitInfo);
+    submit_impl(CGF, /*CallerNeedsEvent=*/false, Loc, IsTopCodeLoc, SubmitInfo);
   }
 
   /// Performs a blocking wait for the completion of all enqueued tasks in the
@@ -885,38 +879,15 @@ class queue_impl : public std::enable_shared_from_this<queue_impl> {
     PostProcess(IsKernel, KernelUsesAssert, Event);
   }
 
-#ifndef __INTEL_PREVIEW_BREAKING_CHANGES
-  /// Performs command group submission to the queue.
-  ///
-  /// \param CGF is a function object containing command group.
-  /// \param PrimaryQueue is a pointer to the primary queue. This may be the
-  ///        same as this.
-  /// \param SecondaryQueue is a pointer to the secondary queue. This may be the
-  ///        same as this.
-  /// \param CallerNeedsEvent is a boolean indicating whether the event is
-  ///        required by the user after the call.
-  /// \param Loc is the code location of the submit call (default argument)
-  /// \param SubmitInfo is additional optional information for the submission.
-  /// \return a SYCL event representing submitted command group.
-  detail::EventImplPtr
-  submit_impl(const detail::type_erased_cgfo_ty &CGF,
-              const std::shared_ptr<queue_impl> &PrimaryQueue,
-              const std::shared_ptr<queue_impl> &SecondaryQueue,
-              bool CallerNeedsEvent, const detail::code_location &Loc,
-              bool IsTopCodeLoc, const SubmissionInfo &SubmitInfo);
-#endif
-
   /// Performs command group submission to the queue.
   ///
   /// \param CGF is a function object containing command group.
-  /// \param SecondaryQueue is a pointer to the secondary queue.
   /// \param CallerNeedsEvent is a boolean indicating whether the event is
   ///        required by the user after the call.
   /// \param Loc is the code location of the submit call (default argument)
   /// \param SubmitInfo is additional optional information for the submission.
   /// \return a SYCL event representing submitted command group.
   detail::EventImplPtr submit_impl(const detail::type_erased_cgfo_ty &CGF,
-                                   queue_impl *SecondaryQueue,
                                    bool CallerNeedsEvent,
                                    const detail::code_location &Loc,
                                    bool IsTopCodeLoc,

sycl/source/detail/scheduler/scheduler.hpp

@@ -587,14 +587,6 @@ class Scheduler {
 
     void cleanupCommand(Command *Cmd, bool AllowUnsubmitted = false);
 
-    /// Reschedules the command passed using Queue provided.
-    ///
-    /// This can lead to rescheduling of all dependent commands. This can be
-    /// used when the user provides a "secondary" queue to the submit method
-    /// which may be used when the command fails to enqueue/execute in the
-    /// primary queue.
-    void rescheduleCommand(Command *Cmd, queue_impl *Queue);
-
     /// \return a pointer to the corresponding memory object record for the
     /// SYCL memory object provided, or nullptr if it does not exist.
     MemObjRecord *getMemObjRecord(SYCLMemObjI *MemObject);

sycl/source/handler.cpp

@@ -353,22 +353,21 @@ handler::handler(std::unique_ptr<detail::handler_impl> &&HandlerImpl)
 
 handler::handler(std::shared_ptr<detail::queue_impl> Queue,
                  bool CallerNeedsEvent)
-    : impl(std::make_shared<detail::handler_impl>(*Queue, nullptr,
-                                                  CallerNeedsEvent)),
+    : impl(std::make_shared<detail::handler_impl>(*Queue, CallerNeedsEvent)),
       MQueueDoNotUse(std::move(Queue)) {}
 
 handler::handler(
     std::shared_ptr<detail::queue_impl> Queue,
     [[maybe_unused]] std::shared_ptr<detail::queue_impl> PrimaryQueue,
-    std::shared_ptr<detail::queue_impl> SecondaryQueue, bool CallerNeedsEvent)
-    : impl(std::make_shared<detail::handler_impl>(*Queue, SecondaryQueue.get(),
-                                                  CallerNeedsEvent)),
+    [[maybe_unused]] std::shared_ptr<detail::queue_impl> SecondaryQueue,
+    bool CallerNeedsEvent)
+    : impl(std::make_shared<detail::handler_impl>(*Queue, CallerNeedsEvent)),
       MQueueDoNotUse(Queue) {}
 
 handler::handler(std::shared_ptr<detail::queue_impl> Queue,
-                 detail::queue_impl *SecondaryQueue, bool CallerNeedsEvent)
-    : impl(std::make_shared<detail::handler_impl>(*Queue, SecondaryQueue,
-                                                  CallerNeedsEvent)),
+                 [[maybe_unused]] detail::queue_impl *SecondaryQueue,
+                 bool CallerNeedsEvent)
+    : impl(std::make_shared<detail::handler_impl>(*Queue, CallerNeedsEvent)),
       MQueueDoNotUse(std::move(Queue)) {}
 
 handler::handler(
@@ -2021,14 +2020,6 @@ void handler::use_kernel_bundle(
         "Context associated with the primary queue is different from the "
         "context associated with the kernel bundle");
 
-  if (impl->MSubmissionSecondaryQueue &&
-      impl->MSubmissionSecondaryQueue->get_context() !=
-          ExecBundle.get_context())
-    throw sycl::exception(
-        make_error_code(errc::invalid),
-        "Context associated with the secondary queue is different from the "
-        "context associated with the kernel bundle");
-
   setStateExplicitKernelBundle();
   setHandlerKernelBundle(detail::getSyclObjImpl(ExecBundle));
 }
@@ -2649,7 +2640,7 @@ __SYCL_EXPORT void HandlerAccess::preProcess(handler &CGH,
   queue_impl &Q = CGH.impl->get_queue();
   bool EventNeeded = !Q.isInOrder();
 #ifdef __INTEL_PREVIEW_BREAKING_CHANGES
-  handler_impl HandlerImpl{Q, nullptr, EventNeeded};
+  handler_impl HandlerImpl{Q, EventNeeded};
   handler AuxHandler{HandlerImpl};
 #else
   handler AuxHandler{Q.shared_from_this(), EventNeeded};
@@ -2668,8 +2659,7 @@ __SYCL_EXPORT void HandlerAccess::postProcess(handler &CGH,
   if (!InOrder)
     CGH.impl->MEventNeeded = true;
 
-  handler PostProcessHandler{
-      std::make_unique<handler_impl>(Q, nullptr, EventNeeded)};
+  handler PostProcessHandler{std::make_unique<handler_impl>(Q, EventNeeded)};
   PostProcessHandler.copyCodeLoc(CGH);
   // Extend lifetimes of auxiliary resources till the last kernel in the chain
   // finishes: