Keep StorageInitHelper::MEvents in pool.

Author: Alexandr-Konovalov

sycl/source/detail/cg.hpp

@@ -21,6 +21,7 @@
 
 #include <assert.h> // for assert
 #include <memory>   // for shared_ptr, unique_ptr
+#include <memory_resource> // for std::pmr
 #include <stddef.h> // for size_t
 #include <stdint.h> // for int32_t
 #include <string>   // for string
@@ -166,14 +167,24 @@ class CG {
                       std::vector<detail::AccessorImplPtr> AccStorage,
                       std::vector<std::shared_ptr<const void>> SharedPtrStorage,
                       std::vector<AccessorImplHost *> Requirements,
-                      std::vector<detail::EventImplPtr> Events)
+                      std::pmr::vector<detail::EventImplPtr> Events)
         : MArgsStorage(std::move(ArgsStorage)),
           MAccStorage(std::move(AccStorage)),
           MSharedPtrStorage(std::move(SharedPtrStorage)),
           MRequirements(std::move(Requirements)), MEvents(std::move(Events)) {}
-    StorageInitHelper(StorageInitHelper &&) = default;
-    StorageInitHelper(const StorageInitHelper &) = default;
-    // The following storages are needed to ensure that arguments won't die
+    StorageInitHelper(StorageInitHelper &&SIH)
+        : MArgsStorage(std::move(SIH.MArgsStorage)),
+          MAccStorage(std::move(SIH.MAccStorage)),
+          MSharedPtrStorage(std::move(SIH.MSharedPtrStorage)),
+          MRequirements(std::move(SIH.MRequirements)),
+          MEvents(SIH.MEvents.begin(), SIH.MEvents.end()) {}
+    StorageInitHelper(const StorageInitHelper &SIH)
+        : MArgsStorage(SIH.MArgsStorage),
+          MAccStorage(SIH.MAccStorage),
+          MSharedPtrStorage(SIH.MSharedPtrStorage),
+          MRequirements(SIH.MRequirements),
+          MEvents(SIH.MEvents.begin(), SIH.MEvents.end()) {}
+// The following storages are needed to ensure that arguments won't die
     // while we are using them.
     /// Storage for standard layout arguments.
     std::vector<std::vector<char>> MArgsStorage;
@@ -185,8 +196,13 @@ class CG {
     /// List of requirements that specify which memory is needed for the command
     /// group to be executed.
     std::vector<AccessorImplHost *> MRequirements;
+
+    std::array<std::byte, 4 * 1024> MEventsBuf;
+    std::pmr::monotonic_buffer_resource MEventsBufRes{
+      MEventsBuf.data(), MEventsBuf.size()};
+
     /// List of events that order the execution of this CG
-    std::vector<detail::EventImplPtr> MEvents;
+    std::pmr::vector<detail::EventImplPtr> MEvents{&MEventsBufRes};
   };
 
   CG(CGType Type, StorageInitHelper D, detail::code_location loc = {},
@@ -222,7 +238,7 @@ class CG {
   std::vector<AccessorImplHost *> &getRequirements() {
     return MData.MRequirements;
   }
-  std::vector<detail::EventImplPtr> &getEvents() { return MData.MEvents; }
+  std::pmr::vector<detail::EventImplPtr> &getEvents() { return MData.MEvents; }
 
   virtual std::vector<std::shared_ptr<const void>>
   getAuxiliaryResources() const {

sycl/source/detail/graph_impl.cpp

@@ -983,14 +983,14 @@ exec_graph_impl::enqueue(const std::shared_ptr<sycl::detail::queue_impl> &Queue,
   sycl::detail::EventImplPtr NewEvent;
   std::vector<sycl::detail::EventImplPtr> BackupCGDataMEvents;
   if (MPartitions.size() > 1) {
-    BackupCGDataMEvents = CGData.MEvents;
+    BackupCGDataMEvents.assign(CGData.MEvents.begin(), CGData.MEvents.end());
   }
   for (uint32_t currentPartitionsNum = 0;
        currentPartitionsNum < MPartitions.size(); currentPartitionsNum++) {
     auto CurrentPartition = MPartitions[currentPartitionsNum];
     // restore initial MEvents to add only needed additional depenencies
     if (currentPartitionsNum > 0) {
-      CGData.MEvents = BackupCGDataMEvents;
+      CGData.MEvents.assign(BackupCGDataMEvents.begin(), BackupCGDataMEvents.end());
     }
 
     for (auto const &DepPartition : CurrentPartition->MPredecessors) {

sycl/source/detail/jit_compiler.cpp

@@ -786,7 +786,7 @@ jit_compiler::fuseKernels(QueueImplPtr Queue,
   std::vector<std::vector<char>> &ArgsStorage = CGData.MArgsStorage;
   std::vector<detail::AccessorImplPtr> &AccStorage = CGData.MAccStorage;
   std::vector<Requirement *> &Requirements = CGData.MRequirements;
-  std::vector<detail::EventImplPtr> &Events = CGData.MEvents;
+  std::pmr::vector<detail::EventImplPtr> &Events = CGData.MEvents;
   std::vector<::jit_compiler::NDRange> Ranges;
   ur_kernel_cache_config_t KernelCacheConfig = UR_KERNEL_CACHE_CONFIG_DEFAULT;
   unsigned KernelIndex = 0;

sycl/source/detail/scheduler/commands.cpp

@@ -243,8 +243,26 @@ Command::getUrEvents(const std::vector<EventImplPtr> &EventImpls,
 }
 
 std::vector<ur_event_handle_t>
-Command::getUrEvents(const std::vector<EventImplPtr> &EventImpls) const {
-  return getUrEvents(EventImpls, MWorkerQueue, isHostTask());
+Command::getUrEvents(const std::pmr::vector<EventImplPtr> &EventImpls,
+                     const QueueImplPtr &CommandQueue, bool IsHostTaskCommand) {
+  std::vector<ur_event_handle_t> RetUrEvents;
+  for (auto &EventImpl : EventImpls) {
+    auto Handle = EventImpl->getHandle();
+    if (Handle == nullptr)
+      continue;
+
+    // Do not add redundant event dependencies for in-order queues.
+    // At this stage dependency is definitely ur task and need to check if
+    // current one is a host task. In this case we should not skip ur event due
+    // to different sync mechanisms for different task types on in-order queue.
+    if (CommandQueue && EventImpl->getWorkerQueue() == CommandQueue &&
+        CommandQueue->isInOrder() && !IsHostTaskCommand)
+      continue;
+
+    RetUrEvents.push_back(Handle);
+  }
+
+  return RetUrEvents;
 }
 
 // This function is implemented (duplicating getUrEvents a lot) as short term

sycl/source/detail/scheduler/commands.hpp

@@ -238,12 +238,19 @@ class Command {
 
   /// Collect UR events from EventImpls and filter out some of them in case of
   /// in order queue
+  template <typename ContT>
   std::vector<ur_event_handle_t>
-  getUrEvents(const std::vector<EventImplPtr> &EventImpls) const;
-
+  getUrEvents(const ContT &EventImpls) const {
+    return getUrEvents(EventImpls, MWorkerQueue, isHostTask());
+  }
+  
   static std::vector<ur_event_handle_t>
   getUrEvents(const std::vector<EventImplPtr> &EventImpls,
               const QueueImplPtr &CommandQueue, bool IsHostTaskCommand);
+  static std::vector<ur_event_handle_t>
+  getUrEvents(const std::pmr::vector<EventImplPtr> &EventImpls,
+              const QueueImplPtr &CommandQueue, bool IsHostTaskCommand);
+              
   /// Collect UR events from EventImpls and filter out some of them in case of
   /// in order queue. Does blocking enqueue if event is expected to produce ur
   /// event but has empty native handle.

sycl/source/detail/scheduler/graph_builder.cpp

@@ -926,7 +926,7 @@ Command *Scheduler::GraphBuilder::addCG(
     ur_exp_command_buffer_handle_t CommandBuffer,
     const std::vector<ur_exp_command_buffer_sync_point_t> &Dependencies) {
   std::vector<Requirement *> &Reqs = CommandGroup->getRequirements();
-  std::vector<detail::EventImplPtr> &Events = CommandGroup->getEvents();
+  std::pmr::vector<detail::EventImplPtr> &Events = CommandGroup->getEvents();
 
   auto NewCmd = std::make_unique<ExecCGCommand>(std::move(CommandGroup), Queue,
                                                 EventNeeded, CommandBuffer,

sycl/source/detail/scheduler/scheduler.cpp

@@ -688,6 +688,15 @@ bool Scheduler::areEventsSafeForSchedulerBypass(
                      });
 }
 
+bool Scheduler::areEventsSafeForSchedulerBypass(
+  const std::pmr::vector<EventImplPtr> &DepEvents, ContextImplPtr Context) {
+
+  return std::all_of(DepEvents.begin(), DepEvents.end(),
+                     [&Context](const EventImplPtr &SyclEventImplPtr) {
+                       return CheckEventReadiness(Context, SyclEventImplPtr);
+                     });
+}
+
 } // namespace detail
 } // namespace _V1
 } // namespace sycl

sycl/source/detail/scheduler/scheduler.hpp

@@ -484,6 +484,10 @@ class Scheduler {
   areEventsSafeForSchedulerBypass(const std::vector<EventImplPtr> &DepEvents,
                                   const ContextImplPtr &Context);
 
+  static bool
+  areEventsSafeForSchedulerBypass(const std::pmr::vector<EventImplPtr> &DepEvents,
+                                  ContextImplPtr Context);
+
 protected:
   using RWLockT = std::shared_timed_mutex;
   using ReadLockT = std::shared_lock<RWLockT>;

sycl/unittests/scheduler/SchedulerTestUtils.hpp

@@ -248,7 +248,7 @@ class MockHandler : public sycl::handler {
   std::vector<sycl::detail::Requirement *> &getRequirements() {
     return impl->CGData.MRequirements;
   }
-  std::vector<sycl::detail::EventImplPtr> &getEvents() {
+  std::pmr::vector<sycl::detail::EventImplPtr> &getEvents() {
     return impl->CGData.MEvents;
   }
   std::vector<sycl::detail::ArgDesc> &getArgs() { return impl->MArgs; }