[L0 v2] implement native handle API for event

Author: igchor

source/adapters/level_zero/v2/api.cpp

@@ -181,21 +181,6 @@ ur_result_t urKernelGetSuggestedLocalWorkSize(ur_kernel_handle_t hKernel,
   return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
 
-ur_result_t urEventGetNativeHandle(ur_event_handle_t hEvent,
-                                   ur_native_handle_t *phNativeEvent) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-ur_result_t
-urEventCreateWithNativeHandle(ur_native_handle_t hNativeEvent,
-                              ur_context_handle_t hContext,
-                              const ur_event_native_properties_t *pProperties,
-                              ur_event_handle_t *phEvent) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
 ur_result_t urEventSetCallback(ur_event_handle_t hEvent,
                                ur_execution_info_t execStatus,
                                ur_event_callback_t pfnNotify, void *pUserData) {

source/adapters/level_zero/v2/event.cpp

@@ -10,39 +10,36 @@
 
 #include <ze_api.h>
 
+#include "context.hpp"
 #include "event.hpp"
 #include "event_pool.hpp"
 #include "event_provider.hpp"
 #include "queue_api.hpp"
 
 #include "../ur_interface_loader.hpp"
 
-ur_event_handle_t_::ur_event_handle_t_(
-    v2::raii::cache_borrowed_event eventAllocation, v2::event_pool *pool)
-    : zeEvent(std::move(eventAllocation)), pool(pool),
+ur_event_handle_t_::ur_event_handle_t_(ze_event_handle_t hZeEvent,
+                                       ur_device_handle_t hDevice,
+                                       ur_command_t commandType,
+                                       v2::event_flags_t flags)
+    : hZeEvent(hZeEvent), hDevice(hDevice), flags(flags),
       adjustedEventStartTimestamp(0), recordEventEndTimestamp(0),
       adjustedEventEndTimestamp(0),
       zeTimerResolution(getDevice()->ZeDeviceProperties->timerResolution),
-      timestampMaxValue(getDevice()->getTimestampMask()) {}
-
-void ur_event_handle_t_::resetQueueAndCommand(ur_queue_handle_t hQueue,
-                                              ur_command_t commandType) {
-  this->hQueue = hQueue;
-  this->commandType = commandType;
-}
+      timestampMaxValue(getDevice()->getTimestampMask()),
+      commandType(commandType) {}
 
 void ur_event_handle_t_::reset() {
   // consider make an abstraction for regular/counter based
   // events if there's more of this type of conditions
-  if (pool->getFlags() & v2::EVENT_FLAGS_COUNTER) {
-    zeEventHostReset(zeEvent.get());
+  if (flags & v2::EVENT_FLAGS_COUNTER) {
+    zeEventHostReset(hZeEvent);
   }
 }
 
 ze_event_handle_t ur_event_handle_t_::getZeEvent() const {
-  assert(hQueue);
   assert(commandType != UR_COMMAND_FORCE_UINT32);
-  return zeEvent.get();
+  return hZeEvent;
 }
 
 ur_result_t ur_event_handle_t_::retain() {
@@ -51,11 +48,10 @@ ur_result_t ur_event_handle_t_::retain() {
 }
 
 ur_result_t ur_event_handle_t_::releaseDeferred() {
-  assert(zeEventQueryStatus(zeEvent.get()) == ZE_RESULT_SUCCESS);
+  assert(zeEventQueryStatus(hZeEvent) == ZE_RESULT_SUCCESS);
   assert(RefCount.load() == 0);
 
-  pool->free(this);
-  return UR_RESULT_SUCCESS;
+  return this->forceRelease();
 }
 
 ur_result_t ur_event_handle_t_::release() {
@@ -69,14 +65,12 @@ ur_result_t ur_event_handle_t_::release() {
     // L0 will write end timestamp to this event some time in the future,
     // so we can't release it yet.
 
-    assert(hQueue);
+    assert(hQueue); // hQueue must have been set in recordStartTimestamp
     hQueue->deferEventFree(this);
     return UR_RESULT_SUCCESS;
   }
 
-  pool->free(this);
-
-  return UR_RESULT_SUCCESS;
+  return this->forceRelease();
 }
 
 bool ur_event_handle_t_::isTimestamped() const {
@@ -85,12 +79,10 @@ bool ur_event_handle_t_::isTimestamped() const {
 }
 
 bool ur_event_handle_t_::isProfilingEnabled() const {
-  return pool->getFlags() & v2::EVENT_FLAGS_PROFILING_ENABLED;
+  return flags & v2::EVENT_FLAGS_PROFILING_ENABLED;
 }
 
-ur_device_handle_t ur_event_handle_t_::getDevice() const {
-  return pool->getProvider()->device();
-}
+ur_device_handle_t ur_event_handle_t_::getDevice() const { return hDevice; }
 
 uint64_t ur_event_handle_t_::getEventStartTimestmap() const {
   return adjustedEventStartTimestamp;
@@ -121,7 +113,7 @@ uint64_t ur_event_handle_t_::getEventEndTimestamp() {
   if (adjustedEventEndTimestamp)
     return adjustedEventEndTimestamp;
 
-  auto status = zeEventQueryStatus(zeEvent.get());
+  auto status = zeEventQueryStatus(hZeEvent);
   if (status != ZE_RESULT_SUCCESS) {
     // profiling info not ready
     return 0;
@@ -134,7 +126,12 @@ uint64_t ur_event_handle_t_::getEventEndTimestamp() {
   return adjustedEventEndTimestamp;
 }
 
-void ur_event_handle_t_::recordStartTimestamp() {
+void ur_event_handle_t_::recordStartTimestamp(ur_queue_handle_t hQueue) {
+  // this->hQueue can be for events created from native handle
+  assert(hQueue == this->hQueue || !this->hQueue);
+
+  this->hQueue = hQueue;
+
   uint64_t deviceStartTimestamp = 0;
   UR_CALL_THROWS(ur::level_zero::urDeviceGetGlobalTimestamps(
       getDevice(), &deviceStartTimestamp, nullptr));
@@ -145,13 +142,45 @@ void ur_event_handle_t_::recordStartTimestamp() {
 
 std::pair<uint64_t *, ze_event_handle_t>
 ur_event_handle_t_::getEventEndTimestampAndHandle() {
-  return {&recordEventEndTimestamp, zeEvent.get()};
+  return {&recordEventEndTimestamp, hZeEvent};
 }
 
 ur_queue_handle_t ur_event_handle_t_::getQueue() const { return hQueue; }
 
 ur_command_t ur_event_handle_t_::getCommandType() const { return commandType; }
 
+ur_pooled_event_t::ur_pooled_event_t(
+    v2::raii::cache_borrowed_event eventAllocation, v2::event_pool *pool)
+    : ur_event_handle_t_(eventAllocation.get(), pool->getProvider()->device(),
+                         UR_COMMAND_FORCE_UINT32, pool->getFlags()),
+      zeEvent(std::move(eventAllocation)), pool(pool) {}
+
+void ur_pooled_event_t::resetQueueAndCommand(ur_queue_handle_t hQueue,
+                                             ur_command_t commandType) {
+  this->hQueue = hQueue;
+  this->commandType = commandType;
+}
+
+ur_result_t ur_pooled_event_t::forceRelease() {
+  pool->free(this);
+  return UR_RESULT_SUCCESS;
+}
+
+ur_native_event_t::ur_native_event_t(
+    ur_native_handle_t hNativeEvent, ur_context_handle_t hContext,
+    const ur_event_native_properties_t *pProperties)
+    : ur_event_handle_t_(reinterpret_cast<ze_event_handle_t>(hNativeEvent),
+                         hContext->getDevices()[0],
+                         UR_EXT_COMMAND_TYPE_USER, v2::EVENT_FLAGS_PROFILING_ENABLED /* TODO: this follows legacy adapter logic, we could check this with zeEventGetPool */),
+      zeEvent(reinterpret_cast<ze_event_handle_t>(hNativeEvent),
+              pProperties ? pProperties->isNativeHandleOwned : false) {}
+
+ur_result_t ur_native_event_t::forceRelease() {
+  zeEvent.release();
+  delete this;
+  return UR_RESULT_SUCCESS;
+}
+
 namespace ur::level_zero {
 ur_result_t urEventRetain(ur_event_handle_t hEvent) { return hEvent->retain(); }
 
@@ -288,4 +317,24 @@ ur_result_t urEventGetProfilingInfo(
 
   return UR_RESULT_SUCCESS;
 }
+
+ur_result_t urEventGetNativeHandle(ur_event_handle_t hEvent,
+                                   ur_native_handle_t *phNativeEvent) try {
+  *phNativeEvent = reinterpret_cast<ur_native_handle_t>(hEvent->getZeEvent());
+  return UR_RESULT_SUCCESS;
+} catch (...) {
+  return exceptionToResult(std::current_exception());
+}
+
+ur_result_t
+urEventCreateWithNativeHandle(ur_native_handle_t hNativeEvent,
+                              ur_context_handle_t hContext,
+                              const ur_event_native_properties_t *pProperties,
+                              ur_event_handle_t *phEvent) try {
+  *phEvent = new ur_native_event_t(hNativeEvent, hContext, pProperties);
+  return UR_RESULT_SUCCESS;
+} catch (...) {
+  return exceptionToResult(std::current_exception());
+}
+
 } // namespace ur::level_zero

source/adapters/level_zero/v2/event.hpp

@@ -24,16 +24,19 @@ class event_pool;
 
 struct ur_event_handle_t_ : _ur_object {
 public:
-  ur_event_handle_t_(v2::raii::cache_borrowed_event eventAllocation,
-                     v2::event_pool *pool);
+  ur_event_handle_t_(ze_event_handle_t hZeEvent, ur_device_handle_t hDevice,
+                     ur_command_t commandType, v2::event_flags_t flags);
 
-  // Set the queue and command that this event is associated with
-  void resetQueueAndCommand(ur_queue_handle_t hQueue, ur_command_t commandType);
+  // releases event immediately
+  virtual ur_result_t forceRelease() = 0;
+  virtual ~ur_event_handle_t_() = default;
 
   void reset();
   ze_event_handle_t getZeEvent() const;
 
   ur_result_t retain();
+
+  // releases event immediately, or adds to a list for deffered deletion
   ur_result_t release();
 
   // releases a signaled and no longer in-use event, that's on the
@@ -47,7 +50,7 @@ struct ur_event_handle_t_ : _ur_object {
   // Tells if this event comes from a pool that has profiling enabled.
   bool isProfilingEnabled() const;
 
-  // Device associated with this event
+  // Get the device associated with this event
   ur_device_handle_t getDevice() const;
 
   // Queue associated with this event
@@ -56,7 +59,9 @@ struct ur_event_handle_t_ : _ur_object {
   // Get the type of the command that this event is associated with
   ur_command_t getCommandType() const;
 
-  void recordStartTimestamp();
+  // record the start timestamp of the event, to be obtained by
+  // urEventGetProfilingInfo
+  void recordStartTimestamp(ur_queue_handle_t);
 
   // Get pointer to the end timestamp, and ze event handle.
   // Caller is responsible for signaling the event once the timestamp is ready.
@@ -65,16 +70,47 @@ struct ur_event_handle_t_ : _ur_object {
   uint64_t getEventStartTimestmap() const;
   uint64_t getEventEndTimestamp();
 
-private:
-  ur_queue_handle_t hQueue = nullptr;
-  ur_command_t commandType = UR_COMMAND_FORCE_UINT32;
-  v2::raii::cache_borrowed_event zeEvent;
-  v2::event_pool *pool;
+protected:
+  // non-owning handle to the L0 event
+  const ze_event_handle_t hZeEvent;
+
+  // device associated with this event
+  const ur_device_handle_t hDevice;
+
+  v2::event_flags_t flags;
 
   uint64_t adjustedEventStartTimestamp;
   uint64_t recordEventEndTimestamp;
   uint64_t adjustedEventEndTimestamp;
 
   const uint64_t zeTimerResolution;
   const uint64_t timestampMaxValue;
+
+  ur_queue_handle_t hQueue;
+  ur_command_t commandType;
+};
+
+struct ur_pooled_event_t : ur_event_handle_t_ {
+  ur_pooled_event_t(v2::raii::cache_borrowed_event eventAllocation,
+                    v2::event_pool *pool);
+
+  // Set the queue and command that this event is associated with
+  void resetQueueAndCommand(ur_queue_handle_t hQueue, ur_command_t commandType);
+
+  ur_result_t forceRelease() override;
+
+private:
+  v2::raii::cache_borrowed_event zeEvent;
+  v2::event_pool *pool;
+};
+
+struct ur_native_event_t : ur_event_handle_t_ {
+  ur_native_event_t(ur_native_handle_t hNativeEvent,
+                    ur_context_handle_t hContext,
+                    const ur_event_native_properties_t *pProperties);
+
+  ur_result_t forceRelease() override;
+
+private:
+  v2::raii::ze_event_handle_t zeEvent;
 };

source/adapters/level_zero/v2/event_pool.cpp

@@ -9,14 +9,15 @@
 //===----------------------------------------------------------------------===//
 #include "event_pool.hpp"
 #include "common/latency_tracker.hpp"
+#include "event.hpp"
 #include "ur_api.h"
 
 namespace v2 {
 
 static constexpr size_t EVENTS_BURST = 64;
 
-ur_event_handle_t_ *event_pool::allocate(ur_queue_handle_t hQueue,
-                                         ur_command_t commandType) {
+ur_pooled_event_t *event_pool::allocate(ur_queue_handle_t hQueue,
+                                        ur_command_t commandType) {
   TRACK_SCOPE_LATENCY("event_pool::allocate");
 
   std::unique_lock<std::mutex> lock(*mutex);
@@ -38,7 +39,7 @@ ur_event_handle_t_ *event_pool::allocate(ur_queue_handle_t hQueue,
   return event;
 }
 
-void event_pool::free(ur_event_handle_t_ *event) {
+void event_pool::free(ur_pooled_event_t *event) {
   TRACK_SCOPE_LATENCY("event_pool::free");
 
   std::unique_lock<std::mutex> lock(*mutex);

source/adapters/level_zero/v2/event_pool.hpp

@@ -41,20 +41,20 @@ class event_pool {
   DeviceId Id() { return provider->device()->Id.value(); };
 
   // Allocate an event from the pool. Thread safe.
-  ur_event_handle_t_ *allocate(ur_queue_handle_t hQueue,
-                               ur_command_t commandType);
+  ur_pooled_event_t *allocate(ur_queue_handle_t hQueue,
+                              ur_command_t commandType);
 
   // Free an event back to the pool. Thread safe.
-  void free(ur_event_handle_t_ *event);
+  void free(ur_pooled_event_t *event);
 
   event_provider *getProvider() const;
   event_flags_t getFlags() const;
 
 private:
   std::unique_ptr<event_provider> provider;
 
-  std::deque<ur_event_handle_t_> events;
-  std::vector<ur_event_handle_t_ *> freelist;
+  std::deque<ur_pooled_event_t> events;
+  std::vector<ur_pooled_event_t *> freelist;
 
   std::unique_ptr<std::mutex> mutex;
 };

source/adapters/level_zero/v2/queue_immediate_in_order.cpp

@@ -1049,7 +1049,7 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueTimestampRecordingExp(
   auto [pWaitEvents, numWaitEvents] =
       getWaitListView(phEventWaitList, numEventsInWaitList);
 
-  signalEvent->recordStartTimestamp();
+  signalEvent->recordStartTimestamp(this);
 
   auto [timestampPtr, zeSignalEvent] =
       signalEvent->getEventEndTimestampAndHandle();

test/adapters/level_zero/v2/event_pool_test.cpp

@@ -26,6 +26,14 @@ using namespace v2;
 
 static constexpr size_t MAX_DEVICES = 10;
 
+// mock necessary functions from context, we can't pull in entire context implementation due to
+// a lot of other dependencies
+std::vector<ur_device_handle_t> mockVec{nullptr};
+const std::vector<ur_device_handle_t> &
+ur_context_handle_t_::getDevices() const {
+    return mockVec;
+}
+
 enum ProviderType {
     TEST_PROVIDER_NORMAL,
     TEST_PROVIDER_COUNTER,