Moved hip EvBase to be owned by device

Author: Konrad Kusiak

source/adapters/hip/context.cpp

@@ -47,18 +47,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urContextCreate(
     // Create a scoped context.
     ContextPtr = std::unique_ptr<ur_context_handle_t_>(
         new ur_context_handle_t_{phDevices, DeviceCount});
-
-    static std::once_flag InitFlag;
-    std::call_once(
-        InitFlag,
-        [](ur_result_t &) {
-          // Use default stream to record base event counter
-          UR_CHECK_ERROR(hipEventCreateWithFlags(&ur_platform_handle_t_::EvBase,
-                                                 hipEventDefault));
-          UR_CHECK_ERROR(hipEventRecord(ur_platform_handle_t_::EvBase, 0));
-        },
-        RetErr);
-
     *phContext = ContextPtr.release();
   } catch (ur_result_t Err) {
     RetErr = Err;

source/adapters/hip/device.cpp

@@ -20,6 +20,18 @@ int getAttribute(ur_device_handle_t Device, hipDeviceAttribute_t Attribute) {
   return Value;
 }
 
+uint64_t ur_device_handle_t_::getElapsedTime(hipEvent_t ev) const {
+  float Milliseconds = 0.0f;
+
+  // hipEventSynchronize waits till the event is ready for call to
+  // hipEventElapsedTime.
+  UR_CHECK_ERROR(hipEventSynchronize(EvBase));
+  UR_CHECK_ERROR(hipEventSynchronize(ev));
+  UR_CHECK_ERROR(hipEventElapsedTime(&Milliseconds, EvBase, ev));
+
+  return static_cast<uint64_t>(Milliseconds * 1.0e6);
+}
+
 UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
                                                     ur_device_info_t propName,
                                                     size_t propSize,
@@ -995,11 +1007,7 @@ ur_result_t UR_APICALL urDeviceGetGlobalTimestamps(ur_device_handle_t hDevice,
   if (pDeviceTimestamp) {
     UR_CHECK_ERROR(hipEventCreateWithFlags(&Event, hipEventDefault));
     UR_CHECK_ERROR(hipEventRecord(Event));
-    UR_CHECK_ERROR(hipEventSynchronize(Event));
-    float ElapsedTime = 0.0f;
-    UR_CHECK_ERROR(hipEventElapsedTime(&ElapsedTime,
-                                       ur_platform_handle_t_::EvBase, Event));
-    *pDeviceTimestamp = (uint64_t)(ElapsedTime * (double)1e6);
+    *pDeviceTimestamp = hDevice->getElapsedTime(Event);
   }
 
   if (pHostTimestamp) {

source/adapters/hip/device.hpp

@@ -25,7 +25,9 @@ struct ur_device_handle_t_ {
   std::atomic_uint32_t RefCount;
   ur_platform_handle_t Platform;
   hipCtx_t HIPContext;
+  hipEvent_t EvBase; // HIP event used as base counter
   uint32_t DeviceIndex;
+
   int MaxWorkGroupSize{0};
   int MaxBlockDimX{0};
   int MaxBlockDimY{0};
@@ -36,9 +38,10 @@ struct ur_device_handle_t_ {
 
 public:
   ur_device_handle_t_(native_type HipDevice, hipCtx_t Context,
-                      ur_platform_handle_t Platform, uint32_t DeviceIndex)
+                      hipEvent_t EvBase, ur_platform_handle_t Platform,
+                      uint32_t DeviceIndex)
       : HIPDevice(HipDevice), RefCount{1}, Platform(Platform),
-        HIPContext(Context), DeviceIndex(DeviceIndex) {
+        HIPContext(Context), EvBase(EvBase), DeviceIndex(DeviceIndex) {
 
     UR_CHECK_ERROR(hipDeviceGetAttribute(
         &MaxWorkGroupSize, hipDeviceAttributeMaxThreadsPerBlock, HIPDevice));
@@ -68,6 +71,8 @@ struct ur_device_handle_t_ {
 
   ur_platform_handle_t getPlatform() const noexcept { return Platform; };
 
+  uint64_t getElapsedTime(hipEvent_t) const;
+
   hipCtx_t getNativeContext() const noexcept { return HIPContext; };
 
   // Returns the index of the device relative to the other devices in the same

source/adapters/hip/event.cpp

@@ -90,44 +90,18 @@ bool ur_event_handle_t_::isCompleted() const {
 }
 
 uint64_t ur_event_handle_t_::getQueuedTime() const {
-  float MilliSeconds = 0.0f;
   assert(isStarted());
-
-  // hipEventSynchronize waits till the event is ready for call to
-  // hipEventElapsedTime.
-  UR_CHECK_ERROR(hipEventSynchronize(EvStart));
-  UR_CHECK_ERROR(hipEventSynchronize(EvEnd));
-
-  UR_CHECK_ERROR(hipEventElapsedTime(&MilliSeconds, EvStart, EvEnd));
-  return static_cast<uint64_t>(MilliSeconds * 1.0e6);
+  return Queue->getDevice()->getElapsedTime(EvQueued);
 }
 
 uint64_t ur_event_handle_t_::getStartTime() const {
-  float MiliSeconds = 0.0f;
   assert(isStarted());
-
-  // hipEventSynchronize waits till the event is ready for call to
-  // hipEventElapsedTime.
-  UR_CHECK_ERROR(hipEventSynchronize(ur_platform_handle_t_::EvBase));
-  UR_CHECK_ERROR(hipEventSynchronize(EvStart));
-
-  UR_CHECK_ERROR(hipEventElapsedTime(&MiliSeconds,
-                                     ur_platform_handle_t_::EvBase, EvStart));
-  return static_cast<uint64_t>(MiliSeconds * 1.0e6);
+  return Queue->getDevice()->getElapsedTime(EvStart);
 }
 
 uint64_t ur_event_handle_t_::getEndTime() const {
-  float MiliSeconds = 0.0f;
   assert(isStarted() && isRecorded());
-
-  // hipEventSynchronize waits till the event is ready for call to
-  // hipEventElapsedTime.
-  UR_CHECK_ERROR(hipEventSynchronize(ur_platform_handle_t_::EvBase));
-  UR_CHECK_ERROR(hipEventSynchronize(EvEnd));
-
-  UR_CHECK_ERROR(
-      hipEventElapsedTime(&MiliSeconds, ur_platform_handle_t_::EvBase, EvEnd));
-  return static_cast<uint64_t>(MiliSeconds * 1.0e6);
+  return Queue->getDevice()->getElapsedTime(EvEnd);
 }
 
 ur_result_t ur_event_handle_t_::record() {

source/adapters/hip/platform.cpp

@@ -11,8 +11,6 @@
 #include "platform.hpp"
 #include "context.hpp"
 
-hipEvent_t ur_platform_handle_t_::EvBase{nullptr};
-
 UR_APIEXPORT ur_result_t UR_APICALL
 urPlatformGetInfo(ur_platform_handle_t, ur_platform_info_t propName,
                   size_t propSize, void *pPropValue, size_t *pSizeRet) {
@@ -81,18 +79,15 @@ urPlatformGet(ur_adapter_handle_t *, uint32_t, uint32_t NumEntries,
               UR_CHECK_ERROR(hipDeviceGet(&Device, i));
               hipCtx_t Context;
               UR_CHECK_ERROR(hipDevicePrimaryCtxRetain(&Context, Device));
-              Platform.Devices.emplace_back(
-                  new ur_device_handle_t_{Device, Context, &Platform, i});
-            }
-
-            // Setup EvBase
-            {
-              ScopedContext Active(Platform.Devices.front().get());
               hipEvent_t EvBase;
               UR_CHECK_ERROR(hipEventCreate(&EvBase));
+
+              // Use the default stream to record base event counter
               UR_CHECK_ERROR(hipEventRecord(EvBase, 0));
+              Platform.Devices.emplace_back(new ur_device_handle_t_{
+                  Device, Context, EvBase, &Platform, i});
 
-              ur_platform_handle_t_::EvBase = EvBase;
+              ScopedContext Active(Platform.Devices.front().get());
             }
           } catch (const std::bad_alloc &) {
             // Signal out-of-memory situation

source/adapters/hip/platform.hpp

@@ -20,6 +20,5 @@
 ///  when devices are used.
 ///
 struct ur_platform_handle_t_ {
-  static hipEvent_t EvBase; // HIP event used as base counter
   std::vector<std::unique_ptr<ur_device_handle_t_>> Devices;
 };