Merge pull request #2410 from steffenlarsen/steffen/fix_profiling_tag_eviction_and_caching

[L0] Fix cached and evicted timestamp recordings

Author: kbenzie

source/adapters/level_zero/event.cpp

@@ -581,8 +581,7 @@ ur_result_t urEventGetProfilingInfo(
 
       // End time needs to be adjusted for resolution and valid bits.
       uint64_t ContextEndTime =
-          (EndTimeRecording.RecordEventEndTimestamp & TimestampMaxValue) *
-          ZeTimerResolution;
+          (EndTimeRecording & TimestampMaxValue) * ZeTimerResolution;
 
       // If the result is 0, we have not yet gotten results back and so we just
       // return it.
@@ -755,20 +754,20 @@ ur_result_t urEnqueueTimestampRecordingExp(
   ze_event_handle_t ZeEvent = (*OutEvent)->ZeEvent;
   (*OutEvent)->WaitList = TmpWaitList;
 
+  // Reset the end timestamp, in case it has been previously used.
+  (*OutEvent)->RecordEventEndTimestamp = 0;
+
   uint64_t DeviceStartTimestamp = 0;
   UR_CALL(ur::level_zero::urDeviceGetGlobalTimestamps(
       Device, &DeviceStartTimestamp, nullptr));
   (*OutEvent)->RecordEventStartTimestamp = DeviceStartTimestamp;
 
   // Create a new entry in the queue's recordings.
-  Queue->EndTimeRecordings[*OutEvent] =
-      ur_queue_handle_t_::end_time_recording{};
+  Queue->EndTimeRecordings[*OutEvent] = 0;
 
   ZE2UR_CALL(zeCommandListAppendWriteGlobalTimestamp,
-             (CommandList->first,
-              &Queue->EndTimeRecordings[*OutEvent].RecordEventEndTimestamp,
-              ZeEvent, (*OutEvent)->WaitList.Length,
-              (*OutEvent)->WaitList.ZeEventList));
+             (CommandList->first, &Queue->EndTimeRecordings[*OutEvent], ZeEvent,
+              (*OutEvent)->WaitList.Length, (*OutEvent)->WaitList.ZeEventList));
 
   UR_CALL(
       Queue->executeCommandList(CommandList, Blocking, false /* OkToBatch */));
@@ -1096,10 +1095,11 @@ ur_result_t urEventReleaseInternal(ur_event_handle_t Event) {
     auto Entry = Queue->EndTimeRecordings.find(Event);
     if (Entry != Queue->EndTimeRecordings.end()) {
       auto &EndTimeRecording = Entry->second;
-      if (EndTimeRecording.RecordEventEndTimestamp == 0) {
+      if (EndTimeRecording == 0) {
         // If the end time recording has not finished, we tell the queue that
         // the event is no longer alive to avoid invalid write-backs.
-        EndTimeRecording.EventHasDied = true;
+        Queue->EvictedEndTimeRecordings.insert(
+            Queue->EndTimeRecordings.extract(Entry));
       } else {
         // Otherwise we evict the entry.
         Queue->EndTimeRecordings.erase(Entry);

source/adapters/level_zero/queue.cpp

@@ -1566,24 +1566,23 @@ void ur_queue_handle_t_::clearEndTimeRecordings() {
   for (auto Entry : EndTimeRecordings) {
     auto &Event = Entry.first;
     auto &EndTimeRecording = Entry.second;
-    if (!Entry.second.EventHasDied) {
-      // Write the result back to the event if it is not dead.
-      uint64_t ContextEndTime =
-          (EndTimeRecording.RecordEventEndTimestamp & TimestampMaxValue) *
-          ZeTimerResolution;
-
-      // Handle a possible wrap-around (the underlying HW counter is < 64-bit).
-      // Note, it will not report correct time if there were multiple wrap
-      // arounds, and the longer term plan is to enlarge the capacity of the
-      // HW timestamps.
-      if (ContextEndTime < Event->RecordEventStartTimestamp)
-        ContextEndTime += TimestampMaxValue * ZeTimerResolution;
-
-      // Store it in the event.
-      Event->RecordEventEndTimestamp = ContextEndTime;
-    }
+
+    // Write the result back to the event if it is not dead.
+    uint64_t ContextEndTime =
+        (EndTimeRecording & TimestampMaxValue) * ZeTimerResolution;
+
+    // Handle a possible wrap-around (the underlying HW counter is < 64-bit).
+    // Note, it will not report correct time if there were multiple wrap
+    // arounds, and the longer term plan is to enlarge the capacity of the
+    // HW timestamps.
+    if (ContextEndTime < Event->RecordEventStartTimestamp)
+      ContextEndTime += TimestampMaxValue * ZeTimerResolution;
+
+    // Store it in the event.
+    Event->RecordEventEndTimestamp = ContextEndTime;
   }
   EndTimeRecordings.clear();
+  EvictedEndTimeRecordings.clear();
 }
 
 ur_result_t urQueueReleaseInternal(ur_queue_handle_t Queue) {

source/adapters/level_zero/queue.hpp

@@ -492,15 +492,12 @@ struct ur_queue_handle_t_ : _ur_object {
   // End-times enqueued are stored on the queue rather than on the event to
   // avoid the event objects having been destroyed prior to the write to the
   // end-time member.
-  struct end_time_recording {
-    // RecordEventEndTimestamp is not adjusted for valid bits nor resolution, as
-    // it is written asynchronously.
-    uint64_t RecordEventEndTimestamp = 0;
-    // The event may die before the recording has been written back. In this
-    // case the event will mark this for deletion when the queue sees fit.
-    bool EventHasDied = false;
-  };
-  std::map<ur_event_handle_t, end_time_recording> EndTimeRecordings;
+  // RecordEventEndTimestamp is not adjusted for valid bits nor resolution, as
+  // it is written asynchronously.
+  std::map<ur_event_handle_t, uint64_t> EndTimeRecordings;
+  // The event may die before the recording has been written back. In this case
+  // we move it to a separate map to avoid conflicts.
+  std::multimap<ur_event_handle_t, uint64_t> EvictedEndTimeRecordings;
 
   // Clear the end time recording timestamps entries.
   void clearEndTimeRecordings();