Merge branch 'main' into maxime/imm-cmd-list-support

Author: Maxime France-Pillois

cmake/match.py

@@ -5,77 +5,132 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-# check if all lines match in a file
-# lines in a match file can contain regex inside of double curly braces {{}}
+# Check if all input file content matches match file content.
+# Lines in a match file can contain regex inside of double curly braces {{}}.
+# Regex patterns are limited to single line.
+#
+# List of available special tags:
+# {{OPT}}    - makes content in the same line as the tag optional
+# {{IGNORE}} - ignores all content until the next successfully matched line or the end of the input
+# Special tags are mutually exclusive and are expected to be located at the start of a line.
+#
 
+import os
 import sys
 import re
+from enum import Enum
 
 
 ## @brief print the whole content of input and match files
-def print_content(input_lines, match_lines):
+def print_content(input_lines, match_lines, ignored_lines):
     print("--- Input Lines " + "-" * 64)
     print("".join(input_lines).strip())
     print("--- Match Lines " + "-" * 64)
     print("".join(match_lines).strip())
+    print("--- Ignored Lines " + "-" * 62)
+    print("".join(ignored_lines).strip())
     print("-" * 80)
 
 
-if len(sys.argv) != 3:
-    print("Usage: python match.py <input_file> <match_file>")
-    sys.exit(1)
-
-input_file = sys.argv[1]
-match_file = sys.argv[2]
-
-with open(input_file, 'r') as input, open(match_file, 'r') as match:
-    input_lines = input.readlines()
-    match_lines = match.readlines()
-
-if len(match_lines) < len(input_lines):
-    print(f"Match length < input length (input: {len(input_lines)}, match: {len(match_lines)})")
-    print_content(input_lines, match_lines)
-    sys.exit(1)
-
-input_idx = 0
-opt = "{{OPT}}"
-for i, match_line in enumerate(match_lines):
-    if match_line.startswith(opt):
-        optional_line = True
-        match_line = match_line[len(opt):]
-    else:
-        optional_line = False
-
-    # split into parts at {{ }}
-    match_parts = re.split(r'\{{(.*?)\}}', match_line.strip())
-    pattern = ""
-    for j, part in enumerate(match_parts):
-        if j % 2 == 0:
-            pattern += re.escape(part)
-        else:
-            pattern += part
+## @brief print the incorrect match line
+def print_incorrect_match(match_line, present, expected):
+    print("Line " + str(match_line) + " does not match")
+    print("is:       " + present)
+    print("expected: " + expected)
 
-    # empty input file or end of input file, from now on match file must be optional
-    if not input_lines:
-        if optional_line is True:
-            continue
-        else:
-            print("End of input file or empty file.")
-            print("expected: " + match_line.strip())
+
+## @brief pattern matching script status values
+class Status(Enum):
+    INPUT_END = 1
+    MATCH_END = 2
+    INPUT_AND_MATCH_END = 3
+    PROCESSING = 4
+
+
+## @brief check matching script status
+def check_status(input_lines, match_lines):
+    if not input_lines and not match_lines:
+        return Status.INPUT_AND_MATCH_END
+    elif not input_lines:
+        return Status.INPUT_END
+    elif not match_lines:
+        return Status.MATCH_END
+    return Status.PROCESSING
+
+
+## @brief pattern matching tags.
+## Tags are expected to be at the start of the line.
+class Tag(Enum):
+    OPT = "{{OPT}}"         # makes the line optional
+    IGNORE = "{{IGNORE}}"   # ignores all input until next match or end of input file
+
+
+## @brief main function for the match file processing script
+def main():
+    if len(sys.argv) != 3:
+        print("Usage: python match.py <input_file> <match_file>")
+        sys.exit(1)
+
+    input_file = sys.argv[1]
+    match_file = sys.argv[2]
+
+    with open(input_file, 'r') as input, open(match_file, 'r') as match:
+        input_lines = input.readlines()
+        match_lines = match.readlines()
+
+    ignored_lines = []
+
+    input_idx = 0
+    match_idx = 0
+    tags_in_effect = []
+    while True:
+        # check file status
+        status = check_status(input_lines[input_idx:], match_lines[match_idx:])
+        if (status == Status.INPUT_AND_MATCH_END) or (status == Status.MATCH_END and Tag.IGNORE in tags_in_effect):
+            # all lines matched or the last line in match file is an ignore tag
+            sys.exit(0)
+        elif status == Status.MATCH_END:
+            print_incorrect_match(match_idx + 1, input_lines[input_idx].strip(), "");
+            print_content(input_lines, match_lines, ignored_lines)
             sys.exit(1)
 
-    input_line = input_lines[input_idx].strip()
-    if not re.fullmatch(pattern, input_line):
-        if optional_line is True:
-            continue
+        input_line = input_lines[input_idx].strip() if input_idx < len(input_lines) else ""
+        match_line = match_lines[match_idx]
+
+        # check for tags
+        if match_line.startswith(Tag.OPT.value):
+            tags_in_effect.append(Tag.OPT)
+            match_line = match_line[len(Tag.OPT.value):]
+        elif match_line.startswith(Tag.IGNORE.value):
+            tags_in_effect.append(Tag.IGNORE)
+            match_idx += 1
+            continue # line with ignore tag should be skipped
+
+        # split into parts at {{ }}
+        match_parts = re.split(r'\{{(.*?)\}}', match_line.strip())
+        pattern = ""
+        for j, part in enumerate(match_parts):
+            if j % 2 == 0:
+                pattern += re.escape(part)
+            else:
+                pattern += part
+
+        # match or process tags
+        if re.fullmatch(pattern, input_line):
+            input_idx += 1
+            match_idx += 1
+            tags_in_effect = []
+        elif Tag.OPT in tags_in_effect:
+            match_idx += 1
+            tags_in_effect.remove(Tag.OPT)
+        elif Tag.IGNORE in tags_in_effect:
+            ignored_lines.append(input_line + os.linesep)
+            input_idx += 1
         else:
-            print("Line " + str(i+1) + " does not match")
-            print("is:       " + input_line)
-            print("expected: " + match_line.strip())
-            print_content(input_lines, match_lines)
+            print_incorrect_match(match_idx + 1, input_line, match_line.strip())
+            print_content(input_lines, match_lines, ignored_lines)
             sys.exit(1)
-    else:
-        if (input_idx == len(input_lines) - 1):
-            input_lines = []
-        else:
-            input_idx += 1
+
+
+if __name__ == "__main__":
+    main()

source/adapters/level_zero/command_buffer.cpp

@@ -45,13 +45,13 @@
   │  Prefix  │ Commands added to UR command-buffer by UR user │ Suffix  │
   └──────────┴────────────────────────────────────────────────┴─────────┘
 
-            ┌───────────────────┬──────────────────────────────┐
-  Prefix    │Reset signal event │ Barrier waiting on wait event│
-            └───────────────────┴──────────────────────────────┘
+            ┌───────────────────┬──────────────┐──────────────────────────────┐
+  Prefix    │Reset signal event │ Reset events │ Barrier waiting on wait event│
+            └───────────────────┴──────────────┘──────────────────────────────┘
 
             ┌─────────────────────────────────────────────┐──────────────┐
-  Suffix    │Barrier waiting on sync-point event,         │ Reset events │
-            │signalling the UR command-buffer signal event│              │
+  Suffix    │Barrier waiting on sync-point event,         │  Query CMD   │
+            │signalling the UR command-buffer signal event│  Timestamps  │
             └─────────────────────────────────────────────┘──────────────┘
 
   For a call to `urCommandBufferEnqueueExp` with an event_list `EL`,
@@ -431,6 +431,10 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
 
   ZeStruct<ze_command_list_desc_t> ZeCommandListDesc;
   ZeCommandListDesc.commandQueueGroupOrdinal = QueueGroupOrdinal;
+  // Dependencies between commands are explicitly enforced by sync points when
+  // enqueuing. Consequently, relax the command ordering in the command list
+  // can enable the backend to further optimize the workload
+  ZeCommandListDesc.flags = ZE_COMMAND_LIST_FLAG_RELAXED_ORDERING;
 
   ze_command_list_handle_t ZeCommandList;
   // TODO We could optimize this by pooling both Level Zero command-lists and UR
@@ -491,18 +495,6 @@ urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t CommandBuffer) {
              (CommandBuffer->ZeCommandList, CommandBuffer->SignalEvent->ZeEvent,
               NumEvents, WaitEventList.data()));
 
-  // Reset the wait-event for the UR command-buffer that is signalled when its
-  // submission dependencies have been satisfied.
-  ZE2UR_CALL(zeCommandListAppendEventReset,
-             (CommandBuffer->ZeCommandList, CommandBuffer->WaitEvent->ZeEvent));
-
-  // Reset the L0 events we use for command-buffer internal sync-points to the
-  // non-signalled state
-  for (auto Event : WaitEventList) {
-    ZE2UR_CALL(zeCommandListAppendEventReset,
-               (CommandBuffer->ZeCommandList, Event));
-  }
-
   // Close the command list and have it ready for dispatch.
   ZE2UR_CALL(zeCommandListClose, (CommandBuffer->ZeCommandList));
   return UR_RESULT_SUCCESS;
@@ -876,6 +868,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferEnqueueExp(
 
   // Create command-list to execute before `CommandListPtr` and will signal
   // when `EventWaitList` dependencies are complete.
+  ur_command_list_ptr_t WaitCommandList{};
+  UR_CALL(Queue->Context->getAvailableCommandList(Queue, WaitCommandList, false,
+                                                  false));
+
+  // Create a list of events of all the events that compose the command buffer
+  // workload.
+  // This loop also resets the L0 events we use for command-buffer internal
+  // sync-points to the non-signalled state.
+  // This is required for multiple submissions.
+  const size_t NumEvents = CommandBuffer->SyncPoints.size();
+  std::vector<ze_event_handle_t> WaitEventList{NumEvents};
+  for (size_t i = 0; i < NumEvents; i++) {
+    auto ZeEvent = CommandBuffer->SyncPoints[i]->ZeEvent;
+    WaitEventList[i] = ZeEvent;
+    ZE2UR_CALL(zeCommandListAppendEventReset,
+               (WaitCommandList->first, ZeEvent));
+  }
+
   bool MustSignalWaitEvent = true;
   if (NumEventsInWaitList) {
     _ur_ze_event_list_t TmpWaitList;
@@ -890,10 +900,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferEnqueueExp(
       CommandBuffer->WaitEvent->WaitList.insert(TmpWaitList);
 
     if (!CommandBuffer->WaitEvent->WaitList.isEmpty()) {
-      ur_command_list_ptr_t WaitCommandList{};
-      UR_CALL(Queue->Context->getAvailableCommandList(Queue, WaitCommandList,
-                                                      false, false))
-
       ZE2UR_CALL(zeCommandListAppendBarrier,
                  (WaitCommandList->first, CommandBuffer->WaitEvent->ZeEvent,
                   CommandBuffer->WaitEvent->WaitList.Length,
@@ -916,22 +922,50 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferEnqueueExp(
 
   // Execution event for this enqueue of the UR command-buffer
   ur_event_handle_t RetEvent{};
-  if (Event) {
-    // Create a command-list to signal RetEvent on completion
-    ur_command_list_ptr_t SignalCommandList{};
-    UR_CALL(Queue->Context->getAvailableCommandList(Queue, SignalCommandList,
-                                                    false, false));
 
+  // Create a command-list to signal RetEvent on completion
+  ur_command_list_ptr_t SignalCommandList{};
+  UR_CALL(Queue->Context->getAvailableCommandList(Queue, SignalCommandList,
+                                                  false, false));
+  // Reset the wait-event for the UR command-buffer that is signalled when its
+  // submission dependencies have been satisfied.
+  ZE2UR_CALL(zeCommandListAppendEventReset,
+             (SignalCommandList->first, CommandBuffer->WaitEvent->ZeEvent));
+
+  if (Event) {
     UR_CALL(createEventAndAssociateQueue(Queue, &RetEvent,
                                          UR_COMMAND_COMMAND_BUFFER_ENQUEUE_EXP,
                                          SignalCommandList, false, true));
 
-    ZE2UR_CALL(zeCommandListAppendBarrier,
-               (SignalCommandList->first, RetEvent->ZeEvent, 1,
-                &(CommandBuffer->SignalEvent->ZeEvent)));
-    Queue->executeCommandList(SignalCommandList, false, false);
+    if ((Queue->Properties & UR_QUEUE_FLAG_PROFILING_ENABLE)) {
+      // Multiple submissions of a command buffer implies that we need to save
+      // the event timestamps before resubmiting the command buffer. We
+      // therefore copy the these timestamps in a dedicated USM memory section
+      // before completing the command buffer execution, and then attach this
+      // memory to the event returned to users to allow to allow the profiling
+      // engine to recover these timestamps.
+      command_buffer_profiling_t *Profiling = new command_buffer_profiling_t();
+
+      Profiling->NumEvents = WaitEventList.size();
+      Profiling->Timestamps =
+          new ze_kernel_timestamp_result_t[Profiling->NumEvents];
+
+      ZE2UR_CALL(zeCommandListAppendQueryKernelTimestamps,
+                 (SignalCommandList->first, WaitEventList.size(),
+                  WaitEventList.data(), (void *)Profiling->Timestamps, 0,
+                  RetEvent->ZeEvent, 1,
+                  &(CommandBuffer->SignalEvent->ZeEvent)));
+
+      RetEvent->CommandData = static_cast<void *>(Profiling);
+    } else {
+      ZE2UR_CALL(zeCommandListAppendBarrier,
+                 (SignalCommandList->first, RetEvent->ZeEvent, 1,
+                  &(CommandBuffer->SignalEvent->ZeEvent)));
+    }
   }
 
+  Queue->executeCommandList(SignalCommandList, false, false);
+
   if (Event) {
     *Event = RetEvent;
   }

source/adapters/level_zero/command_buffer.hpp

@@ -19,6 +19,11 @@
 #include "context.hpp"
 #include "queue.hpp"
 
+struct command_buffer_profiling_t {
+  ur_exp_command_buffer_sync_point_t NumEvents;
+  ze_kernel_timestamp_result_t *Timestamps;
+};
+
 struct ur_exp_command_buffer_handle_t_ : public _ur_object {
   ur_exp_command_buffer_handle_t_(ur_context_handle_t Context,
                                   ur_device_handle_t Device,