Creating shim io test support for npu3 full elf flow

Signed-off-by: Hayden Laccabue <[email protected]>

Author: hlaccabu

src/shim/buffer.cpp

@@ -21,8 +21,7 @@ uint8_t
 use_to_fw_debug_type(uint8_t use)
 {
   switch (use) {
-  case XRT_BO_USE_DEBUG:
-    return AMDXDNA_FW_BUF_DEBUG;
+  case XRT_BO_USE_DEBUG: // USE_DEBUG and USE_UC_DEBUG should return same type, 2 different cases for different devices
   case XRT_BO_USE_UC_DEBUG:
     return AMDXDNA_FW_BUF_DEBUG;
   case XRT_BO_USE_DTRACE:

test/shim_test/dev_info.h

@@ -13,6 +13,7 @@ enum kernel_type {
   KERNEL_TYPE_DPU_SEQ = 0,
   KERNEL_TYPE_TXN,
   KERNEL_TYPE_TXN_PREEMPT,
+  KERNEL_TYPE_TXN_FULL_ELF,
   KERNEL_TYPE_TXN_FULL_ELF_PREEMPT,
 };
 

test/shim_test/hwctx.h

@@ -34,7 +34,9 @@ class hw_ctx {
   {
     xrt::xclbin xclbin;
     xrt::elf elf;
-    auto is_full_elf = (get_kernel_type(dev, xclbin_name) == KERNEL_TYPE_TXN_FULL_ELF_PREEMPT);
+    auto kernel_type = get_kernel_type(dev, xclbin_name);
+    auto is_full_elf = (kernel_type == KERNEL_TYPE_TXN_FULL_ELF_PREEMPT ||
+		        kernel_type == KERNEL_TYPE_TXN_FULL_ELF);
     auto path = get_xclbin_path(dev, xclbin_name);
 
     try {

test/shim_test/io.cpp

@@ -384,6 +384,47 @@ elf_io_test_bo_set(device* dev, const std::string& xclbin_name) :
   }
 }
 
+elf_full_io_test_bo_set::
+elf_full_io_test_bo_set(device* dev, const std::string& xclbin_name)
+  : io_test_bo_set_base(dev, xclbin_name)
+{
+  auto elf_path = get_xclbin_path(dev, xclbin_name.c_str());
+  m_elf = xrt::elf(elf_path);
+  auto mod = xrt::module{m_elf};
+  auto kernel_name = get_kernel_name(dev, xclbin_name.c_str());
+
+  try {
+    m_kernel_index = module_int::get_ctrlcode_id(mod, kernel_name);
+  } catch (const std::exception& e) {
+    m_kernel_index = module_int::no_ctrl_code_id;
+  }
+
+  for (int i = 0; i < IO_TEST_BO_MAX_TYPES; i++) {
+    auto& ibo = m_bo_array[i];
+    auto type = static_cast<io_test_bo_type>(i);
+
+    switch(type) {
+    case IO_TEST_BO_CMD:
+      alloc_cmd_bo(ibo, m_dev);
+      break;
+    case IO_TEST_BO_INSTRUCTION:
+      create_ctrl_bo_from_elf(ibo, patcher::buf_type::ctrltext);
+      break;
+    case IO_TEST_BO_INPUT:
+      create_data_bo_from_file(ibo, "ifm.bin", m_FLAG_OPT);
+      break;
+    case IO_TEST_BO_PARAMETERS:
+      create_data_bo_from_file(ibo, "wts.bin", m_FLAG_OPT);
+      break;
+    case IO_TEST_BO_OUTPUT:
+      create_data_bo_from_file(ibo, "ofm.bin", m_FLAG_NO_FILL|m_FLAG_OPT);
+      break;
+    default:
+      break;
+    }
+  }
+}
+
 elf_preempt_io_test_bo_set::
 elf_preempt_io_test_bo_set(device* dev, const std::string& xclbin_name)
   : io_test_bo_set_base(dev, xclbin_name)
@@ -605,6 +646,30 @@ init_cmd(hw_ctx& hwctx, bool dump)
     patcher::buf_type::ctrltext, m_elf, module_int::no_ctrl_code_id);
 }
 
+void
+elf_full_io_test_bo_set::
+init_cmd(hw_ctx& hwctx, bool dump)
+{
+  exec_buf ebuf(*m_bo_array[IO_TEST_BO_CMD].tbo.get(), ERT_START_DPU);
+
+  xrt_core::cuidx_type cu_idx{0};
+  ebuf.set_cu_idx(cu_idx);
+
+  if (m_bo_array[IO_TEST_BO_INPUT].tbo.get()) {
+    ebuf.add_arg_64(3);
+    ebuf.add_arg_bo(*m_bo_array[IO_TEST_BO_INPUT].tbo.get(), "0");
+    ebuf.add_arg_bo(*m_bo_array[IO_TEST_BO_PARAMETERS].tbo.get(), "1");
+    ebuf.add_arg_bo(*m_bo_array[IO_TEST_BO_OUTPUT].tbo.get(), "2");
+  }
+
+  if (dump)
+    ebuf.dump();
+
+  ebuf.add_ctrl_bo(*m_bo_array[IO_TEST_BO_INSTRUCTION].tbo.get());
+  ebuf.patch_ctrl_code(*m_bo_array[IO_TEST_BO_INSTRUCTION].tbo.get(),
+    patcher::buf_type::ctrltext, m_elf, m_kernel_index);
+}
+
 void
 elf_preempt_io_test_bo_set::
 init_cmd(hw_ctx& hwctx, bool dump)
@@ -883,6 +948,32 @@ get_preemption_checkpoints()
   return 0;
 }
 
+void
+io_test_bo_set_base::
+cache_cmd_header()
+{
+  auto cbo = m_bo_array[IO_TEST_BO_CMD].tbo.get();
+  if (!cbo)
+    return;
+  auto pkt = reinterpret_cast<ert_packet *>(cbo->map());
+  if (!m_cached_cmd_header)
+    m_cached_cmd_header = pkt->header;
+}
+
+void
+io_test_bo_set_base::
+restore_cmd_header()
+{
+  auto cbo = m_bo_array[IO_TEST_BO_CMD].tbo.get();
+  if (!cbo || !m_cached_cmd_header)
+    return;
+
+  auto pkt = reinterpret_cast<ert_packet *>(cbo->map());
+  pkt->header = m_cached_cmd_header;
+  pkt->state = ERT_CMD_STATE_NEW;
+  std::atomic_thread_fence(std::memory_order_seq_cst);
+}
+
 unsigned long
 elf_preempt_io_test_bo_set::
 get_preemption_checkpoints()

test/shim_test/io.h

@@ -70,6 +70,12 @@ class io_test_bo_set_base
   virtual void
   verify_result();
 
+  void
+  cache_cmd_header();
+
+  void
+  restore_cmd_header();
+
   static const char *
   bo_type2name(int type);
 
@@ -90,6 +96,7 @@ class io_test_bo_set_base
   const int m_FLAG_OPT =      1 << 1;
   const int m_FLAG_NO_FILL =  1 << 2;
   const int m_FLAG_DEV_BUF =  1 << 3;
+  uint32_t m_cached_cmd_header = 0;
 
   void
   create_data_bo_from_file(io_test_bo& ibo, const std::string filename, int flags);
@@ -124,6 +131,15 @@ class elf_io_test_bo_set : public io_test_bo_set_base
   init_cmd(hw_ctx& hwctx, bool dump) override;
 };
 
+class elf_full_io_test_bo_set : public io_test_bo_set_base
+{
+public:
+  elf_full_io_test_bo_set(device *dev, const std::string& xclbin_name);
+
+  void
+  init_cmd(hw_ctx& hwctx, bool dump) override;
+};
+
 class elf_preempt_io_test_bo_set : public io_test_bo_set_base
 {
 public:

test/shim_test/io_test.cpp

@@ -43,6 +43,9 @@ alloc_and_init_bo_set(device* dev, const char *xclbin)
     base = std::make_unique<elf_io_test_bo_set>(dev, std::string(xclbin));
     break;
   case KERNEL_TYPE_TXN_PREEMPT:
+  case KERNEL_TYPE_TXN_FULL_ELF:
+    base = std::make_unique<elf_full_io_test_bo_set>(dev, xclbin ? std::string(xclbin) : get_xclbin_name(dev));
+    break;
   case KERNEL_TYPE_TXN_FULL_ELF_PREEMPT:
     base = std::make_unique<elf_preempt_io_test_bo_set>(dev, std::string(xclbin));
     break;
@@ -113,16 +116,25 @@ void
 io_test_cmd_submit_and_wait_latency(
   hwqueue_handle *hwq,
   int total_cmd_submission,
-  std::vector< std::pair<std::shared_ptr<bo>, ert_start_kernel_cmd *> >& cmdlist_bos
+  std::vector< std::pair<std::shared_ptr<bo>, ert_start_kernel_cmd *> >& cmdlist_bos,
+  std::vector< std::unique_ptr<io_test_bo_set_base> >* bo_set_ptr = nullptr
   )
 {
   int completed = 0;
-  int wait_idx = 0;
 
   while (completed < total_cmd_submission) {
-    for (auto& cmd : cmdlist_bos) {
+    for (size_t i = 0; i < cmdlist_bos.size(); i++) {
+      auto& cmd = cmdlist_bos[i];
+      // For UMQ, restore header before resubmission
+      if (completed > 0 && bo_set_ptr && i < bo_set_ptr->size()) {
+        (*bo_set_ptr)[i]->restore_cmd_header();
+      }
+      
       hwq->submit_command(std::get<0>(cmd).get()->get());
       io_test_cmd_wait(hwq, std::get<0>(cmd));
+      
+      if (bo_set_ptr && i < bo_set_ptr->size())
+        std::atomic_thread_fence(std::memory_order_acquire);
       auto state = std::get<1>(cmd)->state;
       if (state != ERT_CMD_STATE_COMPLETED) {
         std::string errmsg = "Command ";
@@ -135,7 +147,8 @@ io_test_cmd_submit_and_wait_latency(
       completed++;
       if (completed >= total_cmd_submission)
         break;
-      std::get<1>(cmd)->state = ERT_CMD_STATE_NEW;
+      if (!bo_set_ptr)
+        std::get<1>(cmd)->state = ERT_CMD_STATE_NEW;
     }
   }
 }
@@ -144,15 +157,17 @@ void
 io_test_cmd_submit_and_wait_thruput(
   hwqueue_handle *hwq,
   int total_cmd_submission,
-  std::vector< std::pair<std::shared_ptr<bo>, ert_start_kernel_cmd *> >& cmdlist_bos
+  std::vector< std::pair<std::shared_ptr<bo>, ert_start_kernel_cmd *> >& cmdlist_bos,
+  std::vector< std::unique_ptr<io_test_bo_set_base> >* bo_set_ptr = nullptr
   )
 {
   int issued = 0;
   int completed = 0;
   int wait_idx = 0;
 
   for (auto& cmd : cmdlist_bos) {
-    std::get<1>(cmd)->state = ERT_CMD_STATE_NEW;
+    if (!bo_set_ptr)
+      std::get<1>(cmd)->state = ERT_CMD_STATE_NEW;
     hwq->submit_command(std::get<0>(cmd).get()->get());
     issued++;
     if (issued >= total_cmd_submission)
@@ -167,11 +182,15 @@ io_test_cmd_submit_and_wait_thruput(
     completed++;
 
     if (issued < total_cmd_submission) {
-      std::get<1>(cmdlist_bos[wait_idx])->state = ERT_CMD_STATE_NEW;
+      // For UMQ, restore header; for KMQ, just reset state
+      if (bo_set_ptr && wait_idx < bo_set_ptr->size())
+        (*bo_set_ptr)[wait_idx]->restore_cmd_header();
+      else
+        std::get<1>(cmdlist_bos[wait_idx])->state = ERT_CMD_STATE_NEW;
+      
       hwq->submit_command(std::get<0>(cmdlist_bos[wait_idx]).get()->get());
       issued++;
     }
-
     if (++wait_idx == cmdlist_bos.size())
       wait_idx = 0;
   }
@@ -255,6 +274,7 @@ io_test(device::id_type id, device* dev, int total_hwq_submit, int num_cmdlist,
   // Initialize cmd before submission
   for (auto& boset : bo_set) {
     boset->init_cmd(hwctx, io_test_parameters.debug);
+    boset->cache_cmd_header();
     boset->sync_before_run();
   }
 
@@ -292,10 +312,15 @@ io_test(device::id_type id, device* dev, int total_hwq_submit, int num_cmdlist,
 
   // Submit commands and wait for results
   auto start = clk::now();
+  auto device_id = device_query<query::pcie_device>(dev);
+  bool is_umq = (device_id == npu3_device_id || device_id == npu3_device_id1);
+
   if (io_test_parameters.perf == IO_TEST_THRUPUT_PERF)
-    io_test_cmd_submit_and_wait_thruput(hwq, total_hwq_submit, cmdlist_bos);
+    io_test_cmd_submit_and_wait_thruput(hwq, total_hwq_submit, cmdlist_bos,
+                                        is_umq ? &bo_set : nullptr);
   else
-    io_test_cmd_submit_and_wait_latency(hwq, total_hwq_submit, cmdlist_bos);
+    io_test_cmd_submit_and_wait_latency(hwq, total_hwq_submit, cmdlist_bos, 
+                                        is_umq ? &bo_set : nullptr);
   auto end = clk::now();
 
   // Verify preemption counters

test/shim_test/shim_test.cpp

@@ -779,7 +779,7 @@ std::vector<test_case> test_list {
     TEST_POSITIVE, dev_filter_is_npu4, TEST_instr_invalid_addr_io, {}
   },
   test_case{ "measure no-op kernel latency", {},
-    TEST_POSITIVE, dev_filter_is_aie2, TEST_io_latency, { IO_TEST_NOOP_RUN, IO_TEST_IOCTL_WAIT, 32000 }
+    TEST_POSITIVE, dev_filter_is_xdna, TEST_io_latency, { IO_TEST_NOOP_RUN, IO_TEST_IOCTL_WAIT, 32000 }
   },
   test_case{ "measure real kernel latency", {},
     TEST_POSITIVE, dev_filter_is_aie2, TEST_io_latency, { IO_TEST_NORMAL_RUN, IO_TEST_IOCTL_WAIT, 32000 }