Arm backend: Enable multiple inferences in executor runner (pytorch#13177)

The ET_NUM_INFERENCES macro can now be set to specify a number of
inferences to run with Arm executor runner.

The StopMeasurements function is also given a new argument, int
num_inferences, in order to display data per inference.

Signed-off-by: [email protected]
Co-authored-by: Martin Lindström <[email protected]>

Author: perheld

examples/arm/executor_runner/CMakeLists.txt

@@ -20,6 +20,10 @@ option(ET_DUMP_OUTPUT "Dump output in log" ON)
 option(FETCH_ETHOS_U_CONTENT
        "Fetch ethos_u dependencies instead of relying on pre-downloads" ON
 )
+set(ET_NUM_INFERENCES
+    "1"
+    CACHE STRING "Number of inferences to run"
+)
 
 if(NOT DEFINED ET_PTE_FILE_PATH AND NOT ${SEMIHOSTING})
   message(
@@ -77,6 +81,7 @@ set(MEMORY_MODE
 
 message(STATUS "SYSTEM_CONFIG is ${SYSTEM_CONFIG}")
 message(STATUS "MEMORY_MODE is ${MEMORY_MODE}")
+message(STATUS "ET_NUM_INFERENCES is ${ET_NUM_INFERENCES}")
 
 get_filename_component(ET_BUILD_DIR_PATH ${ET_BUILD_DIR_PATH} REALPATH)
 get_filename_component(ET_DIR_PATH ${ET_DIR_PATH} REALPATH)
@@ -255,6 +260,12 @@ if(ET_DUMP_OUTPUT)
   target_compile_definitions(arm_executor_runner PUBLIC -DET_DUMP_OUTPUT)
 endif()
 
+if(ET_NUM_INFERENCES)
+  target_compile_definitions(
+    arm_executor_runner PUBLIC ET_NUM_INFERENCES=${ET_NUM_INFERENCES}
+  )
+endif()
+
 # Fixup compilation of retarget.c
 if(SEMIHOSTING)
   # Remove this when MLBEDSW-8910 is closed.

examples/arm/executor_runner/arm_executor_runner.cpp

@@ -130,6 +130,12 @@ const float et_rtol = 0.01;
 
 #endif
 
+#if defined(ET_NUM_INFERENCES)
+const int num_inferences = ET_NUM_INFERENCES;
+#else
+const int num_inferences = 1;
+#endif
+
 /**
  * The temp_allocation_pool is used for allocating temporary data during kernel
  * or delegate execution. This will be reset after each kernel or delegate call.
@@ -638,21 +644,6 @@ void runner_init(
   ET_LOG(Info, "Input prepared.");
 }
 
-void run_model(RunnerContext& ctx) {
-  ET_LOG(Info, "Starting the model execution...");
-
-  StartMeasurements();
-  // Run the model.
-  Error status = ctx.method.value()->execute();
-  StopMeasurements();
-
-  ET_CHECK_MSG(
-      status == Error::Ok,
-      "Execution of method %s failed with status 0x%" PRIx32,
-      ctx.method_name,
-      status);
-}
-
 void log_mem_status(const RunnerContext& ctx) {
   size_t executor_memsize =
       ctx.method_allocator->used_size() - ctx.executor_membase;
@@ -853,6 +844,32 @@ void verify_result(RunnerContext& ctx, const void* model_pte) {
 #endif
 }
 
+void run_model(RunnerContext& ctx, const void* model_pte) {
+  Error status;
+  ET_LOG(Info, "Starting running %d inferences...", num_inferences);
+
+  int n = 0;
+  StartMeasurements();
+  for (n = 1; n <= num_inferences; n++) {
+    // Run the model.
+    status = ctx.method.value()->execute();
+    if (status != Error::Ok) {
+      break;
+    }
+  }
+  StopMeasurements(n);
+
+  ET_CHECK_MSG(
+      status == Error::Ok,
+      "Execution of method %s failed with status 0x%" PRIx32,
+      ctx.method_name,
+      status);
+
+  ET_LOG(Info, "%d inferences finished", num_inferences);
+  print_outputs(ctx);
+  verify_result(ctx, model_pte);
+}
+
 } // namespace
 
 int main(int argc, const char* argv[]) {
@@ -934,11 +951,9 @@ int main(int argc, const char* argv[]) {
       Info, "PTE in %p %c Size: %lu bytes", model_pte, model_pte[0], pte_size);
 
   runner_init(ctx, input_buffers, pte_size);
-  run_model(ctx);
+  run_model(ctx, model_pte);
   log_mem_status(ctx);
-  print_outputs(ctx);
   write_etdump(ctx);
-  verify_result(ctx, model_pte);
 
   ET_LOG(Info, "Program complete, exiting.");
 #if defined(SEMIHOSTING)

examples/arm/executor_runner/arm_perf_monitor.cpp

@@ -4,8 +4,8 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+#include <array>
 #include <cinttypes>
-#include <vector>
 
 #include "arm_perf_monitor.h"
 
@@ -14,29 +14,31 @@
 #include <executorch/runtime/platform/log.h>
 #include <pmu_ethosu.h>
 
-static uint32_t ethosu_inference_count = 0;
-static uint64_t ethosu_ArmCycleCountStart = 0;
-static uint64_t ethosu_ArmBackendExecuteCycleCountStart = 0;
-static uint64_t ethosu_ArmBackendExecuteCycleCount = 0;
-static uint64_t ethosu_ArmWhenNPURunCycleCountStart = 0;
-static uint64_t ethosu_ArmWhenNPURunCycleCount = 0;
-static uint64_t ethosu_pmuCycleCount = 0;
-static std::vector<uint64_t> ethosu_pmuEventCounts(
-    ETHOSU_PMU_Get_NumEventCounters(),
-    0);
+namespace {
 
 #if defined(ETHOSU55) || defined(ETHOSU65)
-static const uint32_t ethosu_pmuCountersUsed = 4;
+const uint32_t ethosu_pmuCountersUsed = 4;
 #elif defined(ETHOSU85)
-static const uint32_t ethosu_pmuCountersUsed = 5;
+const uint32_t ethosu_pmuCountersUsed = 5;
 #else
 #error No NPU target defined
 #endif
 
+uint32_t ethosu_delegation_count = 0;
+uint64_t ethosu_ArmCycleCountStart = 0;
+uint64_t ethosu_ArmBackendExecuteCycleCountStart = 0;
+uint64_t ethosu_ArmBackendExecuteCycleCount = 0;
+uint64_t ethosu_ArmWhenNPURunCycleCountStart = 0;
+uint64_t ethosu_ArmWhenNPURunCycleCount = 0;
+uint64_t ethosu_pmuCycleCount = 0;
+std::array<uint64_t, ethosu_pmuCountersUsed> ethosu_pmuEventCounts = {0};
+
 // ethosu_pmuCountersUsed should match numbers of counters setup in
 // ethosu_inference_begin() and not be more then the HW supports
 static_assert(ETHOSU_PMU_NCOUNTERS >= ethosu_pmuCountersUsed);
 
+} // namespace
+
 extern "C" {
 
 // Callback invoked at start of NPU execution
@@ -85,7 +87,7 @@ void ethosu_inference_begin(struct ethosu_driver* drv, void*) {
 
 // Callback invoked at end of NPU execution
 void ethosu_inference_end(struct ethosu_driver* drv, void*) {
-  ethosu_inference_count++;
+  ethosu_delegation_count++;
   ethosu_pmuCycleCount += ETHOSU_PMU_Get_CCNTR(drv);
 
   for (size_t i = 0; i < ethosu_pmuCountersUsed; i++) {
@@ -113,6 +115,7 @@ void EthosUBackend_execute_end() {
 }
 
 void StartMeasurements() {
+  ethosu_delegation_count = 0;
   ethosu_ArmBackendExecuteCycleCount = 0;
   ethosu_ArmWhenNPURunCycleCount = 0;
   ethosu_pmuCycleCount = 0;
@@ -123,32 +126,43 @@ void StartMeasurements() {
   ethosu_ArmCycleCountStart = ARM_PMU_Get_CCNTR();
 }
 
-void StopMeasurements() {
+void StopMeasurements(int num_inferences) {
   ARM_PMU_CNTR_Disable(
       PMU_CNTENCLR_CCNTR_ENABLE_Msk | PMU_CNTENCLR_CNT0_ENABLE_Msk |
       PMU_CNTENCLR_CNT1_ENABLE_Msk);
   uint32_t cycle_count = ARM_PMU_Get_CCNTR() - ethosu_ArmCycleCountStart;
 
   // Number of comand streams handled by the NPU
-  ET_LOG(Info, "NPU Inferences : %d", ethosu_inference_count);
+  ET_LOG(Info, "NPU Inferences : %d", num_inferences);
+  ET_LOG(
+      Info,
+      "NPU delegations: %d (%.2f per inference)",
+      ethosu_delegation_count,
+      (double)ethosu_delegation_count / num_inferences);
   ET_LOG(Info, "Profiler report, CPU cycles per operator:");
   // This is number of CPU cycles for the ethos-u operator from start to finish
   // in the framework If there is more then one commandstream the time is added
   // together
   ET_LOG(
       Info,
-      "ethos-u : cycle_cnt : %d cycles",
-      ethosu_ArmBackendExecuteCycleCount);
+      "ethos-u : cycle_cnt : %d cycles (%.2f per inference)",
+      ethosu_ArmBackendExecuteCycleCount,
+      (double)ethosu_ArmBackendExecuteCycleCount / num_inferences);
   // We could print a list of the cycles used by the other delegates here in the
   // future but now we only print ethos-u: this means that "Operator(s) total:
   // ..." will be the same number as ethos-u : cycle_cnt and not the sum of all
   ET_LOG(
       Info,
-      "Operator(s) total: %d CPU cycles",
-      ethosu_ArmBackendExecuteCycleCount);
+      "Operator(s) total: %d CPU cycles (%.2f per inference)",
+      ethosu_ArmBackendExecuteCycleCount,
+      (double)ethosu_ArmBackendExecuteCycleCount / num_inferences);
   // Total CPU cycles used in the executorch method->execute()
   // Other delegates and no delegates are counted in this
-  ET_LOG(Info, "Inference runtime: %d CPU cycles total", cycle_count);
+  ET_LOG(
+      Info,
+      "Inference runtime: %d CPU cycles total (%.2f per inference)",
+      cycle_count,
+      (double)cycle_count / num_inferences);
 
   ET_LOG(
       Info,
@@ -174,14 +188,24 @@ void StopMeasurements() {
   // If there is more then one commandstream the time is added together
   ET_LOG(
       Info,
-      "cpu_wait_for_npu_cntr : %" PRIu64 " CPU cycles",
-      ethosu_ArmWhenNPURunCycleCount);
+      "cpu_wait_for_npu_cntr : %" PRIu64 " CPU cycles (%.2f per inference)",
+      ethosu_ArmWhenNPURunCycleCount,
+      (double)ethosu_ArmWhenNPURunCycleCount / num_inferences);
 
   ET_LOG(Info, "Ethos-U PMU report:");
-  ET_LOG(Info, "ethosu_pmu_cycle_cntr : %" PRIu64, ethosu_pmuCycleCount);
+  ET_LOG(
+      Info,
+      "ethosu_pmu_cycle_cntr : % " PRIu64 " (%.2f per inference)",
+      ethosu_pmuCycleCount,
+      (double)ethosu_pmuCycleCount / num_inferences);
 
   for (size_t i = 0; i < ethosu_pmuCountersUsed; i++) {
-    ET_LOG(Info, "ethosu_pmu_cntr%zd : %" PRIu64, i, ethosu_pmuEventCounts[i]);
+    ET_LOG(
+        Info,
+        "ethosu_pmu_cntr%zd : %" PRIu64 " (%.2f per inference)",
+        i,
+        ethosu_pmuEventCounts[i],
+        (double)ethosu_pmuEventCounts[i] / num_inferences);
   }
 #if defined(ETHOSU55) || defined(ETHOSU65)
   ET_LOG(
@@ -199,6 +223,8 @@ void StopMeasurements() {
 #else
 void StartMeasurements() {}
 
-void StopMeasurements() {}
+void StopMeasurements(int num_inferences) {
+  (void)num_inferences;
+}
 
 #endif

examples/arm/executor_runner/arm_perf_monitor.h

@@ -1,4 +1,4 @@
-/* Copyright 2024 Arm Limited and/or its affiliates.
+/* Copyright 2024-2025 Arm Limited and/or its affiliates.
  *
  * This source code is licensed under the BSD-style license found in the
  * LICENSE file in the root directory of this source tree.
@@ -7,4 +7,4 @@
 #pragma once
 
 void StartMeasurements();
-void StopMeasurements();
+void StopMeasurements(int num_inferences);