Shrinked ONNX gbenchs into a single function

Author: fsossai

root/tmva/sofie/CMakeLists.txt

@@ -21,33 +21,52 @@ if(ROOT_tmva_FOUND AND ROOT_tmva-sofie_FOUND)
   if(ONNXRuntime_FOUND)
     message(STATUS "Found ONNXRuntime (build type: ${ONNXRuntime_BUILD_TYPE}, version: ${ONNXRuntime_VERSION_STRING})")
 
-    # Creating an ONNXRuntime inference benchmark for every model
-    foreach(fname ${ONNX_MODELS})
-      get_filename_component(fname ${fname} NAME_WE)
-      set(target_name ONNXRuntime_${fname})
-      configure_file(ONNXRuntimeInference.cxx.in ${target_name}.cxx)
-
-      RB_ADD_GBENCHMARK(${target_name}
-        ${target_name}.cxx
-        LABEL short
-        LIBRARIES TMVA onnxruntime
-      )
-      target_link_directories(${target_name} PRIVATE ${ONNXRuntime_LIBRARIES})
-      target_include_directories(${target_name} PRIVATE ${ONNXRuntime_INCLUDE_DIR})
+    # Configuring ONNXRuntimeInference_Template.cxx.in
+    set(FUNC_NAME "BM_ONNXRuntime_Inference")
+    set(CAPTURE_STR "BENCHMARK_CAPTURE(${FUNC_NAME}, @1,\t@2)@3")
+    set(HEAD_COMMENT "Automatically configured by CMake")
+    set(ALL_CAPTURES "")
+    set(ONNX_MODEL_NAMES "")
+    foreach(model ${ONNX_MODELS})
+      get_filename_component(fname ${model} NAME)
+      get_filename_component(fname_we ${model} NAME_WE)
+      string(REPLACE "@1" ${fname_we} cap ${CAPTURE_STR})
+      string(REPLACE "@2" "\"${ONNX_MODELS_DIR}/${fname}\"" cap ${cap})
+      list(APPEND ALL_CAPTURES ${cap})
     endforeach()
+    string(REPLACE ";" "\n" BENCHMARK_CAPTURES "${ALL_CAPTURES}")         # String[] -> String
+    string(REPLACE "@3" ";" BENCHMARK_CAPTURES "${BENCHMARK_CAPTURES}")   # Adding semicolon
+    configure_file(ONNXRuntimeInference_Template.cxx.in ONNXRuntimeInference.cxx @ONLY)
+
+    RB_ADD_GBENCHMARK(ONNXRuntimeInference
+      ONNXRuntimeInference.cxx
+      LABEL short
+      LIBRARIES TMVA onnxruntime
+    )
+    target_link_directories(ONNXRuntimeInference PRIVATE ${ONNXRuntime_LIBRARIES})
+    target_include_directories(ONNXRuntimeInference PRIVATE ${ONNXRuntime_INCLUDE_DIR})
+
   endif()
 
   # Checking the SOFIE compiler
   if(NOT EXISTS ${ROOTSYS}/tmva/sofie/test/emitFromONNX)
     message(FATAL_ERROR "SOFIE compiler not found")
   endif()
 
-  # Creating an inference benchmark for SOFIE
+  # Benchmark for models emitted by SOFIE
   RB_ADD_GBENCHMARK(SOFIEInference
     SOFIEInference.cxx
     LABEL short
     LIBRARIES TMVA blas
   )
+
+  # Benchmark for models emitted by SOFIE (with RModelProfiler)
+  # This should be useful in measuring profiler's overhead
+  RB_ADD_GBENCHMARK(SOFIEInferenceProfiler
+    SOFIEInferenceProfiler.cxx
+    LABEL short
+    LIBRARIES TMVA blas
+  )
 
 endif()
 

root/tmva/sofie/ONNXRuntimeInference.cxx.in renamed to root/tmva/sofie/ONNXRuntimeInference_Template.cxx.in

@@ -1,3 +1,4 @@
+// @HEAD_COMMENT@
 // Author: Federico Sossai (fsossai), 2021
 
 #include <benchmark/benchmark.h>
@@ -9,10 +10,8 @@
 
 using namespace std;
 
-static void BM_ONNXRuntime_${fname}(benchmark::State& state)
+static void @FUNC_NAME@(benchmark::State& state, string model_path)
 {
-   const string model_path = "${ONNX_MODELS_DIR}/${fname}.onnx";
-
    Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "benchmark");
 
    Ort::SessionOptions session_options;
@@ -34,20 +33,6 @@ static void BM_ONNXRuntime_${fname}(benchmark::State& state)
       .GetInputTypeInfo(0).GetTensorTypeAndShapeInfo().GetShape();
    vector<int64_t> output_node_dims = session
       .GetOutputTypeInfo(0).GetTensorTypeAndShapeInfo().GetShape();
-   
-   // Displaying input tensor shape
-
-   cout << "Input shape: [ ";
-   for (auto& i: input_node_dims)
-      cout << i << " ";
-   cout << "]" << endl;
-
-   // Displaying output tensor shape
-
-   cout << "Output shape: [ ";
-   for (auto& i: output_node_dims)
-      cout << i << " ";
-   cout << "]" << endl;
 
    // Calculating the dimension of the input tensor
 
@@ -70,6 +55,6 @@ static void BM_ONNXRuntime_${fname}(benchmark::State& state)
          &input_tensor, 1, output_node_names.data(), 1);
    }
 }
-BENCHMARK(BM_ONNXRuntime_${fname});
+@BENCHMARK_CAPTURES@
 
 BENCHMARK_MAIN();

root/tmva/sofie/SOFIEInference.cxx

@@ -3,13 +3,42 @@
 #include <benchmark/benchmark.h>
 
 #include <iostream>
+#include <thread>
+#include <chrono>
+#include <utility>
+#include <vector>
+#include <memory>
+#include <functional>
 
-static void BM_SOFIE_Inference(benchmark::State& state)
+#include "input_models/compiled/Linear_16.hxx"
+#include "input_models/compiled/Linear_32.hxx"
+#include "input_models/compiled/Linear_64.hxx"
+
+using namespace std;
+
+static void BM_SOFIE_Inference(benchmark::State& state, std::string model_name)
 {
+    vector<float> input;
+    vector<float> (*infer_func)(float*);
+
+    if (model_name == "Linear_16") {
+        input.resize(1600);
+        infer_func = TMVA_SOFIE_Linear_16::infer;
+    } else if (model_name == "Linear_32") {
+        input.resize(3200);
+        infer_func = TMVA_SOFIE_Linear_32::infer;
+    } else if (model_name == "Linear_64") {
+        input.resize(6400);
+        infer_func = TMVA_SOFIE_Linear_64::infer;
+    }
+    
+    float *input_ptr = input.data();
     for (auto _ : state) {
-        ;
+        infer_func(input_ptr);
     }
 }
-BENCHMARK(BM_SOFIE_Inference);
+BENCHMARK_CAPTURE(BM_SOFIE_Inference, Linear_16, "Linear_16");
+BENCHMARK_CAPTURE(BM_SOFIE_Inference, Linear_32, "Linear_32");
+BENCHMARK_CAPTURE(BM_SOFIE_Inference, Linear_64, "Linear_64");
 
 BENCHMARK_MAIN();

root/tmva/sofie/SOFIEInferenceProfiler.cxx

@@ -0,0 +1,47 @@
+// Author: Federico Sossai (fsossai), 2021
+
+#include <benchmark/benchmark.h>
+
+#include <iostream>
+#include <thread>
+#include <chrono>
+#include <utility>
+#include <vector>
+#include <memory>
+#include <functional>
+
+#include "input_models/compiled/Linear_16_prof.hxx"
+#include "input_models/compiled/Linear_32_prof.hxx"
+#include "input_models/compiled/Linear_64_prof.hxx"
+
+using namespace std;
+
+static void BM_SOFIE_Inference(benchmark::State& state, std::string model_name)
+{
+    vector<float> input;
+    vector<float> (*infer_func)(float*);
+
+    if (model_name == "Linear_16") {
+        input.resize(1600);
+        infer_func = TMVA_SOFIE_Linear_16::infer;
+    } else if (model_name == "Linear_32") {
+        input.resize(3200);
+        infer_func = TMVA_SOFIE_Linear_32::infer;
+    } else if (model_name == "Linear_64") {
+        input.resize(6400);
+        infer_func = TMVA_SOFIE_Linear_64::infer;
+    }
+    
+    float *input_ptr = input.data();
+    for (auto _ : state) {
+        infer_func(input_ptr);
+    }
+    cout << TMVA_SOFIE_Linear_16::profiler_results.size() << endl;
+    cout << TMVA_SOFIE_Linear_32::profiler_results.size() << endl;
+    cout << TMVA_SOFIE_Linear_64::profiler_results.size() << endl;
+}
+BENCHMARK_CAPTURE(BM_SOFIE_Inference, Linear_16, "Linear_16");
+BENCHMARK_CAPTURE(BM_SOFIE_Inference, Linear_32, "Linear_32");
+BENCHMARK_CAPTURE(BM_SOFIE_Inference, Linear_64, "Linear_64");
+
+BENCHMARK_MAIN();