[RF] Added benchmark plot scripts and update RooFitBinned benchmarks

Abhigyan Acherjee · guitargeek · commit 05abafcc4148 · 2024-03-18T16:13:00.000+01:00
diff --git a/root/roofit/roofit/CMakeLists.txt b/root/roofit/roofit/CMakeLists.txt
@@ -19,6 +19,8 @@ if(cuda)
 endif()
 
 file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/benchRooFitBackends_make_plot.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/compare_benchmarks.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/run_benchmarks.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
 
 RB_ADD_GBENCHMARK(benchCodeSquashAD
   benchCodeSquashAD.cxx
diff --git a/root/roofit/roofit/RooFitBinnedBenchmarks.cxx b/root/roofit/roofit/RooFitBinnedBenchmarks.cxx
@@ -19,25 +19,27 @@ using namespace RooStats;
 using namespace HistFactory;
 
 namespace {
-  constexpr bool verbose = false;
+constexpr bool verbose = false;
 
-  // test matrix configuration
-  const std::vector<int> nChannelsVector = {1, 2, 3};
-  const std::vector<int> nBinsVector {5, 10, 15};
-  const int nBinsForChannelScan = 10;
-  const int nChannelsForBinScan = 1;
-  const std::vector<int> nCPUVector {1, 2, 3};
+// test matrix configuration
+const std::vector<int> nChannelsVector = {1, 2, 3};
+const std::vector<int> nBinsVector{5, 10, 15};
+const int nBinsForChannelScan = 10;
+const int nChannelsForBinScan = 1;
+const std::vector<int> nCPUVector{1};
 
-  constexpr auto evalBackend = RooFit::EvalBackend::Value::Cpu;
+////default evaluation backend
+std::string evalBackend = "cpu";
 
-  auto const timeUnit = benchmark::kMillisecond;
+auto const timeUnit = benchmark::kMillisecond;
 
-  void setupRooMsgService() {
-     RooMsgService::instance().setGlobalKillBelow(RooFit::FATAL);
-     RooMsgService::instance().getStream(1).removeTopic(RooFit::Minimization);
-     RooMsgService::instance().getStream(1).removeTopic(RooFit::NumIntegration);
-     RooMsgService::instance().getStream(1).removeTopic(RooFit::Eval);
-  }
+void setupRooMsgService()
+{
+   RooMsgService::instance().setGlobalKillBelow(RooFit::FATAL);
+   RooMsgService::instance().getStream(1).removeTopic(RooFit::Minimization);
+   RooMsgService::instance().getStream(1).removeTopic(RooFit::NumIntegration);
+   RooMsgService::instance().getStream(1).removeTopic(RooFit::Eval);
+}
 
 } // namespace
 
@@ -48,9 +50,9 @@ Sample addVariations(Sample asample, int nnps, bool channel_crosstalk, int chann
       Double_t random = R->Rndm();
       double uncertainty_up = (1 + random) / sqrt(100);
       double uncertainty_down = (1 - random) / sqrt(100);
-      if(verbose) {
-        std::cout << "in channel " << channel << "nuisance +/- [" << uncertainty_up << "," << uncertainty_down << "]"
-                  << std::endl;
+      if (verbose) {
+         std::cout << "in channel " << channel << "nuisance +/- [" << uncertainty_up << "," << uncertainty_down << "]"
+                   << std::endl;
       }
       std::string nuis_name = "norm_uncertainty_" + std::to_string(nuis);
       if (!channel_crosstalk) {
@@ -101,8 +103,8 @@ std::unique_ptr<RooStats::HistFactory::Channel> makeChannel(int channel, int nbi
 
 void buildBinnedTest(int n_channels = 1, int nbins = 10, int nnps = 1, const char *name_rootfile = "")
 {
-   if(verbose) {
-     std::cout << "in build binned test with output" << name_rootfile << std::endl;
+   if (verbose) {
+      std::cout << "in build binned test with output" << name_rootfile << std::endl;
    }
    Measurement meas("meas", "meas");
    meas.SetPOI("SignalStrength");
@@ -121,19 +123,20 @@ void buildBinnedTest(int n_channels = 1, int nbins = 10, int nnps = 1, const cha
    } else {
       ws = std::unique_ptr<RooWorkspace>{hist2workspace.MakeCombinedModel(meas)};
    }
-   for (RooAbsArg * arg : ws->components()) {
+   for (RooAbsArg *arg : ws->components()) {
       if (arg->IsA() == RooRealSumPdf::Class()) {
          arg->setAttribute("BinnedLikelihood");
-         if(verbose) std::cout << "component " << arg->GetName() << " is a binned likelihood" << std::endl;
+         if (verbose)
+            std::cout << "component " << arg->GetName() << " is a binned likelihood" << std::endl;
       }
    }
    ws->SetName("BinnedWorkspace");
    ws->writeToFile(name_rootfile);
 }
 
-//############## End of Base Algorithms ##############################
-//####################################################################
-//############## Start Of # Tests #############################
+// ############## End of Base Algorithms ##############################
+// ####################################################################
+// ############## Start Of # Tests #############################
 
 static void BM_RooFit_BinnedTestMigrad(benchmark::State &state)
 {
@@ -145,7 +148,8 @@ static void BM_RooFit_BinnedTestMigrad(benchmark::State &state)
    auto infile = std::make_unique<TFile>("workspace.root", "RECREATE");
    //   if (infile->IsZombie()) {
    buildBinnedTest(chan, nbins, 2, "workspace.root");
-   if(verbose) std::cout << "Workspace for tests was created!" << std::endl;
+   if (verbose)
+      std::cout << "Workspace for tests was created!" << std::endl;
    //}
    infile.reset(TFile::Open("workspace.root"));
    RooWorkspace *w = static_cast<RooWorkspace *>(infile->Get("BinnedWorkspace"));
@@ -174,7 +178,8 @@ static void BM_RooFit_BinnedTestHesse(benchmark::State &state)
    TFile *infile = new TFile("workspace.root");
    //   if (infile->IsZombie()) {
    buildBinnedTest(chan, nbins, 2, "workspace.root");
-   if(verbose) std::cout << "Workspace for tests was created!" << std::endl;
+   if (verbose)
+      std::cout << "Workspace for tests was created!" << std::endl;
    //   }
    infile = TFile::Open("workspace.root");
    RooWorkspace *w = static_cast<RooWorkspace *>(infile->Get("BinnedWorkspace"));
@@ -211,7 +216,8 @@ static void BM_RooFit_BinnedTestMinos(benchmark::State &state)
    auto infile = std::make_unique<TFile>("workspace.root");
    //   if (infile->IsZombie()) {
    buildBinnedTest(chan, nbins, 2, "workspace.root");
-   if(verbose) std::cout << "Workspace for tests was created!" << std::endl;
+   if (verbose)
+      std::cout << "Workspace for tests was created!" << std::endl;
    //}
    infile.reset(TFile::Open("workspace.root"));
    RooWorkspace *w = static_cast<RooWorkspace *>(infile->Get("BinnedWorkspace"));
@@ -235,11 +241,11 @@ static void BM_RooFit_BinnedTestMinos(benchmark::State &state)
    }
 }
 
-//############## Run # Tests ###############################
+// ############## Run # Tests ###############################
 
 static void ChanArguments(benchmark::internal::Benchmark *b)
 {
-      // channel scan
+   // channel scan
    for (int nChannels : nChannelsVector) {
       for (int nCPU : nCPUVector) {
          b->Args({nChannels, nBinsForChannelScan, nCPU});
@@ -254,24 +260,26 @@ static void ChanArguments(benchmark::internal::Benchmark *b)
    }
 }
 
-BENCHMARK(BM_RooFit_BinnedTestMigrad)
-   ->Apply(ChanArguments)
-   ->UseRealTime()
-   ->Unit(timeUnit)
-   ->Iterations(1);
-BENCHMARK(BM_RooFit_BinnedTestHesse)
-   ->Apply(ChanArguments)
-   ->UseRealTime()
-   ->Unit(timeUnit)
-   ->Iterations(1);
-BENCHMARK(BM_RooFit_BinnedTestMinos)
-   ->Apply(ChanArguments)
-   ->UseRealTime()
-   ->Unit(timeUnit)
-   ->Iterations(1);
+BENCHMARK(BM_RooFit_BinnedTestMigrad)->Apply(ChanArguments)->UseRealTime()->Unit(timeUnit)->Iterations(1);
+BENCHMARK(BM_RooFit_BinnedTestHesse)->Apply(ChanArguments)->UseRealTime()->Unit(timeUnit)->Iterations(1);
+BENCHMARK(BM_RooFit_BinnedTestMinos)->Apply(ChanArguments)->UseRealTime()->Unit(timeUnit)->Iterations(1);
+
+// ############## End Of Tests ########################################
+// ####################################################################
+// ############## RUN #################################################
+
+int main(int argc, char **argv)
+{
 
-//############## End Of Tests ########################################
-//####################################################################
-//############## RUN #################################################
+   benchmark::Initialize(&argc, argv);
 
-BENCHMARK_MAIN();
+   for (int i = 1; i < argc; ++i) {
+      if (std::string(argv[i]) == "-b") {
+         if (i + 1 < argc) {
+            // Set the evalBackend value from the next command-line argument
+            evalBackend = argv[i + 1];
+         }
+      }
+   }
+   benchmark::RunSpecifiedBenchmarks();
+}
diff --git a/root/roofit/roofit/compare_benchmarks.py b/root/roofit/roofit/compare_benchmarks.py
@@ -0,0 +1,52 @@
+import pandas as pd
+import csv
+import matplotlib.pyplot as plt
+import numpy as np
+from io import StringIO
+
+
+def read_csv(csv_file_path):
+    with open(csv_file_path, "r") as csvfile:
+        lines = [line for line in csvfile if ("BM_RooFit" in line or "name," in line)]
+    print("\n".join(lines))
+    return pd.read_csv(StringIO("\n".join(lines)))
+
+
+codegen_df = read_csv("out_codegen.csv")
+codegen_nograd_df = read_csv("out_codegen_ngrad.csv")
+legacy_df = read_csv("out_legacy.csv")
+cpu_df = read_csv("out_cpu.csv")
+
+# Plotting
+plt.figure(figsize=(10, 6))
+
+x = np.arange(len(codegen_df["name"].unique()))
+
+
+for i, benchmark in enumerate(codegen_df["name"].unique()):
+
+    codegen_time = codegen_df.loc[codegen_df["name"] == benchmark, "real_time"]
+    codegen_nograd_time = codegen_nograd_df.loc[codegen_nograd_df["name"] == benchmark, "real_time"]
+    cpu_time = cpu_df.loc[cpu_df["name"] == benchmark, "real_time"]
+    legacy_time = legacy_df.loc[legacy_df["name"] == benchmark, "real_time"]
+
+    plt.bar(x[i] - 0.10, codegen_time, width=0.15, align="center", label="codegen", color="lightblue")
+    plt.bar(x[i], codegen_nograd_time, width=0.15, align="edge", label="codegen_nograd", color="navy")
+    plt.bar(x[i] + 0.15, cpu_time, width=0.15, align="edge", label="cpu", color="cyan")
+    plt.bar(x[i] + 0.30, legacy_time, width=0.15, align="edge", label="legacy", color="gray")
+
+
+# Customize legend
+legend_labels = ["codegen", "codegen_nograd", "cpu", "legacy"]
+legend_colors = ["lightblue", "navy", "cyan", "gray"]
+legend_handles = [plt.Rectangle((0, 0), 1, 1, color=color) for color in legend_colors]
+plt.legend(legend_handles, legend_labels)
+
+plt.yscale("log")
+
+plt.xlabel("Benchmark")
+plt.ylabel("Time (milliseconds)")
+plt.title("Comparison of Benchmarks for Different Evaluation Backends")
+plt.xticks(x, rotation=90)
+plt.tight_layout()
+plt.savefig("comparision_plot.png")
diff --git a/root/roofit/roofit/run_benchmarks.sh b/root/roofit/roofit/run_benchmarks.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Function to run a benchmark script and wait until the CSV file is generated
+# to make it executable chmod +x run_benchmarks.sh
+#then run it using ./run_benchmarks.sh
+#!/bin/bash
+
+# Function to run the benchmark command and wait for CSV file to be generated
+run_benchmark() {
+    echo "Running benchmark: $1"
+    $1 &
+    local pid=$!
+    while [ ! -f $2 ]; do
+        sleep 1
+    done
+    wait $pid
+    echo "CSV file generated: $2"
+}
+
+# Run benchmarks
+run_benchmark "./benchRooFitBinned -b codegen --benchmark_out_format=csv --benchmark_out=out_codegen.csv" "out_codegen.csv"
+run_benchmark "./benchRooFitBinned -b codegen_no_grad --benchmark_out_format=csv --benchmark_out=out_codegen_ngrad.csv" "out_codegen_ngrad.csv"
+run_benchmark "./benchRooFitBinned -b legacy --benchmark_out_format=csv --benchmark_out=out_legacy.csv" "out_legacy.csv"
+run_benchmark "./benchRooFitBinned -b cpu --benchmark_out_format=csv --benchmark_out=out_cpu.csv" "out_cpu.csv"
+
+# Run Python script
+python3 compare_benchmarks.py
