From 191bd945b2be8c6342d6d1e363d3e1b124c31181 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Wed, 28 Jul 2021 09:04:14 +0200 Subject: [PATCH 01/24] Adding new benchmarks for BDT...trying out different options... --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 54 ++++++++++++++++++++++++++ root/tmva/tmva/CMakeLists.txt | 5 +++ root/tmva/tmva/MakeRandomTTree.h | 38 ++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 root/tmva/tmva/BoostedDTBenchmarks.cxx create mode 100644 root/tmva/tmva/MakeRandomTTree.h diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx new file mode 100644 index 000000000..c359baadf --- /dev/null +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -0,0 +1,54 @@ +#include "TMVA/CrossValidation.h" + +#include "TRandom3.h" +#include "TTree.h" + +#include "TMVA/DataLoader.h" +#include "TFile.h" + +#include "benchmark/benchmark.h" + +#include "MakeRandomTTree.h" + +static void BM_TMVA_BDTTraining(benchmark::State &state){ + UInt_t nVars = 4; + UInt_t nEvents = 10000; + + // Set up + TTree *sigTree = genTree(nEvents, nVars,0.3, 0.5, 100); + TTree *bkgTree = genTree(nEvents, nVars,-0.3, 0.5, 101); + + auto *dataloader = new TMVA::DataLoader("bdt-bench"); + dataloader->AddSignalTree(sigTree); + dataloader->AddBackgroundTree(bkgTree); + + for(UInt_t i = 0; i < nVars; i++){ + std::string var_name = "var" + std::to_string(i); + std::string var_leaflist = var_name + "/F"; + + dataloader->AddVariable(var_name.c_str(), 'D'); + } + + dataloader->PrepareTrainingAndTestTree("", Form("SplitMode=Block:nTrain_Signal=%i:nTrain_Background=%i:!V", + nEvents, nEvents)); + + // Benchmarking + for(auto& s: state){ + // Create a ROOT output file where TMVA will store ntuples, histograms, etc. + TString outfileName( "bdt_bench_output.root" ); + TFile* outputFile = TFile::Open(outfileName, "RECREATE"); + auto factory = new TMVA::Factory("bdt-bench", outputFile, + "!V:!DrawProgressBar:AnalysisType=Classification"); + + // Train a TMVA method + factory->BookMethod(dataloader, TMVA::Types::kBDT, "BDT", "!V:!H:NTrees=300:MaxDepth=2"); + factory->TrainAllMethods(); + } + + // Teardown + delete sigTree; + delete bkgTree; +} +BENCHMARK(BM_TMVA_BDTTraining); + +BENCHMARK_MAIN(); diff --git a/root/tmva/tmva/CMakeLists.txt b/root/tmva/tmva/CMakeLists.txt index a55abf4fc..43fbd5ca1 100644 --- a/root/tmva/tmva/CMakeLists.txt +++ b/root/tmva/tmva/CMakeLists.txt @@ -3,6 +3,11 @@ if(ROOT_tmva_FOUND) CrossValidationBenchmarks.cxx LABEL short LIBRARIES Core Tree MathCore TMVA) + + RB_ADD_GBENCHMARK(BoostedDTBenchmarks + BoostedDTBenchmarks.cxx + LABEL short + LIBRARIES Core Tree MathCore TMVA) endif() if(ROOT_tmva_FOUND AND ROOT_tmva-cpu_FOUND AND ROOT_imt_FOUND) diff --git a/root/tmva/tmva/MakeRandomTTree.h b/root/tmva/tmva/MakeRandomTTree.h new file mode 100644 index 000000000..5751568f3 --- /dev/null +++ b/root/tmva/tmva/MakeRandomTTree.h @@ -0,0 +1,38 @@ +#include "TRandom3.h" +#include "TTree.h" + +// Utility function for generating a random TTree with Gaussian float data, for the specified number of points and vars +TTree* genTree(UInt_t nPoints, const UInt_t nVars, Double_t offset, Double_t scale = 0.3, UInt_t seed = 100){ + // Initialisation + TRandom3 rng(seed); + Float_t vars[nVars]; for(auto& var: vars){ var = 0.0;} + UInt_t id = 0; + + // Create new TTree instance + auto data = new TTree(); + + // Add a branch corresponding to each variable + for(UInt_t i = 0; i < nVars; i++){ + std::string var_name = "var" + std::to_string(i); + std::string var_leaflist = var_name + "/F"; + + data->Branch(var_name.c_str(), vars + i, var_leaflist.c_str()); + } + + // And add a branch for the (unique) Event identifier + data->Branch("EventNumber", &id, "EventNumber/I"); + + // Populate TTree instance with Gaussian data + for(UInt_t j = 0; j < nPoints; j++){ + for(UInt_t i = 0; i < nVars; i++){ + vars[i] = rng.Gaus(offset, scale); + } + + data->Fill(); + id++; + } + + // Important: Disconnects the tree from the memory locations of x and y. + data->ResetBranchAddresses(); + return data; +} \ No newline at end of file From d5184ebfed5322bc9b0c99b69580232fd8ab5487 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Wed, 28 Jul 2021 09:25:11 +0200 Subject: [PATCH 02/24] Minor changes --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index c359baadf..5c0de6a53 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -33,15 +33,19 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ nEvents, nEvents)); // Benchmarking - for(auto& s: state){ + for(auto _: state){ // Create a ROOT output file where TMVA will store ntuples, histograms, etc. TString outfileName( "bdt_bench_output.root" ); TFile* outputFile = TFile::Open(outfileName, "RECREATE"); auto factory = new TMVA::Factory("bdt-bench", outputFile, "!V:!DrawProgressBar:AnalysisType=Classification"); + // Construct training options string + std::string opts = "!V:!H:NTrees=" + std::to_string(state.range(0)) + + ":MaxDepth=" + std::to_string(state.range(1)); + // Train a TMVA method - factory->BookMethod(dataloader, TMVA::Types::kBDT, "BDT", "!V:!H:NTrees=300:MaxDepth=2"); + factory->BookMethod(dataloader, TMVA::Types::kBDT, "BDT", opts); factory->TrainAllMethods(); } @@ -49,6 +53,6 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ delete sigTree; delete bkgTree; } -BENCHMARK(BM_TMVA_BDTTraining); +BENCHMARK(BM_TMVA_BDTTraining)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10}}); BENCHMARK_MAIN(); From 9a8da6b9ed684f7dfff7fc7b86b14a2f89c10b07 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Wed, 4 Aug 2021 12:34:39 +0200 Subject: [PATCH 03/24] Bug fixes --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 11 ++++++++--- root/tmva/tmva/CMakeLists.txt | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 5c0de6a53..832851f7a 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -29,14 +29,15 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ dataloader->AddVariable(var_name.c_str(), 'D'); } - dataloader->PrepareTrainingAndTestTree("", Form("SplitMode=Block:nTrain_Signal=%i:nTrain_Background=%i:!V", - nEvents, nEvents)); + dataloader->PrepareTrainingAndTestTree("", + Form("SplitMode=Block:nTrain_Signal=%i:nTrain_Background=%i:nTest_Signal=%i:nTest_Background=%i:!V", + nEvents-1, nEvents-1, 1, 1)); // Benchmarking for(auto _: state){ // Create a ROOT output file where TMVA will store ntuples, histograms, etc. TString outfileName( "bdt_bench_output.root" ); - TFile* outputFile = TFile::Open(outfileName, "RECREATE"); + auto outputFile = TFile::Open(outfileName, "RECREATE"); auto factory = new TMVA::Factory("bdt-bench", outputFile, "!V:!DrawProgressBar:AnalysisType=Classification"); @@ -47,6 +48,10 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // Train a TMVA method factory->BookMethod(dataloader, TMVA::Types::kBDT, "BDT", opts); factory->TrainAllMethods(); + + outputFile->Close(); + delete factory; + delete outputFile; } // Teardown diff --git a/root/tmva/tmva/CMakeLists.txt b/root/tmva/tmva/CMakeLists.txt index 43fbd5ca1..a7e3da44c 100644 --- a/root/tmva/tmva/CMakeLists.txt +++ b/root/tmva/tmva/CMakeLists.txt @@ -7,7 +7,7 @@ if(ROOT_tmva_FOUND) RB_ADD_GBENCHMARK(BoostedDTBenchmarks BoostedDTBenchmarks.cxx LABEL short - LIBRARIES Core Tree MathCore TMVA) + LIBRARIES Core Tree MathCore RIO TMVA) endif() if(ROOT_tmva_FOUND AND ROOT_tmva-cpu_FOUND AND ROOT_imt_FOUND) From 9859d617a1cf61ed917a87b2ec2193e3077064ff Mon Sep 17 00:00:00 2001 From: xmif1 Date: Thu, 5 Aug 2021 09:32:00 +0200 Subject: [PATCH 04/24] Major changes to fix from errors on teardown. Thank to @lmoneta for the suggestions. --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 48 +++++++++++++++++--------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 832851f7a..40414f27d 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -1,9 +1,14 @@ +/* Authored by Xandru Mifsud (CERN Summer Student) and Lorenzo Moneta (Summer Project Supervisor) */ + #include "TMVA/CrossValidation.h" #include "TRandom3.h" #include "TTree.h" #include "TMVA/DataLoader.h" +#include "TMVA/Factory.h" +#include "TMVA/MethodBase.h" +#include "TMVA/Types.h" #include "TFile.h" #include "benchmark/benchmark.h" @@ -12,16 +17,24 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ UInt_t nVars = 4; - UInt_t nEvents = 10000; + UInt_t nEvents = 500; + + // Set up (generate one extra event for testing) + TTree *sigTree = genTree(nEvents+1, nVars,0.3, 0.5, 100); + TTree *bkgTree = genTree(nEvents+1, nVars,-0.3, 0.5, 101); - // Set up - TTree *sigTree = genTree(nEvents, nVars,0.3, 0.5, 100); - TTree *bkgTree = genTree(nEvents, nVars,-0.3, 0.5, 101); + // one could maybe avoid TMVA to create output file by passing a nullptr to the Factory constructor + TString outfileName( "bdt_bench_output.root" ); + TFile* outputFile = TFile::Open(outfileName, "RECREATE"); + auto factory = new TMVA::Factory("bdt-bench", outputFile, + "Silent:!DrawProgressBar:AnalysisType=Classification"); + // Prepare a DataLoader instance, registering the signal and background TTrees auto *dataloader = new TMVA::DataLoader("bdt-bench"); dataloader->AddSignalTree(sigTree); dataloader->AddBackgroundTree(bkgTree); + // Register variables in dataloader, using naming convention for randomly generated TTrees in MakeRandomTTree.h for(UInt_t i = 0; i < nVars; i++){ std::string var_name = "var" + std::to_string(i); std::string var_leaflist = var_name + "/F"; @@ -29,32 +42,35 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ dataloader->AddVariable(var_name.c_str(), 'D'); } + // We add the single test event to avoid the printing of any errors from TMVA. + // For each benchmark we specifically ignore this test event such that we exclusively benchmark training. dataloader->PrepareTrainingAndTestTree("", - Form("SplitMode=Block:nTrain_Signal=%i:nTrain_Background=%i:nTest_Signal=%i:nTest_Background=%i:!V", - nEvents-1, nEvents-1, 1, 1)); + Form("SplitMode=Block:nTrain_Signal=%i:nTrain_Background=%i:nTest_Signal=%i:nTest_Background=%i:!V", + nEvents, nEvents, 1, 1)); // Benchmarking for(auto _: state){ // Create a ROOT output file where TMVA will store ntuples, histograms, etc. - TString outfileName( "bdt_bench_output.root" ); - auto outputFile = TFile::Open(outfileName, "RECREATE"); - auto factory = new TMVA::Factory("bdt-bench", outputFile, - "!V:!DrawProgressBar:AnalysisType=Classification"); // Construct training options string std::string opts = "!V:!H:NTrees=" + std::to_string(state.range(0)) + ":MaxDepth=" + std::to_string(state.range(1)); // Train a TMVA method - factory->BookMethod(dataloader, TMVA::Types::kBDT, "BDT", opts); - factory->TrainAllMethods(); - - outputFile->Close(); - delete factory; - delete outputFile; + auto method = factory->BookMethod(dataloader, TMVA::Types::kBDT, "BDT", opts); + TMVA::Event::SetIsTraining(kTRUE); + method->TrainMethod(); + TMVA::Event::SetIsTraining(kFALSE); + method->Data()->DeleteAllResults(TMVA::Types::kTraining, method->GetAnalysisType()); + + factory->DeleteAllMethods(); + factory->fMethodsMap.clear(); } // Teardown + outputFile->Close(); + delete factory; + delete outputFile; delete sigTree; delete bkgTree; } From fbfbc182339b21f973ba39320e752aa8c1cb5a9b Mon Sep 17 00:00:00 2001 From: xmif1 Date: Thu, 5 Aug 2021 13:54:32 +0200 Subject: [PATCH 05/24] Adding evaluation benchmark --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 29 +++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 40414f27d..f2ee941dd 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -5,6 +5,8 @@ #include "TRandom3.h" #include "TTree.h" +#include "TMVA/RReader.hxx" +#include "TMVA/RTensorUtils.hxx" #include "TMVA/DataLoader.h" #include "TMVA/Factory.h" #include "TMVA/MethodBase.h" @@ -15,6 +17,8 @@ #include "MakeRandomTTree.h" +using namespace TMVA::Experimental; + static void BM_TMVA_BDTTraining(benchmark::State &state){ UInt_t nVars = 4; UInt_t nEvents = 500; @@ -57,7 +61,8 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ ":MaxDepth=" + std::to_string(state.range(1)); // Train a TMVA method - auto method = factory->BookMethod(dataloader, TMVA::Types::kBDT, "BDT", opts); + std::string methodTitle = "BDT_" + std::to_string(state.range(0)) + "_" + std::to_string(state.range(1)); + auto method = factory->BookMethod(dataloader, TMVA::Types::kBDT, methodTitle, opts); TMVA::Event::SetIsTraining(kTRUE); method->TrainMethod(); TMVA::Event::SetIsTraining(kFALSE); @@ -76,4 +81,26 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ } BENCHMARK(BM_TMVA_BDTTraining)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10}}); +static void BM_TMVA_BDTTesting(benchmark::State &state){ + UInt_t nVars = 4; + UInt_t nEvents = 500; + + // Set up + TTree *testTree = genTree(nEvents, nVars,0.3, 0.5, 102); + ROOT::RDataFrame testDF(*testTree); + auto testTensor = AsTensor(testDF); + + // Benchmarking + for(auto _: state){ + // Test a TMVA method via RReader + std::string methodTitle = std::to_string(state.range(0)) + "_" + std::to_string(state.range(1)); + RReader model("bdt-bench_BDT_" + methodTitle + ".weights.xml"); + model.Compute(testTensor); + } + + // Teardown + delete testTree; +} +BENCHMARK(BM_TMVA_BDTTesting)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10}}); + BENCHMARK_MAIN(); From a985ad74aee7351518c894b0e4ec728e8699c601 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Fri, 6 Aug 2021 08:53:33 +0200 Subject: [PATCH 06/24] Fixed CMakeFile --- root/tmva/tmva/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/root/tmva/tmva/CMakeLists.txt b/root/tmva/tmva/CMakeLists.txt index a7e3da44c..e57484a32 100644 --- a/root/tmva/tmva/CMakeLists.txt +++ b/root/tmva/tmva/CMakeLists.txt @@ -7,7 +7,7 @@ if(ROOT_tmva_FOUND) RB_ADD_GBENCHMARK(BoostedDTBenchmarks BoostedDTBenchmarks.cxx LABEL short - LIBRARIES Core Tree MathCore RIO TMVA) + LIBRARIES Core Tree TreePlayer MathCore RIO XMLIO ROOTDataFrame TMVA) endif() if(ROOT_tmva_FOUND AND ROOT_tmva-cpu_FOUND AND ROOT_imt_FOUND) From b56f429b2b4e330e9ef52203ebb2382a82b31d1d Mon Sep 17 00:00:00 2001 From: xmif1 Date: Fri, 6 Aug 2021 09:43:34 +0200 Subject: [PATCH 07/24] Minor changes --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 2 +- root/tmva/tmva/MakeRandomTTree.h | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index f2ee941dd..bace62af8 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -86,7 +86,7 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ UInt_t nEvents = 500; // Set up - TTree *testTree = genTree(nEvents, nVars,0.3, 0.5, 102); + TTree *testTree = genTree(nEvents, nVars,0.3, 0.5, 102, false); ROOT::RDataFrame testDF(*testTree); auto testTensor = AsTensor(testDF); diff --git a/root/tmva/tmva/MakeRandomTTree.h b/root/tmva/tmva/MakeRandomTTree.h index 5751568f3..2696f1aa0 100644 --- a/root/tmva/tmva/MakeRandomTTree.h +++ b/root/tmva/tmva/MakeRandomTTree.h @@ -2,7 +2,8 @@ #include "TTree.h" // Utility function for generating a random TTree with Gaussian float data, for the specified number of points and vars -TTree* genTree(UInt_t nPoints, const UInt_t nVars, Double_t offset, Double_t scale = 0.3, UInt_t seed = 100){ +TTree* genTree(UInt_t nPoints, const UInt_t nVars, Double_t offset, Double_t scale = 0.3, UInt_t seed = 100, + bool evtCol = true){ // Initialisation TRandom3 rng(seed); Float_t vars[nVars]; for(auto& var: vars){ var = 0.0;} @@ -20,7 +21,9 @@ TTree* genTree(UInt_t nPoints, const UInt_t nVars, Double_t offset, Double_t sca } // And add a branch for the (unique) Event identifier - data->Branch("EventNumber", &id, "EventNumber/I"); + if(evtCol){ + data->Branch("EventNumber", &id, "EventNumber/I"); + } // Populate TTree instance with Gaussian data for(UInt_t j = 0; j < nPoints; j++){ @@ -32,7 +35,7 @@ TTree* genTree(UInt_t nPoints, const UInt_t nVars, Double_t offset, Double_t sca id++; } - // Important: Disconnects the tree from the memory locations of x and y. + // Important: Disconnects the tree from the memory locations of vars[i] data->ResetBranchAddresses(); return data; } \ No newline at end of file From ccaef5061bbb282df7dc371d8df41bbb4a805f6c Mon Sep 17 00:00:00 2001 From: xmif1 Date: Fri, 6 Aug 2021 10:08:44 +0200 Subject: [PATCH 08/24] Minor changes --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index bace62af8..69b8de715 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -94,7 +94,7 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ for(auto _: state){ // Test a TMVA method via RReader std::string methodTitle = std::to_string(state.range(0)) + "_" + std::to_string(state.range(1)); - RReader model("bdt-bench_BDT_" + methodTitle + ".weights.xml"); + RReader model("./bdr-bench/weights/bdt-bench_BDT_" + methodTitle + ".weights.xml"); model.Compute(testTensor); } From 6b2c1df43797c610830aea177c47b7d916675616 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Fri, 6 Aug 2021 11:40:59 +0200 Subject: [PATCH 09/24] Minor changes --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 69b8de715..a16f63ebc 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -94,7 +94,7 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ for(auto _: state){ // Test a TMVA method via RReader std::string methodTitle = std::to_string(state.range(0)) + "_" + std::to_string(state.range(1)); - RReader model("./bdr-bench/weights/bdt-bench_BDT_" + methodTitle + ".weights.xml"); + RReader model("./bdt-bench/weights/bdt-bench_BDT_" + methodTitle + ".weights.xml"); model.Compute(testTensor); } From 29bbdbd9adbc6062a6afc263cb8f7ced2882ecaf Mon Sep 17 00:00:00 2001 From: xmif1 Date: Fri, 6 Aug 2021 13:52:51 +0200 Subject: [PATCH 10/24] Began adding (primitive) memory benchmarking --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 73 ++++++++++++++++++++------ 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index a16f63ebc..b8d8260cd 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -1,9 +1,8 @@ /* Authored by Xandru Mifsud (CERN Summer Student) and Lorenzo Moneta (Summer Project Supervisor) */ -#include "TMVA/CrossValidation.h" - -#include "TRandom3.h" +#include "TSystem.h" #include "TTree.h" +#include "TFile.h" #include "TMVA/RReader.hxx" #include "TMVA/RTensorUtils.hxx" @@ -11,18 +10,25 @@ #include "TMVA/Factory.h" #include "TMVA/MethodBase.h" #include "TMVA/Types.h" -#include "TFile.h" #include "benchmark/benchmark.h" #include "MakeRandomTTree.h" using namespace TMVA::Experimental; +using namespace std; static void BM_TMVA_BDTTraining(benchmark::State &state){ + // Parameters UInt_t nVars = 4; UInt_t nEvents = 500; + // Memory benchmark data placeholder + auto mem_benchmarks = new multimap>; + ProcInfo_t pinfo; + Long_t curr_mem_res, temp_mem_res; curr_mem_res = temp_mem_res = 0; + Long_t curr_mem_vir, temp_mem_vir; curr_mem_vir = temp_mem_vir = 0; + // Set up (generate one extra event for testing) TTree *sigTree = genTree(nEvents+1, nVars,0.3, 0.5, 100); TTree *bkgTree = genTree(nEvents+1, nVars,-0.3, 0.5, 101); @@ -40,8 +46,8 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // Register variables in dataloader, using naming convention for randomly generated TTrees in MakeRandomTTree.h for(UInt_t i = 0; i < nVars; i++){ - std::string var_name = "var" + std::to_string(i); - std::string var_leaflist = var_name + "/F"; + string var_name = "var" + to_string(i); + string var_leaflist = var_name + "/F"; dataloader->AddVariable(var_name.c_str(), 'D'); } @@ -52,19 +58,31 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ Form("SplitMode=Block:nTrain_Signal=%i:nTrain_Background=%i:nTest_Signal=%i:nTest_Background=%i:!V", nEvents, nEvents, 1, 1)); + // Get current memory usage statistics after setup + gSystem->GetProcInfo(&pinfo); + curr_mem_res = pinfo.fMemResident; + curr_mem_vir = pinfo.fMemVirtual; + // Benchmarking for(auto _: state){ - // Create a ROOT output file where TMVA will store ntuples, histograms, etc. - // Construct training options string - std::string opts = "!V:!H:NTrees=" + std::to_string(state.range(0)) + - ":MaxDepth=" + std::to_string(state.range(1)); + string opts = "!V:!H:NTrees=" + to_string(state.range(0)) + ":MaxDepth=" + to_string(state.range(1)); // Train a TMVA method - std::string methodTitle = "BDT_" + std::to_string(state.range(0)) + "_" + std::to_string(state.range(1)); - auto method = factory->BookMethod(dataloader, TMVA::Types::kBDT, methodTitle, opts); + string key = to_string(state.range(0)) + "_" + to_string(state.range(1)); + auto method = factory->BookMethod(dataloader, TMVA::Types::kBDT, "BDT_" + key, opts); TMVA::Event::SetIsTraining(kTRUE); method->TrainMethod(); + + // Maintain Memory statistics (independent from Google Benchmark) + gSystem->GetProcInfo(&pinfo); + temp_mem_res = pinfo.fMemResident; + temp_mem_vir = pinfo.fMemVirtual; + mem_benchmarks->insert(make_pair(key, + pair(temp_mem_res - curr_mem_res, temp_mem_vir - curr_mem_vir))); + curr_mem_res = temp_mem_res; + curr_mem_vir = temp_mem_vir; + TMVA::Event::SetIsTraining(kFALSE); method->Data()->DeleteAllResults(TMVA::Types::kTraining, method->GetAnalysisType()); @@ -72,12 +90,37 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ factory->fMethodsMap.clear(); } + // Print memory benchmark statistics + std::cout << "----------------------------------------------------------------------" << std::endl; + std::cout << "Benchmark\t\t\tRes. Memory\t\tVir. Memory\tIterations" << std::endl; + std::cout << "----------------------------------------------------------------------" << std::endl; + for(auto kv = mem_benchmarks->begin(), end = mem_benchmarks->end(); kv != end; + kv = mem_benchmarks->upper_bound(kv->first)){ + + if(auto stats = mem_benchmarks->equal_range(kv->first); stats.first != stats.second){ + Int_t n_iter = mem_benchmarks->count(kv->first); + Float_t avg_mem_res = 0.0; + Float_t avg_mem_vir = 0.0; + + for(auto bm = stats.first; bm != stats.second; bm++){ + avg_mem_res += (bm->second).first; + avg_mem_vir += (bm->second).second; + } + + avg_mem_res /= n_iter; + avg_mem_vir /= n_iter; + + std::cout << "BM_TMVA_BDTTesting_" << kv->first << "\t\t\t" << avg_mem_res << "\t\t" << avg_mem_vir << "\t" << n_iter << std::endl; + } + } + // Teardown outputFile->Close(); delete factory; - delete outputFile; delete sigTree; delete bkgTree; + delete outputFile; + delete mem_benchmarks; } BENCHMARK(BM_TMVA_BDTTraining)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10}}); @@ -93,8 +136,8 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ // Benchmarking for(auto _: state){ // Test a TMVA method via RReader - std::string methodTitle = std::to_string(state.range(0)) + "_" + std::to_string(state.range(1)); - RReader model("./bdt-bench/weights/bdt-bench_BDT_" + methodTitle + ".weights.xml"); + string key = to_string(state.range(0)) + "_" + to_string(state.range(1)); + RReader model("./bdt-bench/weights/bdt-bench_BDT_" + key + ".weights.xml"); model.Compute(testTensor); } From 33cfb5bca6f356c6a62ed332d52dd627b6f243ec Mon Sep 17 00:00:00 2001 From: xmif1 Date: Mon, 9 Aug 2021 13:30:30 +0200 Subject: [PATCH 11/24] Trying new memory benchmark technique --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 75 ++++++++++++++------------ 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index b8d8260cd..8d9a75a12 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -26,8 +26,9 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // Memory benchmark data placeholder auto mem_benchmarks = new multimap>; ProcInfo_t pinfo; - Long_t curr_mem_res, temp_mem_res; curr_mem_res = temp_mem_res = 0; - Long_t curr_mem_vir, temp_mem_vir; curr_mem_vir = temp_mem_vir = 0; + Long_t init_mem_res, term_mem_res; init_mem_res = term_mem_res = 0; + Long_t init_mem_vir, term_mem_vir; init_mem_vir = term_mem_vir = 0; + double mem_res, mem_vir; mem_res = mem_vir = 0.0; // Set up (generate one extra event for testing) TTree *sigTree = genTree(nEvents+1, nVars,0.3, 0.5, 100); @@ -58,13 +59,13 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ Form("SplitMode=Block:nTrain_Signal=%i:nTrain_Background=%i:nTest_Signal=%i:nTest_Background=%i:!V", nEvents, nEvents, 1, 1)); - // Get current memory usage statistics after setup - gSystem->GetProcInfo(&pinfo); - curr_mem_res = pinfo.fMemResident; - curr_mem_vir = pinfo.fMemVirtual; - // Benchmarking for(auto _: state){ + // Get current memory usage statistics after setup + gSystem->GetProcInfo(&pinfo); + init_mem_res = pinfo.fMemResident; + init_mem_vir = pinfo.fMemVirtual; + // Construct training options string string opts = "!V:!H:NTrees=" + to_string(state.range(0)) + ":MaxDepth=" + to_string(state.range(1)); @@ -76,12 +77,12 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // Maintain Memory statistics (independent from Google Benchmark) gSystem->GetProcInfo(&pinfo); - temp_mem_res = pinfo.fMemResident; - temp_mem_vir = pinfo.fMemVirtual; - mem_benchmarks->insert(make_pair(key, - pair(temp_mem_res - curr_mem_res, temp_mem_vir - curr_mem_vir))); - curr_mem_res = temp_mem_res; - curr_mem_vir = temp_mem_vir; + term_mem_res = pinfo.fMemResident; + term_mem_vir = pinfo.fMemVirtual; + mem_res += (double) (term_mem_res - init_mem_res); + mem_vir += (double) (term_mem_vir - init_mem_vir); +// mem_benchmarks->insert(make_pair(key, +// pair(term_mem_res - init_mem_res, term_mem_vir - init_mem_vir))); TMVA::Event::SetIsTraining(kFALSE); method->Data()->DeleteAllResults(TMVA::Types::kTraining, method->GetAnalysisType()); @@ -90,29 +91,33 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ factory->fMethodsMap.clear(); } + state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); + state.counters["Virtual Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); + // Print memory benchmark statistics - std::cout << "----------------------------------------------------------------------" << std::endl; - std::cout << "Benchmark\t\t\tRes. Memory\t\tVir. Memory\tIterations" << std::endl; - std::cout << "----------------------------------------------------------------------" << std::endl; - for(auto kv = mem_benchmarks->begin(), end = mem_benchmarks->end(); kv != end; - kv = mem_benchmarks->upper_bound(kv->first)){ - - if(auto stats = mem_benchmarks->equal_range(kv->first); stats.first != stats.second){ - Int_t n_iter = mem_benchmarks->count(kv->first); - Float_t avg_mem_res = 0.0; - Float_t avg_mem_vir = 0.0; - - for(auto bm = stats.first; bm != stats.second; bm++){ - avg_mem_res += (bm->second).first; - avg_mem_vir += (bm->second).second; - } - - avg_mem_res /= n_iter; - avg_mem_vir /= n_iter; - - std::cout << "BM_TMVA_BDTTesting_" << kv->first << "\t\t\t" << avg_mem_res << "\t\t" << avg_mem_vir << "\t" << n_iter << std::endl; - } - } +// std::cout << "----------------------------------------------------------------------" << std::endl; +// std::cout << "Benchmark\t\tRes. Memory\tVir. Memory\tIterations" << std::endl; +// std::cout << "----------------------------------------------------------------------" << std::endl; +// for(auto kv = mem_benchmarks->begin(), end = mem_benchmarks->end(); kv != end; +// kv = mem_benchmarks->upper_bound(kv->first)){ +// +// if(auto stats = mem_benchmarks->equal_range(kv->first); stats.first != stats.second){ +// +// Int_t n_iter = mem_benchmarks->count(kv->first); +// Float_t avg_mem_res = 0.0; +// Float_t avg_mem_vir = 0.0; +// +// for(auto bm = stats.first; bm != stats.second; bm++){ +// avg_mem_res += (bm->second).first; +// avg_mem_vir += (bm->second).second; +// } +// +// avg_mem_res /= n_iter; +// avg_mem_vir /= n_iter; +// +// std::cout << "BM_TMVA_BDTTesting_" << kv->first << "\t\t" << avg_mem_res << "\t" << avg_mem_vir << "\t" << n_iter << std::endl; +// } +// } // Teardown outputFile->Close(); From 7a436e7fabff462957e2926677b027d1d77d2bd7 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Thu, 12 Aug 2021 09:39:21 +0200 Subject: [PATCH 12/24] Continuing trying to see if there is unreleased memory --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 49 +++++--------------------- 1 file changed, 9 insertions(+), 40 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 8d9a75a12..6896803f0 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -27,18 +27,15 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ auto mem_benchmarks = new multimap>; ProcInfo_t pinfo; Long_t init_mem_res, term_mem_res; init_mem_res = term_mem_res = 0; - Long_t init_mem_vir, term_mem_vir; init_mem_vir = term_mem_vir = 0; - double mem_res, mem_vir; mem_res = mem_vir = 0.0; + double mem_res = 0.0; // Set up (generate one extra event for testing) TTree *sigTree = genTree(nEvents+1, nVars,0.3, 0.5, 100); TTree *bkgTree = genTree(nEvents+1, nVars,-0.3, 0.5, 101); - // one could maybe avoid TMVA to create output file by passing a nullptr to the Factory constructor + // Open output file TString outfileName( "bdt_bench_output.root" ); TFile* outputFile = TFile::Open(outfileName, "RECREATE"); - auto factory = new TMVA::Factory("bdt-bench", outputFile, - "Silent:!DrawProgressBar:AnalysisType=Classification"); // Prepare a DataLoader instance, registering the signal and background TTrees auto *dataloader = new TMVA::DataLoader("bdt-bench"); @@ -53,18 +50,19 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ dataloader->AddVariable(var_name.c_str(), 'D'); } - // We add the single test event to avoid the printing of any errors from TMVA. // For each benchmark we specifically ignore this test event such that we exclusively benchmark training. dataloader->PrepareTrainingAndTestTree("", - Form("SplitMode=Block:nTrain_Signal=%i:nTrain_Background=%i:nTest_Signal=%i:nTest_Background=%i:!V", - nEvents, nEvents, 1, 1)); + Form("SplitMode=Block:nTrain_Signal=%i:nTrain_Background=%i:!V", nEvents, nEvents)); // Benchmarking for(auto _: state){ + // Create factory instance + auto factory = new TMVA::Factory("bdt-bench", outputFile, + "Silent:!DrawProgressBar:AnalysisType=Classification"); + // Get current memory usage statistics after setup gSystem->GetProcInfo(&pinfo); init_mem_res = pinfo.fMemResident; - init_mem_vir = pinfo.fMemVirtual; // Construct training options string string opts = "!V:!H:NTrees=" + to_string(state.range(0)) + ":MaxDepth=" + to_string(state.range(1)); @@ -78,50 +76,21 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // Maintain Memory statistics (independent from Google Benchmark) gSystem->GetProcInfo(&pinfo); term_mem_res = pinfo.fMemResident; - term_mem_vir = pinfo.fMemVirtual; mem_res += (double) (term_mem_res - init_mem_res); - mem_vir += (double) (term_mem_vir - init_mem_vir); -// mem_benchmarks->insert(make_pair(key, -// pair(term_mem_res - init_mem_res, term_mem_vir - init_mem_vir))); TMVA::Event::SetIsTraining(kFALSE); method->Data()->DeleteAllResults(TMVA::Types::kTraining, method->GetAnalysisType()); + // Destroy factory entirely factory->DeleteAllMethods(); factory->fMethodsMap.clear(); + delete factory; } state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); - state.counters["Virtual Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); - - // Print memory benchmark statistics -// std::cout << "----------------------------------------------------------------------" << std::endl; -// std::cout << "Benchmark\t\tRes. Memory\tVir. Memory\tIterations" << std::endl; -// std::cout << "----------------------------------------------------------------------" << std::endl; -// for(auto kv = mem_benchmarks->begin(), end = mem_benchmarks->end(); kv != end; -// kv = mem_benchmarks->upper_bound(kv->first)){ -// -// if(auto stats = mem_benchmarks->equal_range(kv->first); stats.first != stats.second){ -// -// Int_t n_iter = mem_benchmarks->count(kv->first); -// Float_t avg_mem_res = 0.0; -// Float_t avg_mem_vir = 0.0; -// -// for(auto bm = stats.first; bm != stats.second; bm++){ -// avg_mem_res += (bm->second).first; -// avg_mem_vir += (bm->second).second; -// } -// -// avg_mem_res /= n_iter; -// avg_mem_vir /= n_iter; -// -// std::cout << "BM_TMVA_BDTTesting_" << kv->first << "\t\t" << avg_mem_res << "\t" << avg_mem_vir << "\t" << n_iter << std::endl; -// } -// } // Teardown outputFile->Close(); - delete factory; delete sigTree; delete bkgTree; delete outputFile; From 563668bd5c583e05eb04d87b8e70047427fab76a Mon Sep 17 00:00:00 2001 From: xmif1 Date: Thu, 12 Aug 2021 09:56:04 +0200 Subject: [PATCH 13/24] Log --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 6896803f0..667386511 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -85,6 +85,9 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ factory->DeleteAllMethods(); factory->fMethodsMap.clear(); delete factory; + + // DEBUG + cout << "[DEBUG] " << key << ": res_mem_init = " << (double) init_mem_res << ", res_mem_term = " << (double) term_mem_res << endl; } state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); From 069a9c50bc9ee35a1362291e0ad337120430e6fb Mon Sep 17 00:00:00 2001 From: xmif1 Date: Thu, 12 Aug 2021 11:22:37 +0200 Subject: [PATCH 14/24] Extending memory benchmarks --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 667386511..da72e72d3 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -24,7 +24,6 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ UInt_t nEvents = 500; // Memory benchmark data placeholder - auto mem_benchmarks = new multimap>; ProcInfo_t pinfo; Long_t init_mem_res, term_mem_res; init_mem_res = term_mem_res = 0; double mem_res = 0.0; @@ -87,7 +86,7 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ delete factory; // DEBUG - cout << "[DEBUG] " << key << ": res_mem_init = " << (double) init_mem_res << ", res_mem_term = " << (double) term_mem_res << endl; + // cout << "[DEBUG] " << key << ": res_mem_init = " << (double) init_mem_res << ", res_mem_term = " << (double) term_mem_res << endl; } state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); @@ -97,7 +96,6 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ delete sigTree; delete bkgTree; delete outputFile; - delete mem_benchmarks; } BENCHMARK(BM_TMVA_BDTTraining)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10}}); @@ -105,6 +103,11 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ UInt_t nVars = 4; UInt_t nEvents = 500; + // Memory benchmark data placeholder + ProcInfo_t pinfo; + Long_t init_mem_res, term_mem_res; init_mem_res = term_mem_res = 0; + double mem_res = 0.0; + // Set up TTree *testTree = genTree(nEvents, nVars,0.3, 0.5, 102, false); ROOT::RDataFrame testDF(*testTree); @@ -115,11 +118,24 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ // Test a TMVA method via RReader string key = to_string(state.range(0)) + "_" + to_string(state.range(1)); RReader model("./bdt-bench/weights/bdt-bench_BDT_" + key + ".weights.xml"); + + // Get current memory usage statistics after setup + gSystem->GetProcInfo(&pinfo); + init_mem_res = pinfo.fMemResident; + model.Compute(testTensor); + + // Maintain Memory statistics (independent from Google Benchmark) + gSystem->GetProcInfo(&pinfo); + term_mem_res = pinfo.fMemResident; + mem_res += (double) (term_mem_res - init_mem_res); } + state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); + // Teardown delete testTree; + delete &testTensor; } BENCHMARK(BM_TMVA_BDTTesting)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10}}); From f4cd764fd78aac6885696e819a7d63e9ebe33c60 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Thu, 12 Aug 2021 11:33:40 +0200 Subject: [PATCH 15/24] Bug fix --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 1 - 1 file changed, 1 deletion(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index da72e72d3..2dc878bca 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -135,7 +135,6 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ // Teardown delete testTree; - delete &testTensor; } BENCHMARK(BM_TMVA_BDTTesting)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10}}); From e5771ffa28235eb8b2460781f3a2d9c0bb706ed0 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Thu, 12 Aug 2021 11:41:57 +0200 Subject: [PATCH 16/24] Taking memory measure _before_ model is loaded --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 2dc878bca..6da0d757b 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -117,12 +117,12 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ for(auto _: state){ // Test a TMVA method via RReader string key = to_string(state.range(0)) + "_" + to_string(state.range(1)); - RReader model("./bdt-bench/weights/bdt-bench_BDT_" + key + ".weights.xml"); // Get current memory usage statistics after setup gSystem->GetProcInfo(&pinfo); init_mem_res = pinfo.fMemResident; + RReader model("./bdt-bench/weights/bdt-bench_BDT_" + key + ".weights.xml"); model.Compute(testTensor); // Maintain Memory statistics (independent from Google Benchmark) From adfc511f83f25577eb476aadbf71483aeefff46c Mon Sep 17 00:00:00 2001 From: xmif1 Date: Mon, 16 Aug 2021 07:58:48 +0200 Subject: [PATCH 17/24] Trying multithreading benchmark as a single bench rather then multiple ones --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 6da0d757b..1a9d938d2 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -55,6 +55,8 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // Benchmarking for(auto _: state){ + ROOT::EnableImplicitMT(state.range(2)); + // Create factory instance auto factory = new TMVA::Factory("bdt-bench", outputFile, "Silent:!DrawProgressBar:AnalysisType=Classification"); @@ -67,7 +69,7 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ string opts = "!V:!H:NTrees=" + to_string(state.range(0)) + ":MaxDepth=" + to_string(state.range(1)); // Train a TMVA method - string key = to_string(state.range(0)) + "_" + to_string(state.range(1)); + string key = to_string(state.range(0)) + "_" + to_string(state.range(1)) + "_" + to_string(state.range(2)); auto method = factory->BookMethod(dataloader, TMVA::Types::kBDT, "BDT_" + key, opts); TMVA::Event::SetIsTraining(kTRUE); method->TrainMethod(); @@ -97,7 +99,7 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ delete bkgTree; delete outputFile; } -BENCHMARK(BM_TMVA_BDTTraining)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10}}); +BENCHMARK(BM_TMVA_BDTTraining)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10},{1,4,8,16}}); static void BM_TMVA_BDTTesting(benchmark::State &state){ UInt_t nVars = 4; From 7ef49c74f6d3557d4549071d52bea001101ddea6 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Mon, 16 Aug 2021 08:28:42 +0200 Subject: [PATCH 18/24] Trying multithreading benchmark as a single bench rather then multiple ones --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 1a9d938d2..4a71c2916 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -117,8 +117,10 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ // Benchmarking for(auto _: state){ + ROOT::EnableImplicitMT(state.range(2)); + // Test a TMVA method via RReader - string key = to_string(state.range(0)) + "_" + to_string(state.range(1)); + string key = to_string(state.range(0)) + "_" + to_string(state.range(1)) + "_" + to_string(state.range(2)); // Get current memory usage statistics after setup gSystem->GetProcInfo(&pinfo); @@ -138,6 +140,6 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ // Teardown delete testTree; } -BENCHMARK(BM_TMVA_BDTTesting)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10}}); +BENCHMARK(BM_TMVA_BDTTesting)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10},{1,4,8,16}}); BENCHMARK_MAIN(); From a64a1ab191f86f4a60b32e109cf07a11d7e0280a Mon Sep 17 00:00:00 2001 From: xmif1 Date: Mon, 16 Aug 2021 10:57:54 +0200 Subject: [PATCH 19/24] Bug fix --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 4a71c2916..e1100e72a 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -28,14 +28,14 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ Long_t init_mem_res, term_mem_res; init_mem_res = term_mem_res = 0; double mem_res = 0.0; - // Set up (generate one extra event for testing) - TTree *sigTree = genTree(nEvents+1, nVars,0.3, 0.5, 100); - TTree *bkgTree = genTree(nEvents+1, nVars,-0.3, 0.5, 101); - // Open output file - TString outfileName( "bdt_bench_output.root" ); + TString outfileName( "bdt_bench_train_output.root" ); TFile* outputFile = TFile::Open(outfileName, "RECREATE"); + // Set up (generate one extra event for testing) + TTree *sigTree = genTree(nEvents, nVars,0.3, 0.5, 100); + TTree *bkgTree = genTree(nEvents, nVars,-0.3, 0.5, 101); + // Prepare a DataLoader instance, registering the signal and background TTrees auto *dataloader = new TMVA::DataLoader("bdt-bench"); dataloader->AddSignalTree(sigTree); @@ -110,6 +110,10 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ Long_t init_mem_res, term_mem_res; init_mem_res = term_mem_res = 0; double mem_res = 0.0; + // Open output file + TString outfileName( "bdt_bench_train_output.root" ); + TFile* outputFile = TFile::Open(outfileName, "RECREATE"); + // Set up TTree *testTree = genTree(nEvents, nVars,0.3, 0.5, 102, false); ROOT::RDataFrame testDF(*testTree); @@ -138,6 +142,7 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); // Teardown + outputFile->Close(); delete testTree; } BENCHMARK(BM_TMVA_BDTTesting)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10},{1,4,8,16}}); From 63b82805409eb2a6f35b0ace8dbe372c45c74cb4 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Tue, 17 Aug 2021 10:15:28 +0200 Subject: [PATCH 20/24] Bug fix --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 7 ++++++- root/tmva/tmva/MakeRandomTTree.h | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index e1100e72a..3db373ac5 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -115,8 +115,13 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ TFile* outputFile = TFile::Open(outfileName, "RECREATE"); // Set up + auto inputFile = new TFile("bdt_bench_train_input.root","RECREATE"); TTree *testTree = genTree(nEvents, nVars,0.3, 0.5, 102, false); - ROOT::RDataFrame testDF(*testTree); + testTree->Write(); + inputFile->Close(); + delete inputFile; + + ROOT::RDataFrame testDF("tree","bdt_bench_train_input.root"); auto testTensor = AsTensor(testDF); // Benchmarking diff --git a/root/tmva/tmva/MakeRandomTTree.h b/root/tmva/tmva/MakeRandomTTree.h index 2696f1aa0..3a669676b 100644 --- a/root/tmva/tmva/MakeRandomTTree.h +++ b/root/tmva/tmva/MakeRandomTTree.h @@ -10,7 +10,7 @@ TTree* genTree(UInt_t nPoints, const UInt_t nVars, Double_t offset, Double_t sca UInt_t id = 0; // Create new TTree instance - auto data = new TTree(); + auto data = new TTree("tree","tree"); // Add a branch corresponding to each variable for(UInt_t i = 0; i < nVars; i++){ From a73b063c3ff3ed770cffd7c6e14d144c081ac21d Mon Sep 17 00:00:00 2001 From: xmif1 Date: Tue, 17 Aug 2021 11:08:31 +0200 Subject: [PATCH 21/24] Bug fix --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 12 ++++++------ root/tmva/tmva/MakeRandomTTree.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 3db373ac5..cc813e919 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -33,8 +33,8 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ TFile* outputFile = TFile::Open(outfileName, "RECREATE"); // Set up (generate one extra event for testing) - TTree *sigTree = genTree(nEvents, nVars,0.3, 0.5, 100); - TTree *bkgTree = genTree(nEvents, nVars,-0.3, 0.5, 101); + TTree *sigTree = genTree("sigTree", nEvents, nVars,0.3, 0.5, 100); + TTree *bkgTree = genTree("bkgTree", nEvents, nVars,-0.3, 0.5, 101); // Prepare a DataLoader instance, registering the signal and background TTrees auto *dataloader = new TMVA::DataLoader("bdt-bench"); @@ -111,17 +111,17 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ double mem_res = 0.0; // Open output file - TString outfileName( "bdt_bench_train_output.root" ); + TString outfileName( "bdt_bench_test_output.root" ); TFile* outputFile = TFile::Open(outfileName, "RECREATE"); // Set up - auto inputFile = new TFile("bdt_bench_train_input.root","RECREATE"); - TTree *testTree = genTree(nEvents, nVars,0.3, 0.5, 102, false); + auto inputFile = new TFile("bdt_bench_test_input.root","RECREATE"); + TTree *testTree = genTree("testTree", nEvents, nVars,0.3, 0.5, 102, false); testTree->Write(); inputFile->Close(); delete inputFile; - ROOT::RDataFrame testDF("tree","bdt_bench_train_input.root"); + ROOT::RDataFrame testDF("tree","bdt_bench_test_input.root"); auto testTensor = AsTensor(testDF); // Benchmarking diff --git a/root/tmva/tmva/MakeRandomTTree.h b/root/tmva/tmva/MakeRandomTTree.h index 3a669676b..59b234834 100644 --- a/root/tmva/tmva/MakeRandomTTree.h +++ b/root/tmva/tmva/MakeRandomTTree.h @@ -2,7 +2,7 @@ #include "TTree.h" // Utility function for generating a random TTree with Gaussian float data, for the specified number of points and vars -TTree* genTree(UInt_t nPoints, const UInt_t nVars, Double_t offset, Double_t scale = 0.3, UInt_t seed = 100, +TTree* genTree(std::string name, UInt_t nPoints, const UInt_t nVars, Double_t offset, Double_t scale = 0.3, UInt_t seed = 100, bool evtCol = true){ // Initialisation TRandom3 rng(seed); @@ -10,7 +10,7 @@ TTree* genTree(UInt_t nPoints, const UInt_t nVars, Double_t offset, Double_t sca UInt_t id = 0; // Create new TTree instance - auto data = new TTree("tree","tree"); + auto data = new TTree(name.c_str(),name.c_str()); // Add a branch corresponding to each variable for(UInt_t i = 0; i < nVars; i++){ From d64d7104444f05c2d45e8fb713415336f8d065e2 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Mon, 23 Aug 2021 09:27:47 +0200 Subject: [PATCH 22/24] Bug fix (thanks to @lmoneta) + added option to turn of memory statistics. --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 46 +++++++++++++++++--------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index cc813e919..3d2ca043a 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -22,6 +22,7 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // Parameters UInt_t nVars = 4; UInt_t nEvents = 500; + Bool_t mem_stats = false; // Memory benchmark data placeholder ProcInfo_t pinfo; @@ -62,8 +63,10 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ "Silent:!DrawProgressBar:AnalysisType=Classification"); // Get current memory usage statistics after setup - gSystem->GetProcInfo(&pinfo); - init_mem_res = pinfo.fMemResident; + if(mem_stats){ + gSystem->GetProcInfo(&pinfo); + init_mem_res = pinfo.fMemResident; + } // Construct training options string string opts = "!V:!H:NTrees=" + to_string(state.range(0)) + ":MaxDepth=" + to_string(state.range(1)); @@ -75,9 +78,11 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ method->TrainMethod(); // Maintain Memory statistics (independent from Google Benchmark) - gSystem->GetProcInfo(&pinfo); - term_mem_res = pinfo.fMemResident; - mem_res += (double) (term_mem_res - init_mem_res); + if(mem_stats){ + gSystem->GetProcInfo(&pinfo); + term_mem_res = pinfo.fMemResident; + mem_res += (double) (term_mem_res - init_mem_res); + } TMVA::Event::SetIsTraining(kFALSE); method->Data()->DeleteAllResults(TMVA::Types::kTraining, method->GetAnalysisType()); @@ -91,19 +96,24 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // cout << "[DEBUG] " << key << ": res_mem_init = " << (double) init_mem_res << ", res_mem_term = " << (double) term_mem_res << endl; } - state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); + if(mem_stats){ + state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); + } // Teardown - outputFile->Close(); delete sigTree; delete bkgTree; + + outputFile->Close(); delete outputFile; } BENCHMARK(BM_TMVA_BDTTraining)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10},{1,4,8,16}}); static void BM_TMVA_BDTTesting(benchmark::State &state){ + // Parameters UInt_t nVars = 4; UInt_t nEvents = 500; + Bool_t mem_stats = true; // Memory benchmark data placeholder ProcInfo_t pinfo; @@ -118,10 +128,11 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ auto inputFile = new TFile("bdt_bench_test_input.root","RECREATE"); TTree *testTree = genTree("testTree", nEvents, nVars,0.3, 0.5, 102, false); testTree->Write(); + delete testTree; inputFile->Close(); delete inputFile; - ROOT::RDataFrame testDF("tree","bdt_bench_test_input.root"); + ROOT::RDataFrame testDF("testTree","bdt_bench_test_input.root"); auto testTensor = AsTensor(testDF); // Benchmarking @@ -132,23 +143,28 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ string key = to_string(state.range(0)) + "_" + to_string(state.range(1)) + "_" + to_string(state.range(2)); // Get current memory usage statistics after setup - gSystem->GetProcInfo(&pinfo); - init_mem_res = pinfo.fMemResident; + if(mem_stats){ + gSystem->GetProcInfo(&pinfo); + init_mem_res = pinfo.fMemResident; + } RReader model("./bdt-bench/weights/bdt-bench_BDT_" + key + ".weights.xml"); model.Compute(testTensor); // Maintain Memory statistics (independent from Google Benchmark) - gSystem->GetProcInfo(&pinfo); - term_mem_res = pinfo.fMemResident; - mem_res += (double) (term_mem_res - init_mem_res); + if(mem_stats){ + gSystem->GetProcInfo(&pinfo); + term_mem_res = pinfo.fMemResident; + mem_res += (double) (term_mem_res - init_mem_res); + } } - state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); + if(mem_stats){ + state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); + } // Teardown outputFile->Close(); - delete testTree; } BENCHMARK(BM_TMVA_BDTTesting)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10},{1,4,8,16}}); From 49c226ccc9af9a8be2b26f464a4f6ab2bca9f8f9 Mon Sep 17 00:00:00 2001 From: xmif1 Date: Mon, 23 Aug 2021 11:42:29 +0200 Subject: [PATCH 23/24] Updated to take memory measurement only on first iteration of gbench. --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index 3d2ca043a..f87bab189 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -22,7 +22,7 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // Parameters UInt_t nVars = 4; UInt_t nEvents = 500; - Bool_t mem_stats = false; + Bool_t mem_stats = true; // Memory benchmark data placeholder ProcInfo_t pinfo; @@ -55,6 +55,7 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ Form("SplitMode=Block:nTrain_Signal=%i:nTrain_Background=%i:!V", nEvents, nEvents)); // Benchmarking + UInt_t iter_c = 0; for(auto _: state){ ROOT::EnableImplicitMT(state.range(2)); @@ -63,7 +64,7 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ "Silent:!DrawProgressBar:AnalysisType=Classification"); // Get current memory usage statistics after setup - if(mem_stats){ + if(mem_stats && iter_c == 0){ gSystem->GetProcInfo(&pinfo); init_mem_res = pinfo.fMemResident; } @@ -78,7 +79,7 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ method->TrainMethod(); // Maintain Memory statistics (independent from Google Benchmark) - if(mem_stats){ + if(mem_stats && iter_c == 0){ gSystem->GetProcInfo(&pinfo); term_mem_res = pinfo.fMemResident; mem_res += (double) (term_mem_res - init_mem_res); @@ -94,9 +95,12 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // DEBUG // cout << "[DEBUG] " << key << ": res_mem_init = " << (double) init_mem_res << ", res_mem_term = " << (double) term_mem_res << endl; + + iter_c++; } if(mem_stats){ + mem_res *= iter_c; state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); } @@ -136,6 +140,7 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ auto testTensor = AsTensor(testDF); // Benchmarking + UInt_t iter_c = 0; for(auto _: state){ ROOT::EnableImplicitMT(state.range(2)); @@ -143,7 +148,7 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ string key = to_string(state.range(0)) + "_" + to_string(state.range(1)) + "_" + to_string(state.range(2)); // Get current memory usage statistics after setup - if(mem_stats){ + if(mem_stats && iter_c == 0){ gSystem->GetProcInfo(&pinfo); init_mem_res = pinfo.fMemResident; } @@ -152,14 +157,17 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ model.Compute(testTensor); // Maintain Memory statistics (independent from Google Benchmark) - if(mem_stats){ + if(mem_stats && iter_c == 0){ gSystem->GetProcInfo(&pinfo); term_mem_res = pinfo.fMemResident; mem_res += (double) (term_mem_res - init_mem_res); } + + iter_c++; } if(mem_stats){ + mem_res *= iter_c; state.counters["Resident Memory"] = benchmark::Counter(mem_res, benchmark::Counter::kAvgIterations); } From 89e7a100dfebcdf23de983b2c0b68528f6d3ef4c Mon Sep 17 00:00:00 2001 From: xmif1 Date: Mon, 23 Aug 2021 12:27:10 +0200 Subject: [PATCH 24/24] Updated to take memory measurement only on first iteration of gbench. --- root/tmva/tmva/BoostedDTBenchmarks.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/root/tmva/tmva/BoostedDTBenchmarks.cxx b/root/tmva/tmva/BoostedDTBenchmarks.cxx index f87bab189..4e772e85d 100644 --- a/root/tmva/tmva/BoostedDTBenchmarks.cxx +++ b/root/tmva/tmva/BoostedDTBenchmarks.cxx @@ -22,7 +22,7 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ // Parameters UInt_t nVars = 4; UInt_t nEvents = 500; - Bool_t mem_stats = true; + Bool_t mem_stats = (state.range(0) == 2000) && (state.range(1) == 10) && (state.range(2) == 1); // Memory benchmark data placeholder ProcInfo_t pinfo; @@ -111,13 +111,13 @@ static void BM_TMVA_BDTTraining(benchmark::State &state){ outputFile->Close(); delete outputFile; } -BENCHMARK(BM_TMVA_BDTTraining)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10},{1,4,8,16}}); +BENCHMARK(BM_TMVA_BDTTraining)->ArgsProduct({{2000, 1000, 400, 100}, {10, 8, 6, 4, 2}, {1, 4, 8, 16}}); static void BM_TMVA_BDTTesting(benchmark::State &state){ // Parameters UInt_t nVars = 4; UInt_t nEvents = 500; - Bool_t mem_stats = true; + Bool_t mem_stats = (state.range(0) == 2000) && (state.range(1) == 10) && (state.range(2) == 1); // Memory benchmark data placeholder ProcInfo_t pinfo; @@ -174,6 +174,6 @@ static void BM_TMVA_BDTTesting(benchmark::State &state){ // Teardown outputFile->Close(); } -BENCHMARK(BM_TMVA_BDTTesting)->ArgsProduct({{100, 400, 1000, 2000},{2, 4, 6, 8, 10},{1,4,8,16}}); +BENCHMARK(BM_TMVA_BDTTesting)->ArgsProduct({{2000, 1000, 400, 100}, {10, 8, 6, 4, 2}, {1, 4, 8, 16}}); BENCHMARK_MAIN();