[WIP] Flamegraph generation for rootbench.git

CMakeLists.txt

@@ -13,11 +13,24 @@ set(CMAKE_MODULE_PATH
   "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules"
   )
 
-#Define ROOTbench source tree
+#---Define ROOTbench source tree
 set(ROOTBENCH_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
+#---Include rootbench options
+include(RootBenchOptions)
+
 include(AddRootBench)
 
+#---ROOTbench dependencies
+find_program(TIME_EXECUTABLE time)
+# Testing valid output of /usr/bin/time -v
+exec_program(${TIME_EXECUTABLE} ARGS -v ${CMAKE_COMMAND} -E environment OUTPUT_VARIABLE TIME_OUTPUT RETURN_VALUE TIME_EXECUTABLE_VALID)
+if(NOT TIME_EXECUTABLE)
+    if(NOT TIME_EXECUTABLE_VALID)
+      message(FATAL_ERROR "/usr/bin/time is a requirement for rootbench.git")
+    endif()
+endif()
+
 # You need first to tell CMake where to find the ROOT installation. This can either be the
 # final ROOT installation or a local build directory. In both cases it is using
 # the $ROOTSYS environment variable to locate it.
@@ -50,14 +63,22 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 include(GoogleBenchmark)
+if(flamegraphCPU OR flamegraphMem)
+  # Check if perf is available in OS:
+  find_program(PERF_EXECUTABLE perf)
+  if(NOT PERF_EXECUTABLE)
+    message(WARNING "Perf is not available in your system, please install it.")
+    set(flamegraph OFF CACHE BOOL "")
+  else()
+    include(FlameGraph)
+    add_custom_target(flamegraph-download DEPENDS FlameGraph)
+  endif()
+endif()
 
 #---Add ROOT include direcories and used compilation flags
 include_directories(${ROOT_INCLUDE_DIRS})
 add_definitions(${ROOT_CXX_FLAGS})
 
-#---Include rootbench options
-include(RootBenchOptions)
-
 #---Enable test coverage -----------------------------------------------------------------------
 if(coverage)
   set(GCC_COVERAGE_COMPILE_FLAGS "-g -fprofile-arcs -ftest-coverage")
@@ -76,3 +97,4 @@ add_subdirectory(lib)
 
 #---Add the now all the benchmark sub-directories on this repository
 add_subdirectory(root)
+configure_file(${PROJECT_SOURCE_DIR}/rootbench-scripts/.rootrc ${PROJECT_BINARY_DIR} COPYONLY)

cmake/modules/AddRootBench.cmake

@@ -1,8 +1,8 @@
 #----------------------------------------------------------------------------
-# function RB_ADD_GBENCHMARK(<benchmark> source1 source2... LIBRARIES libs)
+# function RB_ADD_GBENCHMARK(<benchmark> source1 source2... LIBRARIES libs POSTCMD cmd)
 #----------------------------------------------------------------------------
 function(RB_ADD_GBENCHMARK benchmark)
-  cmake_parse_arguments(ARG "" "" "LABEL;LIBRARIES" ${ARGN})
+  cmake_parse_arguments(ARG "" "" "LABEL;LIBRARIES;POSTCMD" ${ARGN})
   # FIXME: Move to target_include_directories.
   include_directories(BEFORE ${ROOTBENCH_SOURCE_DIR}/include)
   include_directories(${CMAKE_CURRENT_BINARY_DIR} ${GBENCHMARK_INCLUDE_DIR})
@@ -16,19 +16,31 @@ function(RB_ADD_GBENCHMARK benchmark)
   # to implement because some ROOT components create more than one library.
   target_link_libraries(${benchmark} ${ARG_LIBRARIES} gbenchmark RBSupport)
   #ROOT_PATH_TO_STRING(mangled_name ${benchmark} PATH_SEPARATOR_REPLACEMENT "-")
-  #ROOT_ADD_TEST(gbench${mangled_name}
-  #  COMMAND ${benchmark}
-  #  WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}
-  #  LABELS "benchmark")
+  if(ARG_POSTCMD)
+    set(postcmd POSTCMD ${ARG_POSTCMD})
+  endif()
+  if(flamegraphCPU)
+      set(postcmd ${postcmd} "${PROJECT_SOURCE_DIR}/rootbench-scripts/flamegraph.sh -d ${PROJECT_BINARY_DIR} -b ${CMAKE_CURRENT_BINARY_DIR}/${benchmark} -c")
+      add_dependencies(${benchmark} flamegraph-download)
+  endif()
+  if(flamegraphMem)
+      set(postcmd ${postcmd} "${PROJECT_SOURCE_DIR}/rootbench-scripts/flamegraph.sh -d ${PROJECT_BINARY_DIR} -b ${CMAKE_CURRENT_BINARY_DIR}/${benchmark} -m")
+      add_dependencies(${benchmark} flamegraph-download)
+  endif()
   if(${ARG_LABEL} STREQUAL "long")
     set(${TIMEOUT_VALUE} 1200)
   elseif($ARG_LABEL STREQUAL "short")
     set(${TIMEOUT_VALUE} 3600)
   endif()
-  add_test(NAME rootbench-${benchmark} COMMAND ${benchmark} --benchmark_out_format=csv --benchmark_out=rootbench-${benchmark}.csv --benchmark_color=false)
-  set_tests_properties(rootbench-${benchmark} PROPERTIES
-                                              ENVIRONMENT LD_LIBRARY_PATH=${ROOT_LIBRARY_DIR}:$ENV{LD_LIBRARY_PATH}
-                                              TIMEOUT "${TIMEOUT_VALUE}" LABELS "${ARG_LABEL}" RUN_SERIAL TRUE)
+  ROOT_ADD_TEST(rootbench-${benchmark}
+    COMMAND ${benchmark} --benchmark_out_format=csv --benchmark_out=rootbench-${benchmark}.csv --benchmark_color=false
+    WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}
+    POSTCMD ${postcmd}
+    ENVIRONMENT LD_LIBRARY_PATH=${ROOT_LIBRARY_DIR}:$ENV{LD_LIBRARY_PATH}
+    TIMEOUT "${TIMEOUT_VALUE}"
+    LABELS "${ARG_LABEL}"
+    RUN_SERIAL TRUE
+    )
 endfunction(RB_ADD_GBENCHMARK)
 
 #----------------------------------------------------------------------------

cmake/modules/FlameGraph.cmake

@@ -0,0 +1,13 @@
+include(ExternalProject)
+
+ExternalProject_Add(FlameGraph
+	GIT_REPOSITORY "https://github.com/brendangregg/FlameGraph.git"
+	UPDATE_COMMAND ""
+	PATCH_COMMAND patch < ${CMAKE_SOURCE_DIR}/rootbench-scripts/stackcollapse-perf.patch
+	CONFIGURE_COMMAND ""
+	BUILD_COMMAND ""
+	INSTALL_COMMAND ""
+	EXCLUDE_FROM_ALL 1
+)
+
+

cmake/modules/RootBenchOptions.cmake

@@ -3,4 +3,6 @@
 #
 # # TBD: to introduce special function for options (similar to root.git)
 #----------------------------------------------------------------------------
-set(coverage OFF)
+option(coverage "ROOTBench coverage" OFF)
+option(flamegraphCPU "CPU FlameGraph generation option" OFF)
+option(flamegraphMem "Memory FlameGraph generation option" OFF)

root/roofit/roofit/CMakeLists.txt

@@ -7,3 +7,4 @@ RB_ADD_GBENCHMARK(RoofitUnBinnedBenchmark
   RooFitUnBinnedBenchmarks.cxx
   LABEL long
   LIBRARIES Core Hist MathCore RIO RooFit RooStats RooFitCore HistFactory)
+

rootbench-scripts/.rootrc

@@ -0,0 +1 @@
+RooFit.Banner: no

rootbench-scripts/README.md

@@ -0,0 +1,35 @@
+## About
+
+flamegraph.sh is a script that generates Flame Graphs for each benchmark in `rootbench.git`. More information on Flame Graphs can be found [here](http://www.brendangregg.com/flamegraphs.html).
+
+## Options
+
+See the USAGE message (--help) for options. 
+
+To generate only CPU or only memory Flame Graphs, use `-c` or `-m` respectively. To generate both CPU and memory Flame Graphs for all benchmarks at once run the following command:
+```bash
+flamegraph.sh -d path/to/rootbench/build/dir -a -c -m 
+```
+
+To generate Flame Graphs for specific benchmark just run the following command with `-c` or `-m` options or both:
+ ```bash
+flamegraph.sh -d path/to/rootbench/build/dir -b path/to/benchmark 
+``` 
+
+## Configuration
+
+If `perf` cannot find symbols in the program try to execute the following commands
+```bash
+echo 0 > /proc/sys/kernel/kptr_restrict
+echo 1 > /proc/sys/kernel/sched_schedstats
+```
+
+To generate Flame Graphs for each benchmark add `-Dflamegraph=ON` option to your cmake configuration.
+
+```bash
+cmake ../rootbench -Dflamegraph=ON
+make -j4
+ctest -V -R rootbench-CLASSNAMEBenchmarks
+```
+You can also run the script for Flame Graph generation from your rootbench directory. 
+The Flame Graphs will be stored in the local rootbench build directory under FlameGraph folder.

rootbench-scripts/flamegraph.sh

@@ -0,0 +1,184 @@
+#!/bin/bash
+
+BENCHMARKPATTERN="*benchmark*"
+MKDIR=$(which mkdir)
+BASENAME=$(which basename)
+DIRNAME=$(which dirname)
+FIND=$(which find)
+SED=$(which sed)
+PERF=$(which perf)
+STACKCOLLAPSE=stackcollapse-perf.pl
+FLAMEGRAPH=flamegraph.pl
+
+usage() {
+	echo
+	echo "--------------------------FlameGraph Generator---------------------------"
+	echo
+	echo "Usage: $0 [OPTION]..."
+	echo "This script generates FlameGraphs for benchmarks !!!"
+	echo "OPTIONS"
+	echo "   -d, --builddir path            Create all benchmarks."
+	echo "   -b, --benchmarkfile path       Location of ROOT benchmark file"
+	echo "   -a, --all                      Create all benchmarks."
+	echo "   -c, --cpu                      Generate CPU FlameGraphs"
+	echo "   -m, --memory                   Generate Memory FlameGraphs"
+	echo "   -h, --help                     Display this help and exit"
+}
+
+usage_short() {
+	echo "Usage: $0 -d | --builddir path [-b | --benchmarkfile filepath] [-a | --all] [-c | --cpu] [-m | --memory] [-h | --help]"
+	exit 1
+}
+
+get_bm_fn() {
+	if [ ! -f $1 ]; then
+		echo "Can't find the benchmark file"
+		exit 1
+	else
+		bm_fn_full=$1
+		bm_fn=$($BASENAME $bm_fn_full)
+	fi
+}
+
+get_build_dir() {
+	if [ ! -d $1 ]; then
+		echo "Can't find the build directory. Exiting..."
+		exit 1
+	else
+		build_dir=$1
+		flamegraph_base_dir=$build_dir/FlameGraph
+	fi
+}
+
+get_bm_fn_interactive() {
+	read -p "Enter benchmark filename: " bm_fn_full
+	if [ -z $bm_fn_full ]; then
+		echo "You did not enter any filename. Exiting..."
+		exit 1
+	fi
+	get_bm_fn $bm_fn_full
+}
+
+perf_record_cpu() {
+	$PERF record -F 50 --call-graph dwarf $1 --benchmark_filter=${2}$
+}
+
+perf_script_cpu() {
+	$PERF script | stackcollapse-perf.pl | flamegraph.pl --title $1 >$2.svg
+}
+
+perf_record_mem() {
+	$PERF record -F 50 -e page-faults --call-graph dwarf $1 --benchmark_filter=${2}$
+}
+
+perf_script_mem() {
+	$PERF script | stackcollapse-perf.pl | flamegraph.pl --color=mem --title=$1 --countname="pages" >$2.svg
+}
+
+perf_rec_scr_cpu() {
+	perf_record_cpu $1 $2
+	perf_script_cpu $2 $3
+}
+
+perf_rec_scr_mem() {
+	perf_record_mem $1 $2
+	perf_script_mem $2 $3
+}
+
+get_bm_files() {
+	bm_file_list=$($FIND $build_dir/root -iname "$BENCHMARKPATTERN" | grep -v "CMakeFiles\|pyroot\|interpreter\|.csv")
+}
+
+run_bm() {
+	for bm_fn_full in $bm_file_list; do
+		bm_fn=$($BASENAME $bm_fn_full)
+		bm_dn=$($DIRNAME $bm_fn_full)
+		bm_path=$(echo "$bm_dn" | $SED -n "s|^$build_dir/||p")
+		bm_sub_list=$($bm_fn_full --benchmark_list_tests)
+		flamegraph_base_dir_mem=${flamegraph_base_dir}/FlameGraph_Memory
+		flamegraph_base_dir_cpu=${flamegraph_base_dir}/FlameGraph_CPU
+		outputdir_full_mem=${flamegraph_base_dir_mem}/$bm_path/$bm_fn
+		outputdir_full_cpu=${flamegraph_base_dir_cpu}/$bm_path/$bm_fn
+		if [ "$memory" = "y" ]; then
+			$MKDIR -p $outputdir_full_mem
+			if [ $? -ne 0 ]; then
+				echo "Can't create directory $1. Exiting..."
+				exit 1
+			fi
+		fi
+		if [ "$cpu" = "y" ]; then
+			$MKDIR -p $outputdir_full_cpu
+			if [ $? -ne 0 ]; then
+				echo "Can't create directory $1. Exiting..."
+				exit 1
+			fi
+		fi
+		for bm in $bm_sub_list; do
+			bm_modified_fn=$(echo "$bm" | $SED "s|[/:]|-|g") #replacing all "/" and ":" with "-"
+			if [ "$cpu"="y" ]; then
+				perf_rec_scr_cpu $bm_fn_full $bm ${outputdir_full_cpu}/${bm_modified_fn}_FlameGraph
+			fi
+			if [ "$memory"="y" ]; then
+				perf_rec_scr_mem $bm_fn_full $bm ${outputdir_full_mem}/${bm_modified_fn}_FlameGraph
+			fi
+		done
+	done
+}
+
+#####  Main  #####
+
+[ $# -eq 0 ] && usage_short
+
+while [ $# -gt 0 ]; do
+	case "$1" in
+	-b | --benchmarkfile)
+		shift
+		get_bm_fn $1
+		;;
+	-d | --builddir)
+		shift
+		get_build_dir $1
+		;;
+
+	-a | --all)
+		all=y
+		;;
+	-c | --cpu)
+		cpu=y
+		;;
+	-m | --memory)
+		memory=y
+		;;
+	-h | --help)
+		usage
+		exit
+		;;
+	*)
+		usage
+		exit 1
+		;;
+	esac
+	shift
+done
+
+if [ -z $build_dir ]; then
+	echo "************* Rootbench Build dir is mandatory *****************"
+	usage_short
+fi
+
+if [ -z $cpu -a -z $memory ]; then
+	echo "************* Please specify the type of the FlameGraphs *****************"
+	usage_short
+fi
+
+if [ "$all" = "y" ]; then
+	get_bm_files
+	run_bm
+else
+	if [ ! -z $bm_fn_full ]; then
+		bm_file_list=$bm_fn_full
+		run_bm
+	fi
+fi
+
+exit $?

rootbench-scripts/stackcollapse-perf.patch

@@ -0,0 +1,13 @@
+diff --git a/stackcollapse-perf.pl b/stackcollapse-perf.pl
+index e91f7de..fe88660 100755
+--- a/stackcollapse-perf.pl
++++ b/stackcollapse-perf.pl
+@@ -80,7 +80,7 @@ my $include_pid = 0;    # include process ID with process name
+ my $include_tid = 0;    # include process & thread ID with process name
+ my $include_addrs = 0;    # include raw address where a symbol can't be found
+ my $tidy_java = 1;    # condense Java signatures
+-my $tidy_generic = 1;    # clean up function names a little
++my $tidy_generic = 0;    # clean up function names a little
+ my $target_pname;    # target process name from perf invocation
+ my $event_filter = "";    # event type filter, defaults to first encountered event
+ my $event_defaulted = 0;  # whether we defaulted to an event (none provided)