diff --git a/libcxx/docs/TestingLibcxx.rst b/libcxx/docs/TestingLibcxx.rst index cf092fabd046f..e98b96bfb478f 100644 --- a/libcxx/docs/TestingLibcxx.rst +++ b/libcxx/docs/TestingLibcxx.rst @@ -459,6 +459,29 @@ we only want to make sure they don't rot. Do not rely on the results of benchmar run through ``check-cxx`` for anything, instead run the benchmarks manually using the instructions for running individual tests. +If you want to compare the results of different benchmark runs, we recommend using the +``libcxx-compare-benchmarks`` helper tool. First, configure CMake in a build directory +and run the benchmark: + +.. code-block:: bash + + $ cmake -S runtimes -B [...] + $ libcxx/utils/libcxx-lit libcxx/test/benchmarks/string.bench.cpp --param optimization=speed + +Then, do the same for the second configuration you want to test. Use a different build +directory for that configuration: + +.. code-block:: bash + + $ cmake -S runtimes -B [...] + $ libcxx/utils/libcxx-lit libcxx/test/benchmarks/string.bench.cpp --param optimization=speed + +Finally, use ``libcxx-compare-benchmarks`` to compare both: + +.. code-block:: bash + + $ libcxx/utils/libcxx-compare-benchmarks libcxx/test/benchmarks/string.bench.cpp + .. _`Google Benchmark`: https://github.com/google/benchmark .. _testing-hardening-assertions: diff --git a/libcxx/utils/libcxx-benchmark-json b/libcxx/utils/libcxx-benchmark-json new file mode 100755 index 0000000000000..7f743c32caf40 --- /dev/null +++ b/libcxx/utils/libcxx-benchmark-json @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -e + +PROGNAME="$(basename "${0}")" +MONOREPO_ROOT="$(realpath $(dirname "${PROGNAME}"))" +function usage() { +cat < benchmarks... + +Print the path to the JSON files containing benchmark results for the given benchmarks. + +This requires those benchmarks to have already been run, i.e. this only resolves the path +to the benchmark .json file within the build directory. + + The path to the build directory. +benchmarks... Paths of the benchmarks to extract the results for. Those paths are relative to ''. + +Example +======= +$ cmake -S runtimes -B build/ -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" +$ libcxx-lit build/ -sv libcxx/test/benchmarks/algorithms/for_each.bench.cpp +$ less \$(${PROGNAME} build/ libcxx/test/benchmarks/algorithms/for_each.bench.cpp) +EOF +} + +if [[ "${1}" == "-h" || "${1}" == "--help" ]]; then + usage + exit 0 +fi + +if [[ $# -lt 1 ]]; then + usage + exit 1 +fi + +build_dir="${1}" +shift + +for benchmark in ${@}; do + # Normalize the paths by turning all benchmarks paths into absolute ones and then making them + # relative to the root of the monorepo. + benchmark="$(realpath ${benchmark})" + relative=$(python -c "import os; import sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))" "${benchmark}" "${MONOREPO_ROOT}") + + # Extract components of the benchmark path + directory="$(dirname ${relative})" + file="$(basename ${relative})" + + # Reconstruct the (slightly weird) path to the benchmark json file. This should be kept in sync + # whenever the test suite changes. + json="${build_dir}/${directory}/Output/${file}.dir/benchmark-result.json" + if [[ -f "${json}" ]]; then + echo "${json}" + fi +done diff --git a/libcxx/utils/libcxx-compare-benchmarks b/libcxx/utils/libcxx-compare-benchmarks new file mode 100755 index 0000000000000..e04820fc57ed9 --- /dev/null +++ b/libcxx/utils/libcxx-compare-benchmarks @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -e + +PROGNAME="$(basename "${0}")" +MONOREPO_ROOT="$(realpath $(dirname "${PROGNAME}"))" +function usage() { +cat < benchmarks... + +Compare the given benchmarks between the baseline and the candidate build directories. + +This requires those benchmarks to have already been generated in both build directories. + + The path to the build directory considered the baseline. + The path to the build directory considered the candidate. +benchmarks... Paths of the benchmarks to compare. Those paths are relative to ''. + +Example +======= +$ libcxx-lit build1/ -sv libcxx/test/benchmarks/algorithms/for_each.bench.cpp +$ libcxx-lit build2/ -sv libcxx/test/benchmarks/algorithms/for_each.bench.cpp +$ ${PROGNAME} build1/ build2/ libcxx/test/benchmarks/algorithms/for_each.bench.cpp +EOF +} + +if [[ "${1}" == "-h" || "${1}" == "--help" ]]; then + usage + exit 0 +fi + +if [[ $# -lt 1 ]]; then + usage + exit 1 +fi + +baseline="${1}" +candidate="${2}" +shift; shift + +GBENCH="${MONOREPO_ROOT}/third-party/benchmark" + +python3 -m venv /tmp/libcxx-compare-benchmarks-venv +source /tmp/libcxx-compare-benchmarks-venv/bin/activate +pip3 install -r ${GBENCH}/tools/requirements.txt + +for benchmark in ${@}; do + base="$(${MONOREPO_ROOT}/libcxx/utils/libcxx-benchmark-json ${baseline} ${benchmark})" + cand="$(${MONOREPO_ROOT}/libcxx/utils/libcxx-benchmark-json ${candidate} ${benchmark})" + + if [[ ! -e "${base}" ]]; then + echo "Benchmark ${benchmark} does not exist in the baseline" + continue + fi + if [[ ! -e "${cand}" ]]; then + echo "Benchmark ${benchmark} does not exist in the candidate" + continue + fi + + "${GBENCH}/tools/compare.py" benchmarks "${base}" "${cand}" +done