Skip to content

Commit d25606f

Browse files
roverschMikhail Katliar
roversch
authored and
Mikhail Katliar
committed
Make benchmarks great again
1 parent 5886fd3 commit d25606f

File tree

5 files changed

+27
-24
lines changed

5 files changed

+27
-24
lines changed

‎Makefile‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,19 @@ BENCH_EIGEN = build/bin/bench-eigen
55
BENCH_BLAST = build/bin/bench-blast
66
BENCH_BLAST_OUTPUT_DIR = $(shell git rev-parse --short HEAD)
77
BENCH_LIBXSMM = build/bin/bench-libxsmm
8-
BENCHMARK_OPTIONS = --benchmark_repetitions=30 --benchmark_counters_tabular=true --benchmark_out_format=json --benchmark_enable_random_interleaving=true --benchmark_min_warmup_time=10 --benchmark_min_time=1000000x
8+
BENCHMARK_OPTIONS = --benchmark_repetitions=3 --benchmark_counters_tabular=true --benchmark_out_format=json
99
RUN_MATLAB = matlab -nodisplay -nosplash -nodesktop -r
1010
BENCH_DATA = bench_result/data
1111
BENCH_IMAGE = bench_result/image
1212
PYTHON = python3
1313

1414
# run-benchmark-blaze-dynamic:
15-
# $(BENCH_TMPC) --benchmark_filter="BM_gemm_blaze_dynamic<double>*" $(BENCHMARK_OPTIONS) \
15+
# $(BENCH_BLAST) --benchmark_filter="BM_gemm_blaze_dynamic<double>*" $(BENCHMARK_OPTIONS) \
1616
# --benchmark_out=${BENCH_DATA}/blaze-dynamic.json
1717

18-
${BENCH_DATA}/loop-naive.json:
19-
$(BENCH_TMPC) --benchmark_filter="BM_gemm_loop_naive<double>*" $(BENCHMARK_OPTIONS) \
20-
--benchmark_out=${BENCH_DATA}/loop-naive.json
18+
# ${BENCH_DATA}/loop-naive.json:
19+
# $(BENCH_BLAS) --benchmark_filter="BM_gemm_loop_naive<double>*" $(BENCHMARK_OPTIONS) \
20+
# --benchmark_out=${BENCH_DATA}/loop-naive.json
2121

2222
#
2323
# DGEMM/SGEMM bechmarks data
@@ -65,18 +65,22 @@ ${BENCH_DATA}/sgemm-blaze-static.json: $(BENCH_BLAZE)
6565
${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-panel.json: $(BENCH_BLAST)
6666
$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_panel<double, .+>" $(BENCHMARK_OPTIONS) \
6767
--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-panel.json
68+
cp $@ ${BENCH_DATA}
6869

6970
${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-panel.json: $(BENCH_BLAST)
7071
$(BENCH_BLAST) --benchmark_filter="BM_gemm_dynamic_panel<double>" $(BENCHMARK_OPTIONS) \
7172
--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-panel.json
73+
cp $@ ${BENCH_DATA}
7274

7375
${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-plain.json: $(BENCH_BLAST)
7476
$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_plain<double, .+>" $(BENCHMARK_OPTIONS) \
7577
--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-plain.json
78+
cp $@ ${BENCH_DATA}
7679

7780
${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-plain.json: $(BENCH_BLAST)
7881
$(BENCH_BLAST) --benchmark_filter="BM_gemm_dynamic_plain<double>" $(BENCHMARK_OPTIONS) \
7982
--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-plain.json
83+
cp $@ ${BENCH_DATA}
8084

8185
${BENCH_DATA}/sgemm-blast-static-panel.json: $(BENCH_BLAST)
8286
$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_panel<float, .+>" $(BENCHMARK_OPTIONS) \

‎bench/analysis/dgemm_performance.py‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
matplotlib.use("Agg")
33
import matplotlib.pyplot as plt
44
import json
5-
import glob
6-
import pathlib
75

86
def filter_aggregate(benchmarks, name):
97
result = []
@@ -16,30 +14,31 @@ def filter_aggregate(benchmarks, name):
1614
return result
1715

1816

17+
def load_benchmark(file_name):
18+
with open(file_name) as f:
19+
return filter_aggregate(json.load(f)['benchmarks'], 'median')
20+
21+
1922
factor = 1e+9 # Giga
2023
style = 'o-'
2124
line_width = 1
2225

23-
# prop_cycle = plt.rcParams['axes.prop_cycle']
24-
# c = prop_cycle.by_key()['color']
25-
2626
plots = [
27-
# {'data_file': 'dgemm-openblas.json', 'label': 'OpenBLAS'},
28-
# {'data_file': 'dgemm-mkl.json', 'label': 'MKL'},
27+
{'data_file': 'dgemm-openblas.json', 'label': 'OpenBLAS'},
28+
{'data_file': 'dgemm-mkl.json', 'label': 'MKL'},
2929
{'data_file': 'dgemm-blasfeo.json', 'label': 'BLASFEO'},
3030
# {'data_file': 'dgemm-blasfeo-blas.json', 'label': 'BLASFEO*'},
31-
# {'data_file': 'dgemm-libxsmm.json', 'label': 'LIBXSMM'},
31+
{'data_file': 'dgemm-libxsmm.json', 'label': 'LIBXSMM'},
3232
# {'data_file': 'dgemm-eigen-dynamic.json', 'label': 'Eigen (D)'},
33-
# {'data_file': 'dgemm-eigen-static.json', 'label': 'Eigen (S)'},
33+
{'data_file': 'dgemm-eigen-static.json', 'label': 'Eigen (S)'},
3434
# {'data_file': 'dgemm-blaze-dynamic.json', 'label': 'Blaze (D)'},
3535
# {'data_file': 'dgemm-blaze-static.json', 'label': 'Blaze (S)'},
36+
{'data_file': 'dgemm-blast-static-panel.json', 'label': 'StaPan'},
37+
{'data_file': 'dgemm-blast-static-plain.json', 'label': 'StaPla'},
38+
{'data_file': 'dgemm-blast-dynamic-panel.json', 'label': 'DynPan'},
39+
{'data_file': 'dgemm-blast-dynamic-plain.json', 'label': 'DynPla'},
3640
]
3741

38-
for benchmark_file, benchmark_label in [('dgemm-blast-static-panel.json', 'SP'), ('dgemm-blast-static-plain.json', 'SD'), ('dgemm-blast-dynamic-panel.json', 'DP'), ('dgemm-blast-dynamic-plain.json', 'DD')]:
39-
files = glob.glob('./**/' + benchmark_file, recursive=True, root_dir='bench_result/data')
40-
for file in files:
41-
plots.append({'data_file': file, 'label': f'BLAST ({benchmark_label}) {pathlib.Path(file).parent.stem}'})
42-
4342
fig = plt.figure(figsize=[10, 6])
4443
ax = fig.subplots()
4544

‎bench/analysis/dgemm_performance_ratio.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ def filter_aggregate(benchmarks, name):
1616

1717
def load_benchmark(file_name):
1818
with open(file_name) as f:
19-
return filter_aggregate(json.load(f)['benchmarks'], 'mean')
19+
return filter_aggregate(json.load(f)['benchmarks'], 'median')
2020

2121

2222
factor = 1e+9 # Giga

‎cmake/InitialCache.cmake‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Build type (Debug, Release, etc.)")
2-
set(CMAKE_CXX_FLAGS "-march=native -mfma -mavx -mavx2 -msse4 -fno-math-errno -DXSIMD_DEFAULT_ARCH='fma3<avx2>'" CACHE STRING "Recommended compiler flags")
3-
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG -ffast-math" CACHE STRING "Recommended compiler flags for a Release build")
1+
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type (Debug, Release, etc.)")
2+
set(CMAKE_CXX_FLAGS "-march=native -mfma -mavx2 -fno-math-errno -DXSIMD_DEFAULT_ARCH='fma3<avx2>'" CACHE STRING "Recommended compiler flags")
3+
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG -ffast-math" CACHE STRING "Recommended compiler flags for a Release build")
44
set(BLAST_WITH_TEST ON CACHE BOOL "Build blast tests")
55
set(BLAST_WITH_BENCHMARK ON CACHE BOOL "Build blast benchmarks")
66
set(BLAST_WITH_BLASFEO ON CACHE BOOL "Build blasfeo C++ interface")

‎docker/x86_64/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM ubuntu:kinetic
1+
FROM ubuntu:noble
22
WORKDIR /root
33
RUN apt-get update
44
RUN apt-get upgrade -y

0 commit comments

Comments
 (0)