Skip to content

Make benchmarks great again #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mkatliar merged 2 commits into from
Dec 9, 2024
Merged

Make benchmarks great again #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d25606f
Make benchmarks great again
roversch Oct 9, 2024
efa2065
Small fix
roversch Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,19 @@ BENCH_EIGEN = build/bin/bench-eigen
BENCH_BLAST = build/bin/bench-blast
BENCH_BLAST_OUTPUT_DIR = $(shell git rev-parse --short HEAD)
BENCH_LIBXSMM = build/bin/bench-libxsmm
BENCHMARK_OPTIONS = --benchmark_repetitions=30 --benchmark_counters_tabular=true --benchmark_out_format=json --benchmark_enable_random_interleaving=true --benchmark_min_warmup_time=10 --benchmark_min_time=1000000x
BENCHMARK_OPTIONS = --benchmark_repetitions=3 --benchmark_counters_tabular=true --benchmark_out_format=json
RUN_MATLAB = matlab -nodisplay -nosplash -nodesktop -r
BENCH_DATA = bench_result/data
BENCH_IMAGE = bench_result/image
PYTHON = python3

# run-benchmark-blaze-dynamic:
# $(BENCH_TMPC) --benchmark_filter="BM_gemm_blaze_dynamic<double>*" $(BENCHMARK_OPTIONS) \
# $(BENCH_BLAST) --benchmark_filter="BM_gemm_blaze_dynamic<double>*" $(BENCHMARK_OPTIONS) \
# --benchmark_out=${BENCH_DATA}/blaze-dynamic.json

${BENCH_DATA}/loop-naive.json:
$(BENCH_TMPC) --benchmark_filter="BM_gemm_loop_naive<double>*" $(BENCHMARK_OPTIONS) \
--benchmark_out=${BENCH_DATA}/loop-naive.json
# ${BENCH_DATA}/loop-naive.json:
# $(BENCH_BLAST) --benchmark_filter="BM_gemm_loop_naive<double>*" $(BENCHMARK_OPTIONS) \
# --benchmark_out=${BENCH_DATA}/loop-naive.json

#
# DGEMM/SGEMM bechmarks data
Expand Down Expand Up @@ -65,18 +65,22 @@ ${BENCH_DATA}/sgemm-blaze-static.json: $(BENCH_BLAZE)
${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-panel.json: $(BENCH_BLAST)
$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_panel<double, .+>" $(BENCHMARK_OPTIONS) \
--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-panel.json
cp $@ ${BENCH_DATA}
Copy link
Owner

@mkatliar mkatliar Oct 15, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This copies ${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-panel.json to ${BENCH_DATA}/dgemm-blast-static-panel.json, do I get it right? Why is it needed?

Copy link
Collaborator Author

@roversch roversch Dec 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is for the Python script. I copy everything into the same 'main' folder.


${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-panel.json: $(BENCH_BLAST)
$(BENCH_BLAST) --benchmark_filter="BM_gemm_dynamic_panel<double>" $(BENCHMARK_OPTIONS) \
--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-panel.json
cp $@ ${BENCH_DATA}

${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-plain.json: $(BENCH_BLAST)
$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_plain<double, .+>" $(BENCHMARK_OPTIONS) \
--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-plain.json
cp $@ ${BENCH_DATA}

${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-plain.json: $(BENCH_BLAST)
$(BENCH_BLAST) --benchmark_filter="BM_gemm_dynamic_plain<double>" $(BENCHMARK_OPTIONS) \
--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-plain.json
cp $@ ${BENCH_DATA}

${BENCH_DATA}/sgemm-blast-static-panel.json: $(BENCH_BLAST)
$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_panel<float, .+>" $(BENCHMARK_OPTIONS) \
Expand Down
27 changes: 13 additions & 14 deletions bench/analysis/dgemm_performance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import json
import glob
import pathlib

def filter_aggregate(benchmarks, name):
result = []
Expand All @@ -16,30 +14,31 @@ def filter_aggregate(benchmarks, name):
return result


def load_benchmark(file_name):
with open(file_name) as f:
return filter_aggregate(json.load(f)['benchmarks'], 'median')


factor = 1e+9 # Giga
style = 'o-'
line_width = 1

# prop_cycle = plt.rcParams['axes.prop_cycle']
# c = prop_cycle.by_key()['color']

plots = [
# {'data_file': 'dgemm-openblas.json', 'label': 'OpenBLAS'},
# {'data_file': 'dgemm-mkl.json', 'label': 'MKL'},
{'data_file': 'dgemm-openblas.json', 'label': 'OpenBLAS'},
{'data_file': 'dgemm-mkl.json', 'label': 'MKL'},
{'data_file': 'dgemm-blasfeo.json', 'label': 'BLASFEO'},
# {'data_file': 'dgemm-blasfeo-blas.json', 'label': 'BLASFEO*'},
# {'data_file': 'dgemm-libxsmm.json', 'label': 'LIBXSMM'},
{'data_file': 'dgemm-libxsmm.json', 'label': 'LIBXSMM'},
# {'data_file': 'dgemm-eigen-dynamic.json', 'label': 'Eigen (D)'},
# {'data_file': 'dgemm-eigen-static.json', 'label': 'Eigen (S)'},
{'data_file': 'dgemm-eigen-static.json', 'label': 'Eigen (S)'},
# {'data_file': 'dgemm-blaze-dynamic.json', 'label': 'Blaze (D)'},
# {'data_file': 'dgemm-blaze-static.json', 'label': 'Blaze (S)'},
{'data_file': 'dgemm-blast-static-panel.json', 'label': 'StaPan'},
{'data_file': 'dgemm-blast-static-plain.json', 'label': 'StaPla'},
{'data_file': 'dgemm-blast-dynamic-panel.json', 'label': 'DynPan'},
{'data_file': 'dgemm-blast-dynamic-plain.json', 'label': 'DynPla'},
]

for benchmark_file, benchmark_label in [('dgemm-blast-static-panel.json', 'SP'), ('dgemm-blast-static-plain.json', 'SD'), ('dgemm-blast-dynamic-panel.json', 'DP'), ('dgemm-blast-dynamic-plain.json', 'DD')]:
files = glob.glob('./**/' + benchmark_file, recursive=True, root_dir='bench_result/data')
for file in files:
plots.append({'data_file': file, 'label': f'BLAST ({benchmark_label}) {pathlib.Path(file).parent.stem}'})

fig = plt.figure(figsize=[10, 6])
ax = fig.subplots()

Expand Down
2 changes: 1 addition & 1 deletion bench/analysis/dgemm_performance_ratio.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ def filter_aggregate(benchmarks, name):

def load_benchmark(file_name):
with open(file_name) as f:
return filter_aggregate(json.load(f)['benchmarks'], 'mean')
return filter_aggregate(json.load(f)['benchmarks'], 'median')


factor = 1e+9 # Giga
Expand Down
6 changes: 3 additions & 3 deletions cmake/InitialCache.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Build type (Debug, Release, etc.)")
set(CMAKE_CXX_FLAGS "-march=native -mfma -mavx -mavx2 -msse4 -fno-math-errno -DXSIMD_DEFAULT_ARCH='fma3<avx2>'" CACHE STRING "Recommended compiler flags")
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG -ffast-math" CACHE STRING "Recommended compiler flags for a Release build")
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type (Debug, Release, etc.)")
set(CMAKE_CXX_FLAGS "-march=native -mfma -mavx2 -fno-math-errno -DXSIMD_DEFAULT_ARCH='fma3<avx2>'" CACHE STRING "Recommended compiler flags")
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG -ffast-math" CACHE STRING "Recommended compiler flags for a Release build")
set(BLAST_WITH_TEST ON CACHE BOOL "Build blast tests")
set(BLAST_WITH_BENCHMARK ON CACHE BOOL "Build blast benchmarks")
set(BLAST_WITH_BLASFEO ON CACHE BOOL "Build blasfeo C++ interface")
Expand Down
2 changes: 1 addition & 1 deletion docker/x86_64/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM ubuntu:kinetic
FROM ubuntu:noble
WORKDIR /root
RUN apt-get update
RUN apt-get upgrade -y
Expand Down
Loading