Skip to content

Commit a0cbfc3

Browse files
lerman25lerman25
authored
[MOD-8438] Micro-benchmarks for multi - int8 + benchmark ranges generalization (#603)
* Add multi int 8 dataset to benchmark pipeline * Generify ranges * per type range selection * fix format , remove redundent args * add comment * single to multi template python file * fix format * remove backslashes * format * Meirav correct and useful comments
1 parent 5a86c90 commit a0cbfc3

16 files changed

+224
-58
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ on:
2020
- bm-basics-fp16-single
2121
- bm-basics-fp16-multi
2222
- bm-basics-int8-single
23+
- bm-basics-int8-multi
2324
- bm-batch-iter-fp32-single
2425
- bm-batch-iter-fp32-multi
2526
- bm-batch-iter-fp64-single
@@ -29,6 +30,7 @@ on:
2930
- bm-batch-iter-fp16-single
3031
- bm-batch-iter-fp16-multi
3132
- bm-batch-iter-int8-single
33+
- bm-batch-iter-int8-multi
3234
- bm-updated-fp32-single
3335
- bm-spaces
3436
description: 'Benchmarks set to run'

‎tests/benchmark/benchmarks.sh‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,11 @@ BM_TYPE=$1;
33
if [ -z "$BM_TYPE" ] || [ "$BM_TYPE" = "benchmarks-all" ]; then
44
for bm_class in basics batch_iterator; do
55
for type in single multi; do
6-
for data_type in fp32 fp64 bf16 fp16; do
6+
for data_type in fp32 fp64 bf16 fp16 int8; do
77
echo ${bm_class}_${type}_${data_type};
88
done
99
done
1010
done
11-
echo basics_single_int8
12-
echo batch_iterator_single_int8
1311
echo updated_index_single_fp32
1412
echo spaces_fp32
1513
echo spaces_fp64
@@ -47,6 +45,8 @@ elif [ "$BM_TYPE" = "bm-basics-fp16-multi" ] ; then
4745
echo basics_multi_fp16
4846
elif [ "$BM_TYPE" = "bm-basics-int8-single" ] ; then
4947
echo basics_single_int8
48+
elif [ "$BM_TYPE" = "bm-basics-int8-multi" ] ; then
49+
echo basics_multi_int8
5050

5151
# Batch iterator benchmarks
5252
elif [ "$BM_TYPE" = "bm-batch-iter-fp32-single" ] ; then
@@ -67,6 +67,8 @@ elif [ "$BM_TYPE" = "bm-batch-iter-fp16-multi" ] ; then
6767
echo batch_iterator_multi_fp16
6868
elif [ "$BM_TYPE" = "bm-batch-iter-int8-single" ] ; then
6969
echo batch_iterator_single_int8
70+
elif [ "$BM_TYPE" = "bm-batch-iter-int8-multi" ] ; then
71+
echo batch_iterator_multi_int8
7072

7173
# Updated index benchmarks
7274
elif [ "$BM_TYPE" = "bm-updated-fp32-single" ] ; then

‎tests/benchmark/bm_files.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@ elif [ "$BM_TYPE" = "bm-basics-fp16-single" ] \
2727
then
2828
file_name="basic_fp16"
2929
elif [ "$BM_TYPE" = "bm-basics-int8-single" ] \
30-
|| [ "$BM_TYPE" = "bm-batch-iter-int8-single" ]
30+
|| [ "$BM_TYPE" = "bm-basics-int8-multi" ] \
31+
|| [ "$BM_TYPE" = "bm-batch-iter-int8-single" ] \
32+
|| [ "$BM_TYPE" = "bm-batch-iter-int8-multi" ]
3133
then
3234
file_name="basic_int8"
3335
elif [ "$BM_TYPE" = "bm-updated-fp32-single" ]; then

‎tests/benchmark/bm_initialization/bm_basics_initialize_bf16.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,12 @@ REGISTER_TopK_Tiered(BM_VecSimCommon, BM_FUNC_NAME(TopK, Tiered));
4848
// Range BF
4949
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, BF), bf16_index_t)
5050
(benchmark::State &st) { Range_BF(st); }
51-
REGISTER_Range_BF(BM_FUNC_NAME(Range, BF));
51+
REGISTER_Range_BF(BM_FUNC_NAME(Range, BF), bf16_index_t);
5252

5353
// Range HNSW
5454
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, HNSW), bf16_index_t)
5555
(benchmark::State &st) { Range_HNSW(st); }
56-
REGISTER_Range_HNSW(BM_FUNC_NAME(Range, HNSW));
56+
REGISTER_Range_HNSW(BM_FUNC_NAME(Range, HNSW), bf16_index_t);
5757

5858
// Tiered HNSW add/delete benchmarks
5959
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_TIERED);

‎tests/benchmark/bm_initialization/bm_basics_initialize_fp16.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,12 @@ REGISTER_TopK_Tiered(BM_VecSimCommon, BM_FUNC_NAME(TopK, Tiered));
4848
// Range BF
4949
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, BF), fp16_index_t)
5050
(benchmark::State &st) { Range_BF(st); }
51-
REGISTER_Range_BF(BM_FUNC_NAME(Range, BF));
51+
REGISTER_Range_BF(BM_FUNC_NAME(Range, BF), fp16_index_t);
5252

5353
// Range HNSW
5454
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, HNSW), fp16_index_t)
5555
(benchmark::State &st) { Range_HNSW(st); }
56-
REGISTER_Range_HNSW(BM_FUNC_NAME(Range, HNSW));
56+
REGISTER_Range_HNSW(BM_FUNC_NAME(Range, HNSW), fp16_index_t);
5757

5858
// Tiered HNSW add/delete benchmarks
5959
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_TIERED);

‎tests/benchmark/bm_initialization/bm_basics_initialize_fp32.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,12 @@ REGISTER_TopK_Tiered(BM_VecSimCommon, BM_FUNC_NAME(TopK, Tiered));
4848
// Range BF
4949
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, BF), fp32_index_t)
5050
(benchmark::State &st) { Range_BF(st); }
51-
REGISTER_Range_BF(BM_FUNC_NAME(Range, BF));
51+
REGISTER_Range_BF(BM_FUNC_NAME(Range, BF), fp32_index_t);
5252

5353
// Range HNSW
5454
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, HNSW), fp32_index_t)
5555
(benchmark::State &st) { Range_HNSW(st); }
56-
REGISTER_Range_HNSW(BM_FUNC_NAME(Range, HNSW));
56+
REGISTER_Range_HNSW(BM_FUNC_NAME(Range, HNSW), fp32_index_t);
5757

5858
// Tiered HNSW add/delete benchmarks
5959
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_TIERED);

‎tests/benchmark/bm_initialization/bm_basics_initialize_fp64.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,12 @@ REGISTER_TopK_Tiered(BM_VecSimCommon, BM_FUNC_NAME(TopK, Tiered));
4848
// Range BF
4949
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, BF), fp64_index_t)
5050
(benchmark::State &st) { Range_BF(st); }
51-
REGISTER_Range_BF(BM_FUNC_NAME(Range, BF));
51+
REGISTER_Range_BF(BM_FUNC_NAME(Range, BF), fp64_index_t);
5252

5353
// Range HSNW
5454
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, HNSW), fp64_index_t)
5555
(benchmark::State &st) { Range_HNSW(st); }
56-
REGISTER_Range_HNSW(BM_FUNC_NAME(Range, HNSW));
56+
REGISTER_Range_HNSW(BM_FUNC_NAME(Range, HNSW), fp64_index_t);
5757

5858
// Tiered HNSW add/delete benchmarks
5959
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_TIERED);

‎tests/benchmark/bm_initialization/bm_basics_initialize_int8.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,12 @@ REGISTER_TopK_Tiered(BM_VecSimCommon, BM_FUNC_NAME(TopK, Tiered));
4848
// Range BF
4949
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, BF), int8_index_t)
5050
(benchmark::State &st) { Range_BF(st); }
51-
REGISTER_Range_INT8_BF(BM_FUNC_NAME(Range, BF));
51+
REGISTER_Range_BF(BM_FUNC_NAME(Range, BF), int8_index_t);
5252

5353
// Range HNSW
5454
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, HNSW), int8_index_t)
5555
(benchmark::State &st) { Range_HNSW(st); }
56-
REGISTER_Range_INT8_HNSW(BM_FUNC_NAME(Range, HNSW));
56+
REGISTER_Range_HNSW(BM_FUNC_NAME(Range, HNSW), int8_index_t);
5757

5858
// Tiered HNSW add/delete benchmarks
5959
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_TIERED);

‎tests/benchmark/bm_vecsim_basics.h‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#include <atomic>
33
#include "bm_common.h"
44
#include <chrono>
5+
#include "types_ranges.h"
56

67
using namespace std::chrono;
78

@@ -288,30 +289,36 @@ void BM_VecSimBasics<index_type_t>::Range_HNSW(benchmark::State &st) {
288289

289290
#define UNIT_AND_ITERATIONS Unit(benchmark::kMillisecond)->Iterations(BM_VecSimGeneral::block_size)
290291

292+
// These macros are used to make sure the expansion of other macros happens when needed
293+
#define MACRO_EXPAND_AND_CONCATENATE(a, b) a##b
294+
#define MACRO_CONCATENATE(a, b) MACRO_EXPAND_AND_CONCATENATE(a, b)
295+
291296
// The actual radius will be the given arg divided by 100, since arg must be an integer.
292-
#define REGISTER_Range_BF(BM_FUNC) \
297+
#define REGISTER_Range_BF(BM_FUNC, TYPENAME) \
298+
static void MACRO_CONCATENATE(BM_FUNC, _Args)(benchmark::internal::Benchmark * b) { \
299+
for (int radius : benchmark_range<TYPENAME>::get_radii()) { \
300+
b->Args({radius}); \
301+
} \
302+
} \
293303
BENCHMARK_REGISTER_F(BM_VecSimBasics, BM_FUNC) \
294-
->Arg(20) \
295-
->Arg(35) \
296-
->Arg(50) \
297-
->ArgName("radiusX100") \
304+
->Apply(MACRO_CONCATENATE(BM_FUNC, _Args)) \
305+
->ArgNames({"radiusX100"}) \
298306
->Iterations(10) \
299-
->Unit(benchmark::kMillisecond)
307+
->Unit(benchmark::kMillisecond);
300308

301309
// {radius*100, epsilon*1000}
302310
// The actual radius will be the given arg divided by 100, and the actual epsilon values
303311
// will be the given arg divided by 1000.
304-
#define REGISTER_Range_HNSW(BM_FUNC) \
312+
#define REGISTER_Range_HNSW(BM_FUNC, TYPENAME) \
313+
static void MACRO_CONCATENATE(BM_FUNC, _Args)(benchmark::internal::Benchmark * b) { \
314+
for (int radius : benchmark_range<TYPENAME>::get_radii()) { \
315+
for (int epsilon : benchmark_range<TYPENAME>::get_epsilons()) { \
316+
b->Args({radius, epsilon}); \
317+
} \
318+
} \
319+
} \
305320
BENCHMARK_REGISTER_F(BM_VecSimBasics, BM_FUNC) \
306-
->Args({20, 1}) \
307-
->Args({20, 10}) \
308-
->Args({20, 100}) \
309-
->Args({35, 1}) \
310-
->Args({35, 10}) \
311-
->Args({35, 100}) \
312-
->Args({50, 1}) \
313-
->Args({50, 10}) \
314-
->Args({50, 100}) \
321+
->Apply(MACRO_CONCATENATE(BM_FUNC, _Args)) \
315322
->ArgNames({"radiusX100", "epsilonX1000"}) \
316323
->Iterations(10) \
317324
->Unit(benchmark::kMillisecond)

‎tests/benchmark/data/hnsw_indices/hnsw_indices_all.txt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,8 @@ https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/dbpedia-cosine-dim768-fp
2525
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/fashion_images_multi_value-cosine-dim512-M64-efc512-fp16.hnsw_v3
2626
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/fashion_images_multi_value-cosine-dim512-fp16-test_vectors.raw
2727

28-
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_single-cosine-dim1024-M64-efc512-int8.hnsw_v3
28+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_single-cosine-dim1024-M64-efc512-int8.hnsw_v3
2929
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_single-cosine-dim1024-int8-test_vectors.raw
30+
31+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_multi-cosine-dim1024-M64-efc512-int8.hnsw_v3
32+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_multi-cosine-dim1024-int8-test_vectors.raw

0 commit comments

Comments
 (0)