Skip to content

Commit a9b4663

Browse files
lerman25lerman25
authored
[MOD-8439] Benchmark uint8 - single&multi (#605)
* Add uint8 support * add uint-fy py file * add uint8 to yml * easy test for uint8 * fix bmark.sh * fix amazon files * fix files * fix files2 * format+radii change * more reasonable ranges * revert easy test
1 parent dd137f6 commit a9b4663

16 files changed

+325
-1
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ on:
2121
- bm-basics-fp16-multi
2222
- bm-basics-int8-single
2323
- bm-basics-int8-multi
24+
- bm-basics-uint8-single
25+
- bm-basics-uint8-multi
2426
- bm-batch-iter-fp32-single
2527
- bm-batch-iter-fp32-multi
2628
- bm-batch-iter-fp64-single
@@ -31,6 +33,8 @@ on:
3133
- bm-batch-iter-fp16-multi
3234
- bm-batch-iter-int8-single
3335
- bm-batch-iter-int8-multi
36+
- bm-batch-iter-uint8-single
37+
- bm-batch-iter-uint8-multi
3438
- bm-updated-fp32-single
3539
- bm-spaces
3640
description: 'Benchmarks set to run'

‎tests/benchmark/benchmarks.sh‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ BM_TYPE=$1;
33
if [ -z "$BM_TYPE" ] || [ "$BM_TYPE" = "benchmarks-all" ]; then
44
for bm_class in basics batch_iterator; do
55
for type in single multi; do
6-
for data_type in fp32 fp64 bf16 fp16 int8; do
6+
for data_type in fp32 fp64 bf16 fp16 int8 uint8; do
77
echo ${bm_class}_${type}_${data_type};
88
done
99
done
@@ -26,6 +26,7 @@ elif [ "$BM_TYPE" = "benchmarks-default" ]; then
2626
echo spaces_int8
2727
echo spaces_uint8
2828

29+
2930
# Basic benchmarks
3031
elif [ "$BM_TYPE" = "bm-basics-fp32-single" ] ; then
3132
echo basics_single_fp32
@@ -47,6 +48,10 @@ elif [ "$BM_TYPE" = "bm-basics-int8-single" ] ; then
4748
echo basics_single_int8
4849
elif [ "$BM_TYPE" = "bm-basics-int8-multi" ] ; then
4950
echo basics_multi_int8
51+
elif [ "$BM_TYPE" = "bm-basics-uint8-single" ] ; then
52+
echo basics_single_uint8
53+
elif [ "$BM_TYPE" = "bm-basics-uint8-multi" ] ; then
54+
echo basics_multi_uint8
5055

5156
# Batch iterator benchmarks
5257
elif [ "$BM_TYPE" = "bm-batch-iter-fp32-single" ] ; then
@@ -69,6 +74,10 @@ elif [ "$BM_TYPE" = "bm-batch-iter-int8-single" ] ; then
6974
echo batch_iterator_single_int8
7075
elif [ "$BM_TYPE" = "bm-batch-iter-int8-multi" ] ; then
7176
echo batch_iterator_multi_int8
77+
elif [ "$BM_TYPE" = "bm-batch-iter-uint8-single" ] ; then
78+
echo batch_iterator_single_uint8
79+
elif [ "$BM_TYPE" = "bm-batch-iter-uint8-multi" ] ; then
80+
echo batch_iterator_multi_uint8
7281

7382
# Updated index benchmarks
7483
elif [ "$BM_TYPE" = "bm-updated-fp32-single" ] ; then

‎tests/benchmark/bm_definitions.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ using fp64_index_t = IndexType<VecSimType_FLOAT64, double, double>;
1717
using bf16_index_t = IndexType<VecSimType_BFLOAT16, vecsim_types::bfloat16, float>;
1818
using fp16_index_t = IndexType<VecSimType_FLOAT16, vecsim_types::float16, float>;
1919
using int8_index_t = IndexType<VecSimType_INT8, int8_t, float>;
20+
using uint8_index_t = IndexType<VecSimType_UINT8, uint8_t, float>;
2021

2122
#define INDICES BM_VecSimIndex<index_type_t>::indices
2223
#define QUERIES BM_VecSimIndex<index_type_t>::queries

‎tests/benchmark/bm_files.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@ elif [ "$BM_TYPE" = "bm-basics-int8-single" ] \
3232
|| [ "$BM_TYPE" = "bm-batch-iter-int8-multi" ]
3333
then
3434
file_name="basic_int8"
35+
elif [ "$BM_TYPE" = "bm-basics-uint8-single" ] \
36+
|| [ "$BM_TYPE" = "bm-basics-uint8-multi" ] \
37+
|| [ "$BM_TYPE" = "bm-batch-iter-uint8-single" ] \
38+
|| [ "$BM_TYPE" = "bm-batch-iter-uint8-multi" ] \
39+
|| [ "$BM_TYPE" = "benchmarks-uint8" ]
40+
then
41+
file_name="basic_uint8"
3542
elif [ "$BM_TYPE" = "bm-updated-fp32-single" ]; then
3643
file_name="updated"
3744
else

‎tests/benchmark/bm_initialization/bm_basics_initialize_uint8.h‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#pragma once
2+
/**************************************
3+
Define and register tests
4+
NOTE: benchmarks' tests order can affect their results. Please add new benchmarks at the end of
5+
the file.
6+
***************************************/
7+
8+
// Memory BF
9+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, FLAT), uint8_index_t)
10+
(benchmark::State &st) { Memory_FLAT(st); }
11+
BENCHMARK_REGISTER_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, FLAT))->Iterations(1);
12+
13+
// Memory HNSW
14+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, HNSW), uint8_index_t)
15+
(benchmark::State &st) { Memory_HNSW(st); }
16+
BENCHMARK_REGISTER_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, HNSW))->Iterations(1);
17+
18+
// Memory Tiered
19+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, Tiered), uint8_index_t)
20+
(benchmark::State &st) { Memory_Tiered(st); }
21+
BENCHMARK_REGISTER_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, Tiered))->Iterations(1);
22+
23+
// AddLabel
24+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_ADD_LABEL, uint8_index_t)
25+
(benchmark::State &st) { AddLabel(st); }
26+
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_BF);
27+
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_HNSWLIB);
28+
29+
// DeleteLabel Registration. Definition is placed in the .cpp file.
30+
REGISTER_DeleteLabel(BM_FUNC_NAME(DeleteLabel, BF));
31+
REGISTER_DeleteLabel(BM_FUNC_NAME(DeleteLabel, HNSW));
32+
33+
// TopK BF
34+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(TopK, BF), uint8_index_t)
35+
(benchmark::State &st) { TopK_BF(st); }
36+
REGISTER_TopK_BF(BM_VecSimCommon, BM_FUNC_NAME(TopK, BF));
37+
38+
// TopK HNSW
39+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(TopK, HNSW), uint8_index_t)
40+
(benchmark::State &st) { TopK_HNSW(st); }
41+
REGISTER_TopK_HNSW(BM_VecSimCommon, BM_FUNC_NAME(TopK, HNSW));
42+
43+
// TopK Tiered HNSW
44+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(TopK, Tiered), uint8_index_t)
45+
(benchmark::State &st) { TopK_Tiered(st); }
46+
REGISTER_TopK_Tiered(BM_VecSimCommon, BM_FUNC_NAME(TopK, Tiered));
47+
48+
// Range BF
49+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, BF), uint8_index_t)
50+
(benchmark::State &st) { Range_BF(st); }
51+
REGISTER_Range_BF(BM_FUNC_NAME(Range, BF), uint8_index_t);
52+
53+
// Range HNSW
54+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, HNSW), uint8_index_t)
55+
(benchmark::State &st) { Range_HNSW(st); }
56+
REGISTER_Range_HNSW(BM_FUNC_NAME(Range, HNSW), uint8_index_t);
57+
58+
// Tiered HNSW add/delete benchmarks
59+
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_TIERED);
60+
REGISTER_DeleteLabel(BM_FUNC_NAME(DeleteLabel, Tiered));
61+
62+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_ADD_LABEL_ASYNC, uint8_index_t)
63+
(benchmark::State &st) { AddLabel_AsyncIngest(st); }
64+
BENCHMARK_REGISTER_F(BM_VecSimBasics, BM_ADD_LABEL_ASYNC)
65+
->UNIT_AND_ITERATIONS->Arg(VecSimAlgo_TIERED)
66+
->ArgName("VecSimAlgo_TIERED");
67+
68+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_DELETE_LABEL_ASYNC, uint8_index_t)
69+
(benchmark::State &st) { DeleteLabel_AsyncRepair(st); }

‎tests/benchmark/bm_initialization/bm_batch_initialize_uint8.h‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#pragma once
2+
3+
/**************************************
4+
Define and register benchmarks for batch iterator with index of data type uint8
5+
NOTE: benchmarks' tests order can affect their results. Please add new benchmarks at the end of
6+
the file.
7+
***************************************/
8+
9+
// Fixed size batch BF
10+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(BF, FixedBatchSize), uint8_index_t)
11+
(benchmark::State &st) { BF_FixedBatchSize(st); }
12+
REGISTER_FixedBatchSize(BM_FUNC_NAME(BF, FixedBatchSize));
13+
14+
// Variable size batch BF
15+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(BF, VariableBatchSize), uint8_index_t)
16+
(benchmark::State &st) { BF_VariableBatchSize(st); }
17+
REGISTER_VariableBatchSize(BM_FUNC_NAME(BF, VariableBatchSize));
18+
19+
// Batches to hadoc BF
20+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(BF, BatchesToAdhocBF), uint8_index_t)
21+
(benchmark::State &st) { BF_BatchesToAdhocBF(st); }
22+
REGISTER_BatchesToAdhocBF(BM_FUNC_NAME(BF, BatchesToAdhocBF));
23+
24+
// Fixed size batch HNSW
25+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(HNSW, FixedBatchSize), uint8_index_t)
26+
(benchmark::State &st) { HNSW_FixedBatchSize(st); }
27+
REGISTER_FixedBatchSize(BM_FUNC_NAME(HNSW, FixedBatchSize));
28+
29+
// Variable size batch BF
30+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(HNSW, VariableBatchSize), uint8_index_t)
31+
(benchmark::State &st) { HNSW_VariableBatchSize(st); }
32+
REGISTER_VariableBatchSize(BM_FUNC_NAME(HNSW, VariableBatchSize));
33+
34+
// Batches to hadoc HSNW
35+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(HNSW, BatchesToAdhocBF), uint8_index_t)
36+
(benchmark::State &st) { HNSW_BatchesToAdhocBF(st); }
37+
38+
REGISTER_HNSW_BatchesToAdhocBF(BM_FUNC_NAME(HNSW, BatchesToAdhocBF));

‎tests/benchmark/bm_vecsim_index.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ std::vector<std::vector<vecsim_types::float16>> BM_VecSimIndex<fp16_index_t>::qu
5454
template <>
5555
std::vector<std::vector<int8_t>> BM_VecSimIndex<int8_index_t>::queries{};
5656

57+
template <>
58+
std::vector<std::vector<uint8_t>> BM_VecSimIndex<uint8_index_t>::queries{};
59+
5760
template <>
5861
std::vector<VecSimIndex *> BM_VecSimIndex<fp32_index_t>::indices{};
5962

@@ -69,6 +72,9 @@ std::vector<VecSimIndex *> BM_VecSimIndex<fp16_index_t>::indices{};
6972
template <>
7073
std::vector<VecSimIndex *> BM_VecSimIndex<int8_index_t>::indices{};
7174

75+
template <>
76+
std::vector<VecSimIndex *> BM_VecSimIndex<uint8_index_t>::indices{};
77+
7278
template <typename index_type_t>
7379
BM_VecSimIndex<index_type_t>::~BM_VecSimIndex() {
7480
ref_count--;

‎tests/benchmark/data/hnsw_indices/hnsw_indices_all.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,9 @@ https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_single-cosine-di
3030

3131
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_multi-cosine-dim1024-M64-efc512-int8.hnsw_v3
3232
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_multi-cosine-dim1024-int8-test_vectors.raw
33+
34+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_single_uint8-cosine-dim1024-M64-efc512-uint8.hnsw_v3
35+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_single_uint8-cosine-dim1024-uint8-test_vectors.raw
36+
37+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_multi_uint8-cosine-dim1024-M64-efc512-uint8.hnsw_v3
38+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_multi_uint8-cosine-dim1024-uint8-test_vectors.raw

‎tests/benchmark/data/hnsw_indices/hnsw_indices_basic_uint8.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_single_uint8-cosine-dim1024-M64-efc512-uint8.hnsw_v3
2+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_single_uint8-cosine-dim1024-uint8-test_vectors.raw
3+
4+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_multi_uint8-cosine-dim1024-M64-efc512-uint8.hnsw_v3
5+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia_multi_uint8-cosine-dim1024-uint8-test_vectors.raw

‎tests/benchmark/data/scripts/serializer.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,21 @@
143143
'type': VecSimType_INT8,
144144
'metric': VecSimMetric_Cosine,
145145
},
146+
{
147+
'filename': 'wikipedia_single-1024_eng_v3_uint8',
148+
'nickname': 'wipedia_single_uint8',
149+
'dim': 1024,
150+
'type': VecSimType_UINT8,
151+
'metric': VecSimMetric_Cosine,
152+
},
153+
{
154+
'filename': 'wikipedia_multi-1024_eng_v3_uint8',
155+
'nickname': 'wipedia_multi_uint8',
156+
'dim': 1024,
157+
'type': VecSimType_UINT8,
158+
'metric': VecSimMetric_Cosine,
159+
'multi': True
160+
},
146161
]
147162

148163
TYPES_ATTR = {

0 commit comments

Comments
 (0)