Skip to content

Commit 85749d6

Browse files
nik9000nik9000
authored
Add benchmark script (#126596)
Adds a simple script to run benchmarks for ESQL and collect their results. The script has a `--test` mode which takes about ten minutes. Running without `--test` takes a four hours fifteen minutes. To speed up `--test` I reworked the "self test" that each benchmark runs to be optional and disabled in `--test` mode.
1 parent d870f42 commit 85749d6

16 files changed

+255
-31
lines changed

‎benchmarks/run.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
#
3+
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
4+
# or more contributor license agreements. Licensed under the "Elastic License
5+
# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
6+
# Public License v 1"; you may not use this file except in compliance with, at
7+
# your election, the "Elastic License 2.0", the "GNU Affero General Public
8+
# License v3.0 only", or the "Server Side Public License, v 1".
9+
#
10+
11+
EXTRA=""
12+
POSITIONAL_ARGS=()
13+
while [[ $# -gt 0 ]]; do
14+
case $1 in
15+
--test)
16+
# Get inaccurate results quickly by shortening all measurements
17+
# to 50ms each and skip self tests.
18+
EXTRA="-r 50ms -w 50ms -jvmArgsAppend -DskipSelfTest=true"
19+
shift
20+
;;
21+
*)
22+
POSITIONAL_ARGS+=("$1")
23+
shift
24+
;;
25+
esac
26+
done
27+
28+
set -- "${POSITIONAL_ARGS[@]}"
29+
30+
run() {
31+
../gradlew run --args "$2 -rf json $EXTRA"
32+
mv jmh-result.json build/benchmarks/$1.json
33+
}
34+
35+
cd "$(dirname "$0")"
36+
mkdir -p build/benchmarks
37+
run 'esql_agg' 'AggregatorBenchmark -pgrouping=none,longs -pfilter=none -pblockType=vector_longs,half_null_longs'
38+
run 'esql_block_keep_mask' 'BlockKeepMaskBenchmark -pdataTypeAndBlockKind=BytesRef/array,BytesRef/vector,long/array,long/vector'
39+
run 'esql_block_read' 'BlockReadBenchmark -paccessType=sequential'
40+
run 'esql_eval' 'EvalBenchmark'
41+
run 'esql_parse_ip' 'ParseIpBenchmark'
42+
run 'esql_topn' 'TopNBenchmark'
43+
run 'esql_values_agg' 'ValuesAggregatorBenchmark'
44+
run 'esql_values_source_reader' 'ValuesSourceReaderBenchmark'

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,12 @@ public class AggregatorBenchmark {
113113

114114
static {
115115
// Smoke test all the expected values and force loading subclasses more like prod
116+
if (false == "true".equals(System.getProperty("skipSelfTest"))) {
117+
selfTest();
118+
}
119+
}
120+
121+
static void selfTest() {
116122
try {
117123
for (String grouping : AggregatorBenchmark.class.getField("grouping").getAnnotationsByType(Param.class)[0].value()) {
118124
for (String op : AggregatorBenchmark.class.getField("op").getAnnotationsByType(Param.class)[0].value()) {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java‎

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import java.util.ArrayList;
3838
import java.util.BitSet;
3939
import java.util.Random;
40+
import java.util.stream.IntStream;
4041

4142
public class BlockBenchmark {
4243
/**
@@ -112,15 +113,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
112113
blocks[blockIndex] = blockFactory.newBooleanArrayBlock(
113114
values,
114115
totalPositions,
115-
null,
116+
IntStream.rangeClosed(0, totalPositions).toArray(),
116117
null,
117118
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
118119
);
119120
}
120121
case "array-multivalue-null" -> {
121122
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
122123
int positionCount = firstValueIndexes.length - 1;
123-
BitSet nulls = randomNulls(positionCount);
124+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
124125

125126
blocks[blockIndex] = blockFactory.newBooleanArrayBlock(
126127
values,
@@ -141,7 +142,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
141142
blocks[blockIndex] = new BooleanBigArrayBlock(
142143
valuesBigArray,
143144
totalPositions,
144-
null,
145+
IntStream.rangeClosed(0, totalPositions).toArray(),
145146
null,
146147
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
147148
blockFactory
@@ -150,7 +151,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
150151
case "big-array-multivalue-null" -> {
151152
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
152153
int positionCount = firstValueIndexes.length - 1;
153-
BitSet nulls = randomNulls(positionCount);
154+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
154155
BitArray valuesBigArray = new BitArray(totalPositions, BigArrays.NON_RECYCLING_INSTANCE);
155156
for (int i = 0; i < values.length; i++) {
156157
if (values[i]) {
@@ -211,15 +212,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
211212
blocks[blockIndex] = blockFactory.newBytesRefArrayBlock(
212213
values,
213214
totalPositions,
214-
null,
215+
IntStream.rangeClosed(0, totalPositions).toArray(),
215216
null,
216217
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
217218
);
218219
}
219220
case "array-multivalue-null" -> {
220221
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
221222
int positionCount = firstValueIndexes.length - 1;
222-
BitSet nulls = randomNulls(positionCount);
223+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
223224

224225
blocks[blockIndex] = blockFactory.newBytesRefArrayBlock(
225226
values,
@@ -257,15 +258,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
257258
blocks[blockIndex] = blockFactory.newDoubleArrayBlock(
258259
values,
259260
totalPositions,
260-
null,
261+
IntStream.rangeClosed(0, totalPositions).toArray(),
261262
null,
262263
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
263264
);
264265
}
265266
case "array-multivalue-null" -> {
266267
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
267268
int positionCount = firstValueIndexes.length - 1;
268-
BitSet nulls = randomNulls(positionCount);
269+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
269270

270271
blocks[blockIndex] = blockFactory.newDoubleArrayBlock(
271272
values,
@@ -284,7 +285,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
284285
blocks[blockIndex] = new DoubleBigArrayBlock(
285286
valuesBigArray,
286287
totalPositions,
287-
null,
288+
IntStream.rangeClosed(0, totalPositions).toArray(),
288289
null,
289290
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
290291
blockFactory
@@ -293,7 +294,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
293294
case "big-array-multivalue-null" -> {
294295
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
295296
int positionCount = firstValueIndexes.length - 1;
296-
BitSet nulls = randomNulls(positionCount);
297+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
297298
DoubleArray valuesBigArray = blockFactory.bigArrays().newDoubleArray(totalPositions, false);
298299
for (int i = 0; i < values.length; i++) {
299300
valuesBigArray.set(i, values[i]);
@@ -344,15 +345,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
344345
blocks[blockIndex] = blockFactory.newIntArrayBlock(
345346
values,
346347
totalPositions,
347-
null,
348+
IntStream.rangeClosed(0, totalPositions).toArray(),
348349
null,
349350
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
350351
);
351352
}
352353
case "array-multivalue-null" -> {
353354
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
354355
int positionCount = firstValueIndexes.length - 1;
355-
BitSet nulls = randomNulls(positionCount);
356+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
356357

357358
blocks[blockIndex] = blockFactory.newIntArrayBlock(
358359
values,
@@ -371,7 +372,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
371372
blocks[blockIndex] = new IntBigArrayBlock(
372373
valuesBigArray,
373374
totalPositions,
374-
null,
375+
IntStream.rangeClosed(0, totalPositions).toArray(),
375376
null,
376377
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
377378
blockFactory
@@ -380,7 +381,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
380381
case "big-array-multivalue-null" -> {
381382
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
382383
int positionCount = firstValueIndexes.length - 1;
383-
BitSet nulls = randomNulls(positionCount);
384+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
384385
IntArray valuesBigArray = blockFactory.bigArrays().newIntArray(totalPositions, false);
385386
for (int i = 0; i < values.length; i++) {
386387
valuesBigArray.set(i, values[i]);
@@ -431,15 +432,15 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
431432
blocks[blockIndex] = blockFactory.newLongArrayBlock(
432433
values,
433434
totalPositions,
434-
null,
435+
IntStream.rangeClosed(0, totalPositions).toArray(),
435436
null,
436437
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING
437438
);
438439
}
439440
case "array-multivalue-null" -> {
440441
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
441442
int positionCount = firstValueIndexes.length - 1;
442-
BitSet nulls = randomNulls(positionCount);
443+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
443444

444445
blocks[blockIndex] = blockFactory.newLongArrayBlock(
445446
values,
@@ -458,7 +459,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
458459
blocks[blockIndex] = new LongBigArrayBlock(
459460
valuesBigArray,
460461
totalPositions,
461-
null,
462+
IntStream.rangeClosed(0, totalPositions).toArray(),
462463
null,
463464
Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING,
464465
blockFactory
@@ -467,7 +468,7 @@ static Block[] buildBlocks(String dataType, String blockKind, int totalPositions
467468
case "big-array-multivalue-null" -> {
468469
int[] firstValueIndexes = randomFirstValueIndexes(totalPositions);
469470
int positionCount = firstValueIndexes.length - 1;
470-
BitSet nulls = randomNulls(positionCount);
471+
BitSet nulls = nullsFromFirstValues(firstValueIndexes);
471472
LongArray valuesBigArray = blockFactory.bigArrays().newLongArray(totalPositions, false);
472473
for (int i = 0; i < values.length; i++) {
473474
valuesBigArray.set(i, values[i]);
@@ -526,10 +527,10 @@ private static int[] randomFirstValueIndexes(int totalPositions) {
526527
return firstValueIndexes.stream().mapToInt(x -> x).toArray();
527528
}
528529

529-
private static BitSet randomNulls(int positionCount) {
530-
BitSet nulls = new BitSet(positionCount);
531-
for (int i = 0; i < positionCount; i++) {
532-
if (random.nextDouble() < NULL_PERCENTAGE) {
530+
private static BitSet nullsFromFirstValues(int[] firstValueIndexes) {
531+
BitSet nulls = new BitSet(firstValueIndexes.length - 1);
532+
for (int i = 0; i < firstValueIndexes.length - 1; i++) {
533+
if (firstValueIndexes[i + 1] - firstValueIndexes[i] == 1 && random.nextDouble() < NULL_PERCENTAGE) {
533534
nulls.set(i);
534535
}
535536
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,13 @@
4242
@Fork(1)
4343
public class BlockKeepMaskBenchmark extends BlockBenchmark {
4444
static {
45-
// Smoke test all the expected values and force loading subclasses more like prod
45+
if (false == "true".equals(System.getProperty("skipSelfTest"))) {
46+
// Smoke test all the expected values and force loading subclasses more like prod
47+
selfTest();
48+
}
49+
}
50+
51+
static void selfTest() {
4652
int totalPositions = 10;
4753
for (String paramString : RELEVANT_TYPE_BLOCK_COMBINATIONS) {
4854
String[] params = paramString.split("/");

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,13 @@
2626
@Fork(1)
2727
public class BlockReadBenchmark extends BlockBenchmark {
2828
static {
29+
if (false == "true".equals(System.getProperty("skipSelfTest"))) {
30+
// Smoke test all the expected values and force loading subclasses more like prod
31+
selfTest();
32+
}
33+
}
34+
35+
static void selfTest() {
2936
// Smoke test all the expected values and force loading subclasses more like prod
3037
int totalPositions = 10;
3138
long[] actualCheckSums = new long[NUM_BLOCKS_PER_ITERATION];

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,10 @@ public class EvalBenchmark {
9494

9595
static {
9696
LogConfigurator.configureESLogging();
97-
// Smoke test all the expected values and force loading subclasses more like prod
98-
selfTest();
97+
if (false == "true".equals(System.getProperty("skipSelfTest"))) {
98+
// Smoke test all the expected values and force loading subclasses more like prod
99+
selfTest();
100+
}
99101
}
100102

101103
static void selfTest() {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/TopNBenchmark.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,10 @@ public class TopNBenchmark {
6969

7070
static {
7171
// Smoke test all the expected values and force loading subclasses more like prod
72+
selfTest();
73+
}
74+
75+
static void selfTest() {
7276
try {
7377
for (String data : TopNBenchmark.class.getField("data").getAnnotationsByType(Param.class)[0].value()) {
7478
for (String topCount : TopNBenchmark.class.getField("topCount").getAnnotationsByType(Param.class)[0].value()) {
@@ -98,8 +102,8 @@ private static Operator operator(String data, int topCount) {
98102
case DOUBLES -> List.of(ElementType.DOUBLE);
99103
case BOOLEANS -> List.of(ElementType.BOOLEAN);
100104
case BYTES_REFS -> List.of(ElementType.BYTES_REF);
101-
case TWO_LONGS -> List.of(ElementType.INT, ElementType.INT);
102-
case LONGS_AND_BYTES_REFS -> List.of(ElementType.INT, ElementType.BYTES_REF);
105+
case TWO_LONGS -> List.of(ElementType.LONG, ElementType.LONG);
106+
case LONGS_AND_BYTES_REFS -> List.of(ElementType.LONG, ElementType.BYTES_REF);
103107
default -> throw new IllegalArgumentException("unsupported data type [" + data + "]");
104108
};
105109
List<TopNEncoder> encoders = switch (data) {
@@ -127,7 +131,7 @@ private static Operator operator(String data, int topCount) {
127131
}
128132

129133
private static void checkExpected(int topCount, List<Page> pages) {
130-
if (topCount != pages.size()) {
134+
if (topCount != pages.stream().mapToLong(Page::getPositionCount).sum()) {
131135
throw new AssertionError("expected [" + topCount + "] but got [" + pages.size() + "]");
132136
}
133137
}
@@ -191,7 +195,7 @@ private static void run(String data, int topCount) {
191195
try (Operator operator = operator(data, topCount)) {
192196
Page page = page(data);
193197
for (int i = 0; i < 1024; i++) {
194-
operator.addInput(page);
198+
operator.addInput(page.shallowCopy());
195199
}
196200
operator.finish();
197201
List<Page> results = new ArrayList<>();

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,13 @@ public class ValuesAggregatorBenchmark {
8181
);
8282

8383
static {
84+
if (false == "true".equals(System.getProperty("skipSelfTest"))) {
85+
// Smoke test all the expected values and force loading subclasses more like prod
86+
selfTest();
87+
}
88+
}
89+
90+
static void selfTest() {
8491
// Smoke test all the expected values and force loading subclasses more like prod
8592
try {
8693
for (String groups : ValuesAggregatorBenchmark.class.getField("groups").getAnnotationsByType(Param.class)[0].value()) {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,10 @@ public class ValuesSourceReaderBenchmark {
9292

9393
static {
9494
// Smoke test all the expected values and force loading subclasses more like prod
95+
selfTest();
96+
}
97+
98+
static void selfTest() {
9599
try {
96100
ValuesSourceReaderBenchmark benchmark = new ValuesSourceReaderBenchmark();
97101
benchmark.setupIndex();
@@ -263,7 +267,42 @@ private static BlockLoader numericBlockLoader(WhereAndBaseName w, NumberFieldMap
263267
null,
264268
null,
265269
false
266-
).blockLoader(null);
270+
).blockLoader(new MappedFieldType.BlockLoaderContext() {
271+
@Override
272+
public String indexName() {
273+
return "benchmark";
274+
}
275+
276+
@Override
277+
public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
278+
return MappedFieldType.FieldExtractPreference.NONE;
279+
}
280+
281+
@Override
282+
public IndexSettings indexSettings() {
283+
throw new UnsupportedOperationException();
284+
}
285+
286+
@Override
287+
public SearchLookup lookup() {
288+
throw new UnsupportedOperationException();
289+
}
290+
291+
@Override
292+
public Set<String> sourcePaths(String name) {
293+
throw new UnsupportedOperationException();
294+
}
295+
296+
@Override
297+
public String parentField(String field) {
298+
throw new UnsupportedOperationException();
299+
}
300+
301+
@Override
302+
public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
303+
throw new UnsupportedOperationException();
304+
}
305+
});
267306
}
268307

269308
/**

0 commit comments

Comments
 (0)