Skip to content

Commit e4d293c

Browse files
mjmbischoffmjmbischoff
authored
Merge branch 'main' into esql-string-contains
2 parents 60ebb6b + fedf88f commit e4d293c

File tree

422 files changed

+15016
-3920
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

422 files changed

+15016
-3920
lines changed

‎.buildkite/pipelines/pull-request-performance-benchmark.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
steps:
2+
- label: ":pipeline: Generate steps"
3+
command: bash .buildkite/scripts/generate-pr-performance-benchmark.sh | buildkite-agent pipeline upload

‎.buildkite/pipelines/pull-request/performance-benchmark.yml‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎.buildkite/pull-requests.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
"set_commit_status": false,
2828
"build_on_commit": false,
2929
"build_on_comment": true,
30-
"trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
30+
"target_branch": "main",
31+
"trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\S+)( please)?$"
3132
}
3233
]
3334
}

‎.buildkite/scripts/generate-pr-performance-benchmark.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
env_id_baseline=$(python3 -c 'import uuid; print(uuid.uuid4())')
6+
env_id_contender=$(python3 -c 'import uuid; print(uuid.uuid4())')
7+
merge_base=$(git merge-base "${GITHUB_PR_TARGET_BRANCH}" HEAD)
8+
9+
buildkite-agent meta-data set pr_comment:custom-body:body \
10+
"This build attempted two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
11+
buildkite-agent meta-data set pr_comment:custom-baseline:head \
12+
"* Baseline: ${merge_base} (env ID ${env_id_baseline})"
13+
buildkite-agent meta-data set pr_comment:custom-contender:head \
14+
"* Contender: ${GITHUB_PR_TRIGGERED_SHA} (env ID ${env_id_contender})"
15+
16+
cat << _EOF_
17+
steps:
18+
- label: Trigger baseline benchmark
19+
trigger: elasticsearch-performance-esbench-pr
20+
build:
21+
message: Baseline benchmark for PR${GITHUB_PR_NUMBER}
22+
branch: master
23+
env:
24+
CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
25+
ENV_ID: ${env_id_baseline}
26+
REVISION: ${merge_base}
27+
- label: Trigger contender benchmark
28+
trigger: elasticsearch-performance-esbench-pr
29+
build:
30+
message: Contender benchmark for PR${GITHUB_PR_NUMBER}
31+
branch: master
32+
env:
33+
CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
34+
ENV_ID: ${env_id_contender}
35+
ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
36+
REVISION: ${GITHUB_PR_TRIGGERED_SHA}
37+
- wait: ~
38+
- label: Modify PR comment
39+
command: buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
40+
_EOF_

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11

2+
# claude
3+
.claude
4+
25
# intellij files
36
.idea/
47
*.iml

‎benchmarks/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ dependencies {
4747
api(project(':x-pack:plugin:core'))
4848
api(project(':x-pack:plugin:esql'))
4949
api(project(':x-pack:plugin:esql:compute'))
50+
api(project(':x-pack:plugin:mapper-exponential-histogram'))
5051
implementation project(path: ':libs:native')
5152
implementation project(path: ':libs:simdvec')
5253
implementation project(path: ':libs:exponential-histogram')

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,11 +142,26 @@ static void selfTest() {
142142
private static List<ValuesSourceReaderOperator.FieldInfo> fields(String name) {
143143
return switch (name) {
144144
case "3_stored_keywords" -> List.of(
145-
new ValuesSourceReaderOperator.FieldInfo("keyword_1", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_1")),
146-
new ValuesSourceReaderOperator.FieldInfo("keyword_2", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_2")),
147-
new ValuesSourceReaderOperator.FieldInfo("keyword_3", ElementType.BYTES_REF, shardIdx -> blockLoader("stored_keyword_3"))
145+
new ValuesSourceReaderOperator.FieldInfo(
146+
"keyword_1",
147+
ElementType.BYTES_REF,
148+
false,
149+
shardIdx -> blockLoader("stored_keyword_1")
150+
),
151+
new ValuesSourceReaderOperator.FieldInfo(
152+
"keyword_2",
153+
ElementType.BYTES_REF,
154+
false,
155+
shardIdx -> blockLoader("stored_keyword_2")
156+
),
157+
new ValuesSourceReaderOperator.FieldInfo(
158+
"keyword_3",
159+
ElementType.BYTES_REF,
160+
false,
161+
shardIdx -> blockLoader("stored_keyword_3")
162+
)
148163
);
149-
default -> List.of(new ValuesSourceReaderOperator.FieldInfo(name, elementType(name), shardIdx -> blockLoader(name)));
164+
default -> List.of(new ValuesSourceReaderOperator.FieldInfo(name, elementType(name), false, shardIdx -> blockLoader(name)));
150165
};
151166
}
152167

‎benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java‎

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,15 @@
99

1010
package org.elasticsearch.benchmark.exponentialhistogram;
1111

12+
import org.apache.lucene.util.BytesRef;
13+
import org.elasticsearch.common.io.stream.BytesStreamOutput;
1214
import org.elasticsearch.exponentialhistogram.BucketIterator;
1315
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
1416
import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
1517
import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
1618
import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger;
19+
import org.elasticsearch.xpack.exponentialhistogram.CompressedExponentialHistogram;
20+
import org.elasticsearch.xpack.exponentialhistogram.IndexWithCount;
1721
import org.openjdk.jmh.annotations.Benchmark;
1822
import org.openjdk.jmh.annotations.BenchmarkMode;
1923
import org.openjdk.jmh.annotations.Fork;
@@ -27,6 +31,8 @@
2731
import org.openjdk.jmh.annotations.Threads;
2832
import org.openjdk.jmh.annotations.Warmup;
2933

34+
import java.io.IOException;
35+
import java.util.ArrayList;
3036
import java.util.List;
3137
import java.util.Random;
3238
import java.util.concurrent.ThreadLocalRandom;
@@ -47,6 +53,9 @@ public class ExponentialHistogramMergeBench {
4753
@Param({ "0.01", "0.1", "0.25", "0.5", "1.0", "2.0" })
4854
double mergedHistoSizeFactor;
4955

56+
@Param({ "array-backed", "compressed" })
57+
String histoImplementation;
58+
5059
Random random;
5160
ExponentialHistogramMerger histoMerger;
5261

@@ -81,16 +90,54 @@ public void setUp() {
8190
bucketIndex += 1 + random.nextInt(bucketCount) % (Math.max(1, bucketCount / dataPointSize));
8291
generator.add(Math.pow(1.001, bucketIndex));
8392
}
84-
toMerge[i] = generator.getAndClear();
85-
cnt = getBucketCount(toMerge[i]);
93+
ExponentialHistogram histogram = generator.getAndClear();
94+
cnt = getBucketCount(histogram);
8695
if (cnt < dataPointSize) {
87-
throw new IllegalArgumentException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);
96+
throw new IllegalStateException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);
97+
}
98+
99+
if ("array-backed".equals(histoImplementation)) {
100+
toMerge[i] = histogram;
101+
} else if ("compressed".equals(histoImplementation)) {
102+
toMerge[i] = asCompressedHistogram(histogram);
103+
} else {
104+
throw new IllegalArgumentException("Unknown implementation: " + histoImplementation);
88105
}
89106
}
90107

91108
index = 0;
92109
}
93110

111+
private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogram) {
112+
List<IndexWithCount> negativeBuckets = new ArrayList<>();
113+
List<IndexWithCount> positiveBuckets = new ArrayList<>();
114+
115+
BucketIterator it = histogram.negativeBuckets().iterator();
116+
while (it.hasNext()) {
117+
negativeBuckets.add(new IndexWithCount(it.peekIndex(), it.peekCount()));
118+
it.advance();
119+
}
120+
it = histogram.positiveBuckets().iterator();
121+
while (it.hasNext()) {
122+
positiveBuckets.add(new IndexWithCount(it.peekIndex(), it.peekCount()));
123+
it.advance();
124+
}
125+
126+
long totalCount = histogram.zeroBucket().count() + histogram.negativeBuckets().valueCount() + histogram.positiveBuckets()
127+
.valueCount();
128+
BytesStreamOutput histoBytes = new BytesStreamOutput();
129+
try {
130+
CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
131+
CompressedExponentialHistogram result = new CompressedExponentialHistogram();
132+
BytesRef data = histoBytes.bytes().toBytesRef();
133+
result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, data);
134+
return result;
135+
} catch (IOException e) {
136+
throw new RuntimeException(e);
137+
}
138+
139+
}
140+
94141
private static int getBucketCount(ExponentialHistogram histo) {
95142
int cnt = 0;
96143
for (BucketIterator it : List.of(histo.negativeBuckets().iterator(), histo.positiveBuckets().iterator())) {

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
258258
);
259259
config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
260260
config.setMergePolicy(new LogByteSizeMergePolicy());
261-
var docValuesFormat = new ES819TSDBDocValuesFormat(4096, optimizedMergeEnabled);
261+
var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
262262
config.setCodec(new Elasticsearch900Lucene101Codec() {
263263

264264
@Override

‎build-tools-internal/gradle/wrapper/gradle-wrapper.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
distributionBase=GRADLE_USER_HOME
22
distributionPath=wrapper/dists
3-
distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
4-
distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
3+
distributionSha256Sum=f759b8dd5204e2e3fa4ca3e73f452f087153cf81bac9561eeb854229cc2c5365
4+
distributionUrl=https\://services.gradle.org/distributions/gradle-9.0.0-all.zip
55
networkTimeout=10000
66
validateDistributionUrl=true
77
zipStoreBase=GRADLE_USER_HOME

0 commit comments

Comments
 (0)