Skip to content

Commit ad7ba88

Browse files
DaveCTurnerDaveCTurner
authored
Merge branch 'main' into 2025/04/08/newShardSnapshotTask-run-concurrently
2 parents 75b7114 + 4b2867a commit ad7ba88

File tree

94 files changed

+902
-3080
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

94 files changed

+902
-3080
lines changed

‎.buildkite/pipelines/pull-request/build-benchmark.yml‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,19 @@ steps:
77
command: |
88
.ci/scripts/run-gradle.sh :build-tools-internal:bootstrapPerformanceTests
99
.ci/scripts/install-gradle-profiler.sh
10-
.ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-build-benchmark-{{matrix.part}}.scenarios --project-dir . --output-dir profile-out
11-
mkdir build
12-
tar -czf build/$BUILDKITE_BUILD_NUMBER.tar.bz2 profile-out
10+
.ci/scripts/run-gradle-profiler.sh --benchmark --scenario-file build-tools-internal/build/performanceTests/elasticsearch-{{matrix.part}}.scenarios --measure-config-time --project-dir . --output-dir profile-out
11+
mkdir -p build
12+
buildkite-agent artifact upload "profile-out/*"
13+
cat << EOF | buildkite-agent annotate --style "info"
14+
Benchmark Html Report <a href="artifact://profile-out/benchmark.html">Benchmark Html Report</a>
15+
EOF
1316
matrix:
1417
setup:
1518
part:
16-
- part1
17-
- part2
19+
- esql-tests
1820
env:
1921
BUILD_PERFORMANCE_TEST: "true"
22+
2023
agents:
2124
provider: gcp
2225
image: family/elasticsearch-ubuntu-2004

‎.ci/scripts/install-gradle-profiler.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
set -e
44
# profiler version we wanna install
5-
PROFILER_VERSION="0.16.0"
6-
wget https://repo.gradle.org/gradle/ext-releases-local/org/gradle/profiler/gradle-profiler/$PROFILER_VERSION/gradle-profiler-$PROFILER_VERSION.zip -O $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
5+
PROFILER_VERSION="0.22.0"
6+
wget https://repo1.maven.org/maven2/org/gradle/profiler/gradle-profiler/$PROFILER_VERSION/gradle-profiler-$PROFILER_VERSION.zip -O $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
77
unzip $WORKSPACE/gradle-profiler-$PROFILER_VERSION.zip
88
mv $WORKSPACE/gradle-profiler-$PROFILER_VERSION $WORKSPACE/gradle-profiler

‎build-tools-internal/performance/elasticsearch-build-benchmark-part1.scenarios‎

Lines changed: 0 additions & 72 deletions
This file was deleted.

‎build-tools-internal/performance/elasticsearch-build-benchmark-part2.scenarios‎

Lines changed: 0 additions & 38 deletions
This file was deleted.

‎build-tools-internal/performance/elasticsearch-esql-tests.scenarios‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
default-scenarios = ["TEST_ESQL_ABI_CHANGE_TAPI", TEST_ESQL_ABI_CHANGE_TAPI_MAIN, "TEST_ESQL_ABI_CHANGE_TAPI_CC", "TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT", "TEST_ESQL_ABI_CHANGE_TAPI_CC_MAIN", "TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT_MAIN"]
2+
3+
TEST_ESQL_ABI_CHANGE_TAPI {
4+
apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
5+
tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
6+
gradle-args = ["-I", ".ci/init.gradle"]
7+
run-using = tooling-api // value can be "cli" or "tooling-api"
8+
daemon = warm // value can be "warm", "cold", or "none"
9+
copy-file = {
10+
source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
11+
target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
12+
schedule = SCENARIO
13+
}
14+
git-checkout = {
15+
cleanup = "@testGitCommit@"
16+
build = "@testGitCommit@"
17+
}
18+
warm-ups = 3
19+
iterations = 10
20+
}
21+
22+
TEST_ESQL_ABI_CHANGE_TAPI_CC {
23+
apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
24+
tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
25+
gradle-args = ["--configuration-cache", "-I", ".ci/init.gradle"]
26+
run-using = tooling-api // value can be "cli" or "tooling-api"
27+
daemon = warm // value can be "warm", "cold", or "none"
28+
copy-file = {
29+
source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
30+
target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
31+
schedule = SCENARIO
32+
}
33+
34+
git-checkout = {
35+
cleanup = "@testGitCommit@"
36+
build = "@testGitCommit@"
37+
}
38+
warm-ups = 3
39+
iterations = 10
40+
}
41+
42+
TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT {
43+
apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
44+
tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
45+
gradle-args = ["--configuration-cache"]
46+
run-using = tooling-api // value can be "cli" or "tooling-api"
47+
daemon = warm // value can be "warm", "cold", or "none"
48+
copy-file = {
49+
source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
50+
target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
51+
schedule = SCENARIO
52+
}
53+
54+
git-checkout = {
55+
cleanup = "@testGitCommit@"
56+
build = "@testGitCommit@"
57+
}
58+
warm-ups = 3
59+
iterations = 10
60+
}
61+
62+
TEST_ESQL_ABI_CHANGE_TAPI_MAIN {
63+
apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
64+
tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
65+
gradle-args = ["-I", ".ci/init.gradle"]
66+
run-using = tooling-api // value can be "cli" or "tooling-api"
67+
daemon = warm // value can be "warm", "cold", or "none"
68+
copy-file = {
69+
source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
70+
target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
71+
schedule = SCENARIO
72+
}
73+
git-checkout = {
74+
cleanup = "main"
75+
build = "main"
76+
}
77+
warm-ups = 3
78+
iterations = 10
79+
}
80+
81+
TEST_ESQL_ABI_CHANGE_TAPI_CC_MAIN {
82+
apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
83+
tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
84+
gradle-args = ["--configuration-cache", "-I", ".ci/init.gradle"]
85+
run-using = tooling-api // value can be "cli" or "tooling-api"
86+
daemon = warm // value can be "warm", "cold", or "none"
87+
copy-file = {
88+
source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
89+
target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
90+
schedule = SCENARIO
91+
}
92+
93+
git-checkout = {
94+
cleanup = "main"
95+
build = "main"
96+
}
97+
warm-ups = 3
98+
iterations = 10
99+
}
100+
101+
TEST_ESQL_ABI_CHANGE_TAPI_CC_NOINIT_MAIN {
102+
apply-abi-change-to = "x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ColumnInfoImpl.java"
103+
tasks = [":x-pack:plugin:esql:test", "--tests", "org.elasticsearch.gradle.benchmarking.BenchmarkingTests"]
104+
gradle-args = ["--configuration-cache"]
105+
run-using = tooling-api // value can be "cli" or "tooling-api"
106+
daemon = warm // value can be "warm", "cold", or "none"
107+
copy-file = {
108+
source = "build-tools-internal/performance/test-files/BenchmarkingTests.java"
109+
target = "x-pack/plugin/esql/src/test/java/org/elasticsearch/gradle/benchmarking/BenchmarkingTests.java"
110+
schedule = SCENARIO
111+
}
112+
git-checkout = {
113+
cleanup = "main"
114+
build = "main"
115+
}
116+
warm-ups = 3
117+
iterations = 10
118+
}

‎build-tools-internal/performance/test-files/BenchmarkingTests.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.gradle.benchmarking;
9+
10+
import org.elasticsearch.test.ESTestCase;
11+
12+
public class BenchmarkingTests extends ESTestCase {
13+
14+
public void testBenchmarking() {
15+
assertTrue(true);
16+
}
17+
}

‎docs/changelog/123044.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 123044
2+
summary: Adding validation to `ElasticsearchInternalService`
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ public void cleanup() {
146146

147147
public void testRolloverLifecycle() throws Exception {
148148
// empty lifecycle contains the default rollover
149-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
149+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DATA_DEFAULT;
150150

151151
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
152152
String dataStreamName = "metrics-foo";
@@ -353,7 +353,7 @@ public void testOriginationDate() throws Exception {
353353
}
354354

355355
public void testUpdatingLifecycleAppliesToAllBackingIndices() throws Exception {
356-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
356+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DATA_DEFAULT;
357357

358358
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
359359

@@ -384,7 +384,7 @@ public void testAutomaticForceMerge() throws Exception {
384384
* because all necessary merging has already happened automatically. So in order to detect whether forcemerge has been called, we
385385
* use a SendRequestBehavior in the MockTransportService to detect it.
386386
*/
387-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
387+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DATA_DEFAULT;
388388
disableDataStreamLifecycle();
389389
String dataStreamName = "metrics-foo";
390390
putComposableIndexTemplate(
@@ -477,7 +477,7 @@ private static void disableDataStreamLifecycle() {
477477

478478
public void testErrorRecordingOnRollover() throws Exception {
479479
// empty lifecycle contains the default rollover
480-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
480+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DATA_DEFAULT;
481481
/*
482482
* We set index.auto_expand_replicas to 0-1 so that if we get a single-node cluster it is not yellow. The cluster being yellow
483483
* could result in data stream lifecycle's automatic forcemerge failing, which would result in an unexpected error in the error
@@ -613,7 +613,7 @@ public void testErrorRecordingOnRollover() throws Exception {
613613
public void testErrorRecordingOnRetention() throws Exception {
614614
// starting with a lifecycle without retention so we can rollover the data stream and manipulate the second generation index such
615615
// that its retention execution fails
616-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
616+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DATA_DEFAULT;
617617

618618
/*
619619
* We set index.auto_expand_replicas to 0-1 so that if we get a single-node cluster it is not yellow. The cluster being yellow
@@ -750,7 +750,7 @@ public void testErrorRecordingOnRetention() throws Exception {
750750
}
751751

752752
public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception {
753-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
753+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DATA_DEFAULT;
754754

755755
putComposableIndexTemplate(
756756
"id1",

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ public void cleanup() {
9292

9393
public void testExplainLifecycle() throws Exception {
9494
// empty lifecycle contains the default rollover
95-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
95+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DATA_DEFAULT;
9696

9797
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle);
9898
String dataStreamName = "metrics-foo";
@@ -264,7 +264,7 @@ public void testExplainFailuresLifecycle() throws Exception {
264264
List.of("metrics-foo*"),
265265
null,
266266
null,
267-
DataStreamLifecycle.Template.DEFAULT,
267+
DataStreamLifecycle.Template.DATA_DEFAULT,
268268
new DataStreamOptions.Template(new DataStreamFailureStore.Template(true))
269269
);
270270
String dataStreamName = "metrics-foo";
@@ -381,7 +381,7 @@ public void testExplainFailuresLifecycle() throws Exception {
381381

382382
public void testExplainLifecycleForIndicesWithErrors() throws Exception {
383383
// empty lifecycle contains the default rollover
384-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
384+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DATA_DEFAULT;
385385

386386
putComposableIndexTemplate(
387387
"id1",

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ public void testLifecycleComposition() {
156156
// Enabled is always true unless it's explicitly set to false
157157
{
158158
DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
159-
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DEFAULT);
159+
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DATA_DEFAULT);
160160
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
161161
assertThat(result.enabled(), equalTo(true));
162162
assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get()));

0 commit comments

Comments
 (0)