Skip to content

Commit cdbba96

Browse files
ywangdywangd
committed
Merge remote-tracking branch 'origin/main' into per-project-s3-clients
2 parents 5fc8f5a + 4985a61 commit cdbba96

File tree

353 files changed

+13018
-4356
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

353 files changed

+13018
-4356
lines changed

‎benchmarks/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ dependencies {
4242
api(project(':libs:h3'))
4343
api(project(':modules:aggregations'))
4444
api(project(':x-pack:plugin:esql-core'))
45+
api(project(':x-pack:plugin:core'))
4546
api(project(':x-pack:plugin:esql'))
4647
api(project(':x-pack:plugin:esql:compute'))
4748
implementation project(path: ':libs:simdvec')

‎benchmarks/src/main/java/org/elasticsearch/benchmark/esql/QueryPlanningBenchmark.java‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.esql;
11+
12+
import org.elasticsearch.common.logging.LogConfigurator;
13+
import org.elasticsearch.common.settings.Settings;
14+
import org.elasticsearch.index.IndexMode;
15+
import org.elasticsearch.license.XPackLicenseState;
16+
import org.elasticsearch.xpack.esql.analysis.Analyzer;
17+
import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
18+
import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
19+
import org.elasticsearch.xpack.esql.analysis.Verifier;
20+
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
21+
import org.elasticsearch.xpack.esql.core.type.EsField;
22+
import org.elasticsearch.xpack.esql.core.util.DateUtils;
23+
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
24+
import org.elasticsearch.xpack.esql.index.EsIndex;
25+
import org.elasticsearch.xpack.esql.index.IndexResolution;
26+
import org.elasticsearch.xpack.esql.inference.InferenceResolution;
27+
import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
28+
import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
29+
import org.elasticsearch.xpack.esql.parser.EsqlParser;
30+
import org.elasticsearch.xpack.esql.parser.QueryParams;
31+
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
32+
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
33+
import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
34+
import org.elasticsearch.xpack.esql.session.Configuration;
35+
import org.elasticsearch.xpack.esql.telemetry.Metrics;
36+
import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
37+
import org.openjdk.jmh.annotations.Benchmark;
38+
import org.openjdk.jmh.annotations.BenchmarkMode;
39+
import org.openjdk.jmh.annotations.Fork;
40+
import org.openjdk.jmh.annotations.Measurement;
41+
import org.openjdk.jmh.annotations.Mode;
42+
import org.openjdk.jmh.annotations.OutputTimeUnit;
43+
import org.openjdk.jmh.annotations.Scope;
44+
import org.openjdk.jmh.annotations.Setup;
45+
import org.openjdk.jmh.annotations.State;
46+
import org.openjdk.jmh.annotations.Warmup;
47+
import org.openjdk.jmh.infra.Blackhole;
48+
49+
import java.util.LinkedHashMap;
50+
import java.util.Locale;
51+
import java.util.Map;
52+
import java.util.concurrent.TimeUnit;
53+
54+
import static java.util.Collections.emptyMap;
55+
import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
56+
57+
@Fork(1)
58+
@Warmup(iterations = 5)
59+
@Measurement(iterations = 10)
60+
@BenchmarkMode(Mode.AverageTime)
61+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
62+
@State(Scope.Benchmark)
63+
public class QueryPlanningBenchmark {
64+
65+
static {
66+
LogConfigurator.configureESLogging();
67+
}
68+
69+
private PlanTelemetry telemetry;
70+
private EsqlParser parser;
71+
private Analyzer analyzer;
72+
private LogicalPlanOptimizer optimizer;
73+
74+
@Setup
75+
public void setup() {
76+
77+
var config = new Configuration(
78+
DateUtils.UTC,
79+
Locale.US,
80+
null,
81+
null,
82+
new QueryPragmas(Settings.EMPTY),
83+
EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
84+
EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
85+
"",
86+
false,
87+
Map.of(),
88+
System.nanoTime(),
89+
false
90+
);
91+
92+
var fields = 10_000;
93+
var mapping = LinkedHashMap.<String, EsField>newLinkedHashMap(fields);
94+
for (int i = 0; i < fields; i++) {
95+
mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true));
96+
}
97+
98+
var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
99+
100+
var functionRegistry = new EsqlFunctionRegistry();
101+
102+
telemetry = new PlanTelemetry(functionRegistry);
103+
parser = new EsqlParser();
104+
analyzer = new Analyzer(
105+
new AnalyzerContext(
106+
config,
107+
functionRegistry,
108+
IndexResolution.valid(esIndex),
109+
Map.of(),
110+
new EnrichResolution(),
111+
InferenceResolution.EMPTY
112+
),
113+
new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
114+
);
115+
optimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
116+
}
117+
118+
private LogicalPlan plan(String query) {
119+
var parsed = parser.createStatement(query, new QueryParams(), telemetry);
120+
var analyzed = analyzer.analyze(parsed);
121+
var optimized = optimizer.optimize(analyzed);
122+
return optimized;
123+
}
124+
125+
@Benchmark
126+
public void run(Blackhole blackhole) {
127+
blackhole.consume(plan("FROM test | LIMIT 10"));
128+
}
129+
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
import org.elasticsearch.cluster.metadata.DataStream;
1717
import org.elasticsearch.cluster.metadata.IndexMetadata;
1818
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
19-
import org.elasticsearch.cluster.metadata.Metadata;
19+
import org.elasticsearch.cluster.metadata.ProjectId;
20+
import org.elasticsearch.cluster.metadata.ProjectMetadata;
2021
import org.elasticsearch.cluster.project.DefaultProjectResolver;
2122
import org.elasticsearch.common.settings.Settings;
2223
import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -63,12 +64,12 @@ public void setUp() {
6364
int numDataStreams = toInt(params[0]);
6465
int numIndices = toInt(params[1]);
6566

66-
Metadata.Builder mb = Metadata.builder();
67+
ProjectMetadata.Builder pmb = ProjectMetadata.builder(ProjectId.DEFAULT);
6768
String[] indices = new String[numIndices + numDataStreams * (numIndices + 1)];
6869
int position = 0;
6970
for (int i = 1; i <= numIndices; i++) {
7071
String indexName = INDEX_PREFIX + i;
71-
createIndexMetadata(indexName, mb);
72+
createIndexMetadata(indexName, pmb);
7273
indices[position++] = indexName;
7374
}
7475

@@ -77,14 +78,14 @@ public void setUp() {
7778
List<Index> backingIndices = new ArrayList<>();
7879
for (int j = 1; j <= numIndices; j++) {
7980
String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, j);
80-
backingIndices.add(createIndexMetadata(backingIndexName, mb).getIndex());
81+
backingIndices.add(createIndexMetadata(backingIndexName, pmb).getIndex());
8182
indices[position++] = backingIndexName;
8283
}
8384
indices[position++] = dataStreamName;
84-
mb.put(DataStream.builder(dataStreamName, backingIndices).build());
85+
pmb.put(DataStream.builder(dataStreamName, backingIndices).build());
8586
}
8687
int mid = indices.length / 2;
87-
clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build();
88+
clusterState = ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(pmb).build();
8889
resolver = new IndexNameExpressionResolver(
8990
new ThreadContext(Settings.EMPTY),
9091
new SystemIndices(List.of()),
@@ -97,13 +98,13 @@ public void setUp() {
9798
mixedRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), mixed);
9899
}
99100

100-
private IndexMetadata createIndexMetadata(String indexName, Metadata.Builder mb) {
101+
private IndexMetadata createIndexMetadata(String indexName, ProjectMetadata.Builder pmb) {
101102
IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
102103
.settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
103104
.numberOfShards(1)
104105
.numberOfReplicas(0)
105106
.build();
106-
mb.put(indexMetadata, false);
107+
pmb.put(indexMetadata, false);
107108
return indexMetadata;
108109
}
109110

‎benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@
1212
import org.elasticsearch.cluster.ClusterName;
1313
import org.elasticsearch.cluster.ClusterState;
1414
import org.elasticsearch.cluster.metadata.IndexMetadata;
15-
import org.elasticsearch.cluster.metadata.Metadata;
15+
import org.elasticsearch.cluster.metadata.ProjectId;
16+
import org.elasticsearch.cluster.metadata.ProjectMetadata;
1617
import org.elasticsearch.cluster.node.DiscoveryNodes;
1718
import org.elasticsearch.cluster.project.DefaultProjectResolver;
1819
import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -96,7 +97,7 @@ public void setUp() throws Exception {
9697

9798
AllocationService allocationService = Allocators.createAllocationService(Settings.EMPTY);
9899

99-
Metadata.Builder mb = Metadata.builder();
100+
ProjectMetadata.Builder pmb = ProjectMetadata.builder(ProjectId.DEFAULT);
100101
RoutingTable.Builder rb = RoutingTable.builder();
101102

102103
DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
@@ -160,12 +161,12 @@ public void setUp() throws Exception {
160161
}
161162

162163
routingTable.add(indexRountingTableBuilder);
163-
mb.put(indexMetadata, false);
164+
pmb.put(indexMetadata, false);
164165
}
165166

166167
ClusterState initialClusterState = ClusterState.builder(ClusterName.DEFAULT)
167-
.metadata(mb)
168-
.routingTable(routingTable)
168+
.putProjectMetadata(pmb)
169+
.putRoutingTable(pmb.getId(), routingTable.build())
169170
.nodes(nb)
170171
.build();
171172

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,17 @@ public enum DockerBase {
2828
"apk",
2929
"Dockerfile"
3030
),
31-
FIPS(
32-
"docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
33-
"-fips",
34-
"apk",
35-
"Dockerfile"
36-
),
3731
// spotless:on
3832
// Based on WOLFI above, with more extras. We don't set a base image because
3933
// we programmatically extend from the wolfi image.
40-
CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.cloud-ess"),;
34+
CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.ess"),
35+
36+
CLOUD_ESS_FIPS(
37+
"docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
38+
"-cloud-ess-fips",
39+
"apk",
40+
"Dockerfile.ess-fips"
41+
);
4142

4243
private final String image;
4344
private final String suffix;

‎build-tools-internal/version.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
elasticsearch = 9.1.0
2-
lucene = 10.2.1-snapshot-ae6484f43e6
2+
lucene = 10.2.1
33

44
bundled_jdk_vendor = openjdk
55
bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90

‎distribution/docker/build.gradle‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,7 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base, Strin
314314
filter TransformLog4jConfigFilter
315315
}
316316
}
317-
if(base == DockerBase.FIPS) {
317+
if(base == DockerBase.CLOUD_ESS_FIPS) {
318318

319319
// If we're performing a release build, but `build.id` hasn't been set, we can
320320
// infer that we're not at the Docker building stage of the build, and therefore
@@ -608,19 +608,19 @@ subprojects { Project subProject ->
608608
DockerBase base = DockerBase.DEFAULT
609609
if (subProject.name.contains('ironbank-')) {
610610
base = DockerBase.IRON_BANK
611-
} else if (subProject.name.contains('cloud-ess-')) {
611+
} else if (subProject.name.contains('cloud-ess-docker')) {
612612
base = DockerBase.CLOUD_ESS
613613
} else if (subProject.name.contains('wolfi-')) {
614614
base = DockerBase.WOLFI
615-
} else if (subProject.name.contains('fips-')) {
616-
base = DockerBase.FIPS
615+
} else if (subProject.name.contains('cloud-ess-fips-docker')) {
616+
base = DockerBase.CLOUD_ESS_FIPS
617617
}
618618

619619
final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
620620
final String extension =
621621
(base == DockerBase.IRON_BANK ? 'ironbank.tar' :
622622
(base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
623-
(base == DockerBase.FIPS ? 'fips.tar' :
623+
(base == DockerBase.CLOUD_ESS_FIPS ? 'cloud-ess-fips.tar' :
624624
(base == DockerBase.WOLFI ? 'wolfi.tar' :
625625
'docker.tar'))))
626626
final String artifactName = "elasticsearch${arch}${base.suffix}_test"

‎distribution/docker/fips-docker-aarch64-export/build.gradle‎ renamed to ‎distribution/docker/cloud-ess-fips-docker-aarch64-export/build.gradle‎

File renamed without changes.

‎distribution/docker/fips-docker-export/build.gradle‎ renamed to ‎distribution/docker/cloud-ess-fips-docker-export/build.gradle‎

File renamed without changes.

0 commit comments

Comments
 (0)