Merge remote-tracking branch 'upstream/main' into eut2

Author: jdconrad

BUILDING.md

@@ -144,7 +144,7 @@ To wire this registered cluster into a `TestClusterAware` task (e.g. `RestIntegT
 Additional integration tests for a certain Elasticsearch modules that are specific to certain cluster configuration can be declared in a separate so called `qa` subproject of your module.
 
 The benefit of a dedicated project for these tests are:
-- `qa` projects are dedicated two specific use-cases and easier to maintain
+- `qa` projects are dedicated to specific use-cases and easier to maintain
 - It keeps the specific test logic separated from the common test logic.
 - You can run those tests in parallel to other projects of the build.
 

benchmarks/build.gradle

@@ -42,6 +42,7 @@ dependencies {
   api(project(':libs:h3'))
   api(project(':modules:aggregations'))
   api(project(':x-pack:plugin:esql-core'))
+  api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
   implementation project(path: ':libs:simdvec')

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -25,6 +25,7 @@
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -50,6 +51,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -335,7 +337,7 @@ public void benchmark() {
             fields(name),
             List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
                 throw new UnsupportedOperationException("can't load _source here");
-            })),
+            }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))),
             0
         );
         long sum = 0;

benchmarks/src/main/java/org/elasticsearch/benchmark/esql/QueryPlanningBenchmark.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.esql;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.esql.analysis.Analyzer;
+import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
+import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
+import org.elasticsearch.xpack.esql.analysis.Verifier;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
+import org.elasticsearch.xpack.esql.inference.InferenceResolution;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
+import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
+import org.elasticsearch.xpack.esql.parser.EsqlParser;
+import org.elasticsearch.xpack.esql.parser.QueryParams;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
+import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
+import org.elasticsearch.xpack.esql.session.Configuration;
+import org.elasticsearch.xpack.esql.telemetry.Metrics;
+import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+
+@Fork(1)
+@Warmup(iterations = 5)
+@Measurement(iterations = 10)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class QueryPlanningBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging();
+    }
+
+    private PlanTelemetry telemetry;
+    private EsqlParser parser;
+    private Analyzer analyzer;
+    private LogicalPlanOptimizer optimizer;
+
+    @Setup
+    public void setup() {
+
+        var config = new Configuration(
+            DateUtils.UTC,
+            Locale.US,
+            null,
+            null,
+            new QueryPragmas(Settings.EMPTY),
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
+            "",
+            false,
+            Map.of(),
+            System.nanoTime(),
+            false
+        );
+
+        var fields = 10_000;
+        var mapping = LinkedHashMap.<String, EsField>newLinkedHashMap(fields);
+        for (int i = 0; i < fields; i++) {
+            mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true));
+        }
+
+        var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
+
+        var functionRegistry = new EsqlFunctionRegistry();
+
+        telemetry = new PlanTelemetry(functionRegistry);
+        parser = new EsqlParser();
+        analyzer = new Analyzer(
+            new AnalyzerContext(
+                config,
+                functionRegistry,
+                IndexResolution.valid(esIndex),
+                Map.of(),
+                new EnrichResolution(),
+                InferenceResolution.EMPTY
+            ),
+            new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
+        );
+        optimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
+    }
+
+    private LogicalPlan plan(String query) {
+        var parsed = parser.createStatement(query, new QueryParams(), telemetry);
+        var analyzed = analyzer.analyze(parsed);
+        var optimized = optimizer.optimize(analyzed);
+        return optimized;
+    }
+
+    @Benchmark
+    public void run(Blackhole blackhole) {
+        blackhole.consume(plan("FROM test | LIMIT 10"));
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/indices/resolution/IndexNameExpressionResolverBenchmark.java

@@ -16,7 +16,8 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.project.DefaultProjectResolver;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -63,12 +64,12 @@ public void setUp() {
         int numDataStreams = toInt(params[0]);
         int numIndices = toInt(params[1]);
 
-        Metadata.Builder mb = Metadata.builder();
+        ProjectMetadata.Builder pmb = ProjectMetadata.builder(ProjectId.DEFAULT);
         String[] indices = new String[numIndices + numDataStreams * (numIndices + 1)];
         int position = 0;
         for (int i = 1; i <= numIndices; i++) {
             String indexName = INDEX_PREFIX + i;
-            createIndexMetadata(indexName, mb);
+            createIndexMetadata(indexName, pmb);
             indices[position++] = indexName;
         }
 
@@ -77,14 +78,14 @@ public void setUp() {
             List<Index> backingIndices = new ArrayList<>();
             for (int j = 1; j <= numIndices; j++) {
                 String backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, j);
-                backingIndices.add(createIndexMetadata(backingIndexName, mb).getIndex());
+                backingIndices.add(createIndexMetadata(backingIndexName, pmb).getIndex());
                 indices[position++] = backingIndexName;
             }
             indices[position++] = dataStreamName;
-            mb.put(DataStream.builder(dataStreamName, backingIndices).build());
+            pmb.put(DataStream.builder(dataStreamName, backingIndices).build());
         }
         int mid = indices.length / 2;
-        clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build();
+        clusterState = ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(pmb).build();
         resolver = new IndexNameExpressionResolver(
             new ThreadContext(Settings.EMPTY),
             new SystemIndices(List.of()),
@@ -97,13 +98,13 @@ public void setUp() {
         mixedRequest = new Request(IndicesOptions.lenientExpandOpenHidden(), mixed);
     }
 
-    private IndexMetadata createIndexMetadata(String indexName, Metadata.Builder mb) {
+    private IndexMetadata createIndexMetadata(String indexName, ProjectMetadata.Builder pmb) {
         IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
             .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
             .numberOfShards(1)
             .numberOfReplicas(0)
             .build();
-        mb.put(indexMetadata, false);
+        pmb.put(indexMetadata, false);
         return indexMetadata;
     }
 

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java

@@ -12,7 +12,8 @@
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.project.DefaultProjectResolver;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -96,7 +97,7 @@ public void setUp() throws Exception {
 
         AllocationService allocationService = Allocators.createAllocationService(Settings.EMPTY);
 
-        Metadata.Builder mb = Metadata.builder();
+        ProjectMetadata.Builder pmb = ProjectMetadata.builder(ProjectId.DEFAULT);
         RoutingTable.Builder rb = RoutingTable.builder();
 
         DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
@@ -160,12 +161,12 @@ public void setUp() throws Exception {
             }
 
             routingTable.add(indexRountingTableBuilder);
-            mb.put(indexMetadata, false);
+            pmb.put(indexMetadata, false);
         }
 
         ClusterState initialClusterState = ClusterState.builder(ClusterName.DEFAULT)
-            .metadata(mb)
-            .routingTable(routingTable)
+            .putProjectMetadata(pmb)
+            .putRoutingTable(pmb.getId(), routingTable.build())
             .nodes(nb)
             .build();