Merge branch 'main' of https://github.com/elastic/elasticsearch into esql-completion-inference-operator

Author: afoucret

.backportrc.json

@@ -1,10 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "9.0", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.1.0$" : "main",
-    "^v8.19.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/periodic.template.yml

@@ -237,7 +237,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/periodic.yml

@@ -656,7 +656,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/scripts/dra-workflow.trigger.sh

@@ -2,10 +2,34 @@
 
 set -euo pipefail
 
-echo "steps:"
-
 source .buildkite/scripts/branches.sh
 
+# We use that filtering to keep different schedule for different branches
+if [ -n "${INCLUDED_BRANCHES:-}" ]; then
+  # If set, only trigger the pipeline for the specified branches
+  IFS=',' read -r -a BRANCHES <<< "${INCLUDED_BRANCHES}"
+elif [ -n "${EXCLUDED_BRANCHES:-}" ]; then
+  # If set, listed branches will be excluded from the list of branches in branches.json
+  IFS=',' read -r -a EXCLUDED_BRANCHES <<< "${EXCLUDED_BRANCHES}"
+  FILTERED_BRANCHES=()
+    for BRANCH in "${BRANCHES[@]}"; do
+      EXCLUDE=false
+      for EXCLUDED_BRANCH in "${EXCLUDED_BRANCHES[@]}"; do
+        if [ "$BRANCH" == "$EXCLUDED_BRANCH" ]; then
+          EXCLUDE=true
+          break
+        fi
+      done
+      if [ "$EXCLUDE" = false ]; then
+        FILTERED_BRANCHES+=("$BRANCH")
+      fi
+    done
+    BRANCHES=("${FILTERED_BRANCHES[@]}")
+fi
+
+
+echo "steps:"
+
 for BRANCH in "${BRANCHES[@]}"; do
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"
   BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')

.ci/scripts/resolve-dra-manifest.sh

@@ -24,11 +24,6 @@ if [ "$LATEST_VERSION" != "$ES_VERSION" ]; then
   echo "Latest build for '$ARTIFACT' is version $LATEST_VERSION but expected version $ES_VERSION." 1>&2
   NEW_BRANCH=$(echo $ES_VERSION | sed -E "s/([0-9]+\.[0-9]+)\.[0-9]/\1/g")
 
-  # Temporary
-  if [[ "$ES_VERSION" == "8.16.0" ]]; then
-    NEW_BRANCH="8.x"
-  fi
-
   echo "Using branch $NEW_BRANCH instead of $BRANCH." 1>&2
   LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $NEW_BRANCH)
 fi

BUILDING.md

@@ -82,7 +82,7 @@ In case of updating a dependency, ensure to remove the unused entry of the outda
 
 You can also automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
 ```
->./gradlew --write-verification-metadata sha256 precommit
+./gradlew --write-verification-metadata sha256 precommit
 ```
 
 The `--write-verification-metadata` Gradle option is generally able to resolve reachable configurations,
@@ -92,10 +92,10 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
 We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
 will have the `origin` attribute been set to `Generated by Gradle`.
 
->A manual confirmation of the Gradle generated checksums is currently not mandatory.
->If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
->Please replace the content of the `origin` attribute by `official site` in that case.
->
+> [!Tip]  
+> A manual confirmation of the Gradle generated checksums is currently not mandatory.
+> If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
+> Please replace the content of the `origin` attribute by `official site` in that case.
 
 #### Custom plugin and task implementations
 
@@ -144,7 +144,7 @@ To wire this registered cluster into a `TestClusterAware` task (e.g. `RestIntegT
 Additional integration tests for a certain Elasticsearch modules that are specific to certain cluster configuration can be declared in a separate so called `qa` subproject of your module.
 
 The benefit of a dedicated project for these tests are:
-- `qa` projects are dedicated two specific use-cases and easier to maintain
+- `qa` projects are dedicated to specific use-cases and easier to maintain
 - It keeps the specific test logic separated from the common test logic.
 - You can run those tests in parallel to other projects of the build.
 
@@ -229,13 +229,9 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
----
-
-**NOTE**
-
-You should only use that approach locally or on a developer branch for production dependencies as we do
+> [!Note]  
+> You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
----
 
 #### How to use a custom third party artifact?
 
@@ -265,12 +261,9 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
----
-**NOTE**
-
-As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
+> [!Note]  
+> As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
-For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
+> For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
 that supports meta-data, it will use the second repository to provide the module.
-Therefore, it is recommended to declare a version that is not resolvable from public repositories we use (e.g. Maven Central)
----
+> Therefore, it is recommended to declare a version that is not resolvable from public repositories we use (e.g. Maven Central)

CONTRIBUTING.md

@@ -401,7 +401,8 @@ It is important that the only code covered by the Elastic licence is contained
 within the top-level `x-pack` directory. The build will fail its pre-commit
 checks if contributed code does not have the appropriate license headers.
 
-> **NOTE:** If you have imported the project into IntelliJ IDEA the project will
+> [!NOTE]
+> If you have imported the project into IntelliJ IDEA the project will
 > be automatically configured to add the correct license header to new source
 > files based on the source location.
 

benchmarks/build.gradle

@@ -42,6 +42,7 @@ dependencies {
   api(project(':libs:h3'))
   api(project(':modules:aggregations'))
   api(project(':x-pack:plugin:esql-core'))
+  api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
   implementation project(path: ':libs:simdvec')

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -25,6 +25,7 @@
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -50,6 +51,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -335,7 +337,7 @@ public void benchmark() {
             fields(name),
             List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
                 throw new UnsupportedOperationException("can't load _source here");
-            })),
+            }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))),
             0
         );
         long sum = 0;

benchmarks/src/main/java/org/elasticsearch/benchmark/esql/QueryPlanningBenchmark.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.esql;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.esql.analysis.Analyzer;
+import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
+import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
+import org.elasticsearch.xpack.esql.analysis.Verifier;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
+import org.elasticsearch.xpack.esql.inference.InferenceResolution;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
+import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
+import org.elasticsearch.xpack.esql.parser.EsqlParser;
+import org.elasticsearch.xpack.esql.parser.QueryParams;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
+import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
+import org.elasticsearch.xpack.esql.session.Configuration;
+import org.elasticsearch.xpack.esql.telemetry.Metrics;
+import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+
+@Fork(1)
+@Warmup(iterations = 5)
+@Measurement(iterations = 10)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class QueryPlanningBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging();
+    }
+
+    private PlanTelemetry telemetry;
+    private EsqlParser parser;
+    private Analyzer analyzer;
+    private LogicalPlanOptimizer optimizer;
+
+    @Setup
+    public void setup() {
+
+        var config = new Configuration(
+            DateUtils.UTC,
+            Locale.US,
+            null,
+            null,
+            new QueryPragmas(Settings.EMPTY),
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
+            "",
+            false,
+            Map.of(),
+            System.nanoTime(),
+            false
+        );
+
+        var fields = 10_000;
+        var mapping = LinkedHashMap.<String, EsField>newLinkedHashMap(fields);
+        for (int i = 0; i < fields; i++) {
+            mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true));
+        }
+
+        var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
+
+        var functionRegistry = new EsqlFunctionRegistry();
+
+        telemetry = new PlanTelemetry(functionRegistry);
+        parser = new EsqlParser();
+        analyzer = new Analyzer(
+            new AnalyzerContext(
+                config,
+                functionRegistry,
+                IndexResolution.valid(esIndex),
+                Map.of(),
+                new EnrichResolution(),
+                InferenceResolution.EMPTY
+            ),
+            new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
+        );
+        optimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
+    }
+
+    private LogicalPlan plan(String query) {
+        var parsed = parser.createStatement(query, new QueryParams(), telemetry);
+        var analyzed = analyzer.analyze(parsed);
+        var optimized = optimizer.optimize(analyzed);
+        return optimized;
+    }
+
+    @Benchmark
+    public void manyFields(Blackhole blackhole) {
+        blackhole.consume(plan("FROM test | LIMIT 10"));
+    }
+}