Merge branch 'main' into per-project-settings

Author: alexey-ivanov-es

benchmarks/build.gradle

@@ -42,6 +42,7 @@ dependencies {
   api(project(':libs:h3'))
   api(project(':modules:aggregations'))
   api(project(':x-pack:plugin:esql-core'))
+  api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
   implementation project(path: ':libs:simdvec')

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -25,6 +25,7 @@
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -50,6 +51,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -335,7 +337,7 @@ public void benchmark() {
             fields(name),
             List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
                 throw new UnsupportedOperationException("can't load _source here");
-            })),
+            }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))),
             0
         );
         long sum = 0;

benchmarks/src/main/java/org/elasticsearch/benchmark/esql/QueryPlanningBenchmark.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.esql;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.esql.analysis.Analyzer;
+import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
+import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
+import org.elasticsearch.xpack.esql.analysis.Verifier;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
+import org.elasticsearch.xpack.esql.inference.InferenceResolution;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
+import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
+import org.elasticsearch.xpack.esql.parser.EsqlParser;
+import org.elasticsearch.xpack.esql.parser.QueryParams;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
+import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
+import org.elasticsearch.xpack.esql.session.Configuration;
+import org.elasticsearch.xpack.esql.telemetry.Metrics;
+import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+
+@Fork(1)
+@Warmup(iterations = 5)
+@Measurement(iterations = 10)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class QueryPlanningBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging();
+    }
+
+    private PlanTelemetry telemetry;
+    private EsqlParser parser;
+    private Analyzer analyzer;
+    private LogicalPlanOptimizer optimizer;
+
+    @Setup
+    public void setup() {
+
+        var config = new Configuration(
+            DateUtils.UTC,
+            Locale.US,
+            null,
+            null,
+            new QueryPragmas(Settings.EMPTY),
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
+            "",
+            false,
+            Map.of(),
+            System.nanoTime(),
+            false
+        );
+
+        var fields = 10_000;
+        var mapping = LinkedHashMap.<String, EsField>newLinkedHashMap(fields);
+        for (int i = 0; i < fields; i++) {
+            mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true));
+        }
+
+        var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
+
+        var functionRegistry = new EsqlFunctionRegistry();
+
+        telemetry = new PlanTelemetry(functionRegistry);
+        parser = new EsqlParser();
+        analyzer = new Analyzer(
+            new AnalyzerContext(
+                config,
+                functionRegistry,
+                IndexResolution.valid(esIndex),
+                Map.of(),
+                new EnrichResolution(),
+                InferenceResolution.EMPTY
+            ),
+            new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
+        );
+        optimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
+    }
+
+    private LogicalPlan plan(String query) {
+        var parsed = parser.createStatement(query, new QueryParams(), telemetry);
+        var analyzed = analyzer.analyze(parsed);
+        var optimized = optimizer.optimize(analyzed);
+        return optimized;
+    }
+
+    @Benchmark
+    public void run(Blackhole blackhole) {
+        blackhole.consume(plan("FROM test | LIMIT 10"));
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -28,16 +28,17 @@ public enum DockerBase {
         "apk",
         "Dockerfile"
     ),
-    FIPS(
-        "docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
-        "-fips",
-        "apk",
-        "Dockerfile"
-    ),
     // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.
-    CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.cloud-ess"),;
+    CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.cloud-ess"),
+
+    CLOUD_ESS_FIPS(
+        "docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
+        "-cloud-ess-fips",
+        "apk",
+        "Dockerfile"
+    );
 
     private final String image;
     private final String suffix;

distribution/docker/build.gradle

@@ -314,7 +314,7 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base, Strin
           filter TransformLog4jConfigFilter
         }
       }
-      if(base == DockerBase.FIPS) {
+      if(base == DockerBase.CLOUD_ESS_FIPS) {
 
         // If we're performing a release build, but `build.id` hasn't been set, we can
         // infer that we're not at the Docker building stage of the build, and therefore
@@ -608,19 +608,19 @@ subprojects { Project subProject ->
     DockerBase base = DockerBase.DEFAULT
     if (subProject.name.contains('ironbank-')) {
       base = DockerBase.IRON_BANK
-    } else if (subProject.name.contains('cloud-ess-')) {
+    } else if (subProject.name.contains('cloud-ess-docker')) {
       base = DockerBase.CLOUD_ESS
     } else if (subProject.name.contains('wolfi-')) {
       base = DockerBase.WOLFI
-    } else if (subProject.name.contains('fips-')) {
-      base = DockerBase.FIPS
+    } else if (subProject.name.contains('cloud-ess-fips-docker')) {
+      base = DockerBase.CLOUD_ESS_FIPS
     }
 
     final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
     final String extension =
       (base == DockerBase.IRON_BANK ? 'ironbank.tar' :
         (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
-          (base == DockerBase.FIPS ? 'fips.tar' :
+          (base == DockerBase.CLOUD_ESS_FIPS ? 'cloud-ess-fips.tar' :
             (base == DockerBase.WOLFI ? 'wolfi.tar' :
               'docker.tar'))))
     final String artifactName = "elasticsearch${arch}${base.suffix}_test"

distribution/docker/fips-docker-aarch64-export/build.gradle renamed to distribution/docker/cloud-ess-fips-docker-aarch64-export/build.gradle

@@ -41,7 +41,7 @@ RUN chmod 0555 /bin/tini
 <% } else { %>
 
 # Install required packages to extract the Elasticsearch distribution
-<% if (docker_base == "wolfi" || docker_base == "fips") { %>
+<% if (docker_base == "wolfi" || docker_base == "cloud_ess_fips") { %>
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
 <% } else { %>
 RUN <%= retry.loop(package_manager, "${package_manager} install -y findutils tar gzip") %>
@@ -115,7 +115,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
     chmod 0775 bin config config/jvm.options.d data logs plugins && \\
     find config -type f -exec chmod 0664 {} +
 
-<% if (docker_base == "fips") { %>
+<% if (docker_base == "cloud_ess_fips") { %>
 
 # Add plugins infrastructure
 RUN mkdir -p /opt/plugins/archive
@@ -179,7 +179,7 @@ RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\
       nc shadow-utils zip findutils unzip procps-ng && \\
     ${package_manager} clean all
 
-<% } else if (docker_base == "wolfi" || docker_base == "fips") { %>
+<% } else if (docker_base == "wolfi" || docker_base == "cloud_ess_fips") { %>
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +
           "      ${package_manager} update && \n" +
@@ -208,7 +208,7 @@ RUN <%= retry.loop(
 <% } %>
 
 
-<% if (docker_base == "wolfi" || docker_base == "fips") { %>
+<% if (docker_base == "wolfi" || docker_base == "cloud_ess_fips") { %>
 RUN groupadd -g 1000 elasticsearch && \
     adduser -G elasticsearch -u 1000 elasticsearch -D --home /usr/share/elasticsearch elasticsearch && \
     adduser elasticsearch root && \
@@ -219,17 +219,17 @@ RUN groupadd -g 1000 elasticsearch && \\
     chown -R 0:0 /usr/share/elasticsearch
 <% } %>
 
-ENV ELASTIC_CONTAINER true
+ENV ELASTIC_CONTAINER=true
 
 WORKDIR /usr/share/elasticsearch
 
 COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch
-<% if (docker_base != "wolfi" && docker_base != "fips") { %>
+<% if (docker_base != "wolfi" && docker_base != "cloud_ess_fips") { %>
 COPY --from=builder --chown=0:0 /bin/tini /bin/tini
 <% } %>
 
-ENV PATH /usr/share/elasticsearch/bin:\$PATH
-ENV SHELL /bin/bash
+ENV PATH=/usr/share/elasticsearch/bin:\$PATH
+ENV SHELL=/bin/bash
 COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
 # 1. Sync the user and group permissions of /etc/passwd
@@ -249,7 +249,7 @@ RUN chmod g=u /etc/passwd && \\
     chmod 0775 /usr/share/elasticsearch && \\
     chown elasticsearch bin config config/jvm.options.d data logs plugins
 
-<% if (docker_base == 'wolfi' || docker_base == "fips") { %>
+<% if (docker_base == 'wolfi' || docker_base == "cloud_ess_fips") { %>
 RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts
 <% } else { %>
 RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts
@@ -292,7 +292,7 @@ RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
 COPY LICENSE /licenses/LICENSE.addendum
 <% } %>
 
-<% if (docker_base == "wolfi" || docker_base == "fips") { %>
+<% if (docker_base == "wolfi" || docker_base == "cloud_ess_fips") { %>
 # Our actual entrypoint is `tini`, a minimal but functional init program. It
 # calls the entrypoint we provide, while correctly forwarding signals.
 ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]
@@ -312,9 +312,9 @@ USER 1000:0
 HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1
 <% } %>
 
-<% if (docker_base == 'fips') { %>
+<% if (docker_base == 'cloud_ess_fips') { %>
 COPY --from=builder --chown=0:0 /opt /opt
-ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive
+ENV ES_PLUGIN_ARCHIVE_DIR=/opt/plugins/archive
 WORKDIR /usr/share/elasticsearch
 COPY --from=builder --chown=0:0 /fips/libs/*.jar /usr/share/elasticsearch/lib/
 <% } %>

distribution/docker/fips-docker-export/build.gradle renamed to distribution/docker/cloud-ess-fips-docker-export/build.gradle

@@ -0,0 +1,5 @@
+pr: 127348
+summary: Speed loading stored fields
+area: ES|QL
+type: enhancement
+issues: []

distribution/docker/src/docker/Dockerfile

@@ -0,0 +1,6 @@
+pr: 127522
+summary: Use INTERNAL_INGEST for Inference
+area: Machine Learning
+type: bug
+issues:
+ - 127519