Merge branch 'main' into main

Author: brendan-jugan-elastic

benchmarks/build.gradle

@@ -42,6 +42,7 @@ dependencies {
   api(project(':libs:h3'))
   api(project(':modules:aggregations'))
   api(project(':x-pack:plugin:esql-core'))
+  api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
   implementation project(path: ':libs:simdvec')

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -25,6 +25,7 @@
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BytesRefBlock;
@@ -50,6 +51,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -335,7 +337,7 @@ public void benchmark() {
             fields(name),
             List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
                 throw new UnsupportedOperationException("can't load _source here");
-            })),
+            }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))),
             0
         );
         long sum = 0;

benchmarks/src/main/java/org/elasticsearch/benchmark/esql/QueryPlanningBenchmark.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.esql;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.esql.analysis.Analyzer;
+import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
+import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
+import org.elasticsearch.xpack.esql.analysis.Verifier;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
+import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
+import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
+import org.elasticsearch.xpack.esql.inference.InferenceResolution;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
+import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer;
+import org.elasticsearch.xpack.esql.parser.EsqlParser;
+import org.elasticsearch.xpack.esql.parser.QueryParams;
+import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
+import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
+import org.elasticsearch.xpack.esql.session.Configuration;
+import org.elasticsearch.xpack.esql.telemetry.Metrics;
+import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.LinkedHashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+
+@Fork(1)
+@Warmup(iterations = 5)
+@Measurement(iterations = 10)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+public class QueryPlanningBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging();
+    }
+
+    private PlanTelemetry telemetry;
+    private EsqlParser parser;
+    private Analyzer analyzer;
+    private LogicalPlanOptimizer optimizer;
+
+    @Setup
+    public void setup() {
+
+        var config = new Configuration(
+            DateUtils.UTC,
+            Locale.US,
+            null,
+            null,
+            new QueryPragmas(Settings.EMPTY),
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
+            EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
+            "",
+            false,
+            Map.of(),
+            System.nanoTime(),
+            false
+        );
+
+        var fields = 10_000;
+        var mapping = LinkedHashMap.<String, EsField>newLinkedHashMap(fields);
+        for (int i = 0; i < fields; i++) {
+            mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true));
+        }
+
+        var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
+
+        var functionRegistry = new EsqlFunctionRegistry();
+
+        telemetry = new PlanTelemetry(functionRegistry);
+        parser = new EsqlParser();
+        analyzer = new Analyzer(
+            new AnalyzerContext(
+                config,
+                functionRegistry,
+                IndexResolution.valid(esIndex),
+                Map.of(),
+                new EnrichResolution(),
+                InferenceResolution.EMPTY
+            ),
+            new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
+        );
+        optimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
+    }
+
+    private LogicalPlan plan(String query) {
+        var parsed = parser.createStatement(query, new QueryParams(), telemetry);
+        var analyzed = analyzer.analyze(parsed);
+        var optimized = optimizer.optimize(analyzed);
+        return optimized;
+    }
+
+    @Benchmark
+    public void run(Blackhole blackhole) {
+        blackhole.consume(plan("FROM test | LIMIT 10"));
+    }
+}

docs/changelog/127348.yaml

@@ -0,0 +1,5 @@
+pr: 127348
+summary: Speed loading stored fields
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/127522.yaml

@@ -0,0 +1,6 @@
+pr: 127522
+summary: Use INTERNAL_INGEST for Inference
+area: Machine Learning
+type: bug
+issues:
+ - 127519

docs/changelog/127527.yaml

@@ -0,0 +1,5 @@
+pr: 127527
+summary: "No, line noise isn't a valid ip"
+area: ES|QL
+type: bug
+issues: []

docs/reference/elasticsearch/index-settings/index-modules.md

@@ -49,7 +49,7 @@ $$$index-codec$$$ `index.codec`
 
 $$$index-mode-setting$$$ `index.mode`
 :   The `index.mode` setting is used to control settings applied in specific domains like ingestion of time series data or logs. Different mutually exclusive modes exist, which are used to apply settings or default values controlling indexing of documents, sorting and other parameters whose value affects indexing or query performance.
-        
+
         **Example**
 
       ```console
@@ -248,3 +248,8 @@ $$$index-final-pipeline$$$
 
 $$$index-hidden$$$ `index.hidden`
 :   Indicates whether the index should be hidden by default. Hidden indices are not returned by default when using a wildcard expression. This behavior is controlled per request through the use of the `expand_wildcards` parameter. Possible values are `true` and `false` (default).
+
+$$$index-esql-stored-fields-sequential-proportion$$$
+
+`index.esql.stored_fields_sequential_proportion`
+:   Tuning parameter for deciding when {{esql}} will load [Stored fields](/reference/elasticsearch/rest-apis/retrieve-selected-fields.md#stored-fields) using a strategy tuned for loading dense sequence of documents. Allows values between 0.0 and 1.0 and defaults to 0.2. Indices with documents smaller than 10kb may see speed improvements loading `text` fields by setting this lower.

docs/reference/enrich-processor/date-processor.md

@@ -79,7 +79,7 @@ The `timezone` and `locale` processor parameters are templated. This means that
 
 ### Example dealing with short timezone abbreviations safely [date-processor-short-timezone-example]
 
-In the example below, the `message` field in the input is expected to be a string formed of a local date-time in `yyyyMMddHHmmss` format, a timezone abbreviated to one of `PST`, `CET`, or `JST` representing Pacific, Central European, or Japan time, and a payload. This field is split up using a `grok` processor, then the timezones are converted into full names using a `script` processor, then the date-time is parsed using a `date` processor, and finally the unwanted fields are discarded using a `drop` processor.
+In the example below, the `message` field in the input is expected to be a string formed of a local date-time in `yyyyMMddHHmmss` format, a timezone abbreviated to one of `PST`, `CET`, or `JST` representing Pacific, Central European, or Japan time, and a payload. This field is split up using a `grok` processor, then the timezones are converted into full names using a `script` processor, then the date-time is parsed using a `date` processor, and finally the unwanted fields are discarded using a `remove` processor.
 
 ```js
 {

docs/reference/query-languages/esql/_snippets/commands/layout/lookup-join.md

@@ -41,12 +41,12 @@ added as new columns to that row.
 If multiple documents in the lookup index match a single row in your
 results, the output will contain one row for each matching combination.
 
-**Examples**
-
 ::::{tip}
 In case of name collisions, the newly created columns will override existing columns.
 ::::
 
+**Examples**
+
 **IP Threat correlation**: This query would allow you to see if any source
 IPs match known malicious addresses.
 

muted-tests.yml

@@ -101,9 +101,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_reset/Test reset running transform}
   issue: https://github.com/elastic/elasticsearch/issues/117473
-- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
-  method: test {p0=search.highlight/50_synthetic_source/text multi unified from vectors}
-  issue: https://github.com/elastic/elasticsearch/issues/117815
 - class: org.elasticsearch.xpack.ml.integration.RegressionIT
   method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet
   issue: https://github.com/elastic/elasticsearch/issues/117805
@@ -432,18 +429,15 @@ tests:
 - class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
   method: testLookupExplosionNoFetch
   issue: https://github.com/elastic/elasticsearch/issues/127365
-- class: org.elasticsearch.action.admin.cluster.state.TransportClusterStateActionDisruptionIT
-  method: testNonLocalRequestAlwaysFindsMasterAndWaitsForMetadata
-  issue: https://github.com/elastic/elasticsearch/issues/127422
 - class: org.elasticsearch.xpack.esql.qa.single_node.PushQueriesIT
   method: testPushCaseInsensitiveEqualityOnDefaults
   issue: https://github.com/elastic/elasticsearch/issues/127431
-- class: org.elasticsearch.action.admin.cluster.state.TransportClusterStateActionDisruptionIT
-  method: testLocalRequestAlwaysSucceeds
-  issue: https://github.com/elastic/elasticsearch/issues/127423
-- class: org.elasticsearch.action.admin.cluster.state.TransportClusterStateActionDisruptionIT
-  method: testLocalRequestWaitsForMetadata
-  issue: https://github.com/elastic/elasticsearch/issues/127466
+- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/127536
+- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
+  method: test {union_types.MultiIndexSortIpStringEval ASYNC}
+  issue: https://github.com/elastic/elasticsearch/issues/127537
 
 # Examples:
 #