Merge 7544c88 into openjdk23-bundle

Author: elasticsearchmachine

docs/changelog/115041.yaml

@@ -0,0 +1,6 @@
+pr: 115041
+summary: Increase default `queue_capacity` to 10_000 and decrease max `queue_capacity`
+  to 100_000
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/115308.yaml

@@ -0,0 +1,6 @@
+pr: 115308
+summary: "ESQL: Disable pushdown of WHERE past STATS"
+area: ES|QL
+type: bug
+issues:
+ - 115281

docs/changelog/115359.yaml

@@ -0,0 +1,6 @@
+pr: 115359
+summary: Adding support for simulate ingest mapping adddition for indices with mappings
+  that do not come from templates
+area: Ingest Node
+type: enhancement
+issues: []

docs/reference/esql/functions/kibana/definition/to_date_nanos.json

@@ -6,41 +6,45 @@ include::links.asciidoc[]
 
 include::landing-page.asciidoc[]
 
-include::release-notes/highlights.asciidoc[]
+// overview / install
 
 include::intro.asciidoc[]
 
 include::quickstart/index.asciidoc[]
 
 include::setup.asciidoc[]
 
-include::upgrade.asciidoc[]
+// search solution
 
-include::index-modules.asciidoc[]
+include::search/search-your-data/search-your-data.asciidoc[]
 
-include::mapping.asciidoc[]
+include::reranking/index.asciidoc[]
 
-include::analysis.asciidoc[]
+// data management
+
+include::index-modules.asciidoc[]
 
 include::indices/index-templates.asciidoc[]
 
-include::data-streams/data-streams.asciidoc[]
+include::alias.asciidoc[]
 
-include::ingest.asciidoc[]
+include::mapping.asciidoc[]
 
-include::alias.asciidoc[]
+include::analysis.asciidoc[]
 
-include::search/search-your-data/search-your-data.asciidoc[]
+include::ingest.asciidoc[]
 
-include::reranking/index.asciidoc[]
+include::connector/docs/index.asciidoc[]
 
-include::query-dsl.asciidoc[]
+include::data-streams/data-streams.asciidoc[]
 
-include::aggregations.asciidoc[]
+include::data-management.asciidoc[]
 
-include::geospatial-analysis.asciidoc[]
+include::data-rollup-transform.asciidoc[]
 
-include::connector/docs/index.asciidoc[]
+// analysis tools
+
+include::query-dsl.asciidoc[]
 
 include::eql/eql.asciidoc[]
 
@@ -50,34 +54,48 @@ include::sql/index.asciidoc[]
 
 include::scripting.asciidoc[]
 
-include::data-management.asciidoc[]
+include::aggregations.asciidoc[]
 
-include::autoscaling/index.asciidoc[]
+include::geospatial-analysis.asciidoc[]
+
+include::watcher/index.asciidoc[]
+
+// cluster management
 
 include::monitoring/index.asciidoc[]
 
-include::data-rollup-transform.asciidoc[]
+include::security/index.asciidoc[]
+
+// production tasks
 
 include::high-availability.asciidoc[]
 
+include::how-to.asciidoc[]
+
+include::autoscaling/index.asciidoc[]
+
 include::snapshot-restore/index.asciidoc[]
 
-include::security/index.asciidoc[]
+// reference
 
-include::watcher/index.asciidoc[]
+include::rest-api/index.asciidoc[]
 
 include::commands/index.asciidoc[]
 
-include::how-to.asciidoc[]
-
 include::troubleshooting.asciidoc[]
 
-include::rest-api/index.asciidoc[]
+// upgrades
+
+include::upgrade.asciidoc[]
 
 include::migration/index.asciidoc[]
 
+include::release-notes/highlights.asciidoc[]
+
 include::release-notes.asciidoc[]
 
 include::dependencies-versions.asciidoc[]
 
+// etc
+
 include::redirects.asciidoc[]

docs/reference/index.asciidoc

@@ -1,5 +1,6 @@
+[chapter]
 [[release-highlights]]
-== What's new in {minor-version}
+= What's new in {minor-version}
 
 coming::[{minor-version}]
 
@@ -37,7 +38,7 @@ endif::[]
 
 [discrete]
 [[esql_inlinestats]]
-=== ESQL: INLINESTATS
+== ESQL: INLINESTATS
 This adds the `INLINESTATS` command to ESQL which performs a STATS and
 then enriches the results into the output stream. So, this query:
 
@@ -62,7 +63,7 @@ Produces output like:
 
 [discrete]
 [[always_allow_rebalancing_by_default]]
-=== Always allow rebalancing by default
+== Always allow rebalancing by default
 In earlier versions of {es} the `cluster.routing.allocation.allow_rebalance` setting defaults to
 `indices_all_active` which blocks all rebalancing moves while the cluster is in `yellow` or `red` health. This was
 appropriate for the legacy allocator which might do too many rebalancing moves otherwise. Today's allocator has
@@ -74,7 +75,7 @@ version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy al
 
 [discrete]
 [[add_global_retention_in_data_stream_lifecycle]]
-=== Add global retention in data stream lifecycle
+== Add global retention in data stream lifecycle
 Data stream lifecycle now supports configuring retention on a cluster level,
 namely global retention. Global retention \nallows us to configure two different
 retentions:
@@ -88,7 +89,7 @@ data stream lifecycle and it allows any data stream \ndata to be deleted after t
 
 [discrete]
 [[enable_zstandard_compression_for_indices_with_index_codec_set_to_best_compression]]
-=== Enable ZStandard compression for indices with index.codec set to best_compression
+== Enable ZStandard compression for indices with index.codec set to best_compression
 Before DEFLATE compression was used to compress stored fields in indices with index.codec index setting set to
 best_compression, with this change ZStandard is used as compression algorithm to stored fields for indices with
 index.codec index setting set to best_compression. The usage ZStandard results in less storage usage with a

docs/reference/release-notes/highlights.asciidoc

@@ -43,13 +43,14 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.RemoteTransportException;
 
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
 
 import static org.elasticsearch.ingest.geoip.GeoIpDownloader.DATABASES_INDEX;
 import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER;
@@ -238,14 +239,11 @@ public void clusterChanged(ClusterChangedEvent event) {
     }
 
     static boolean hasAtLeastOneGeoipProcessor(ClusterState clusterState) {
-        if (pipelineConfigurationsWithGeoIpProcessor(clusterState, true).isEmpty() == false) {
+        if (pipelinesWithGeoIpProcessor(clusterState, true).isEmpty() == false) {
             return true;
         }
 
-        Set<String> checkReferencedPipelines = pipelineConfigurationsWithGeoIpProcessor(clusterState, false).stream()
-            .map(PipelineConfiguration::getId)
-            .collect(Collectors.toSet());
-
+        final Set<String> checkReferencedPipelines = pipelinesWithGeoIpProcessor(clusterState, false);
         if (checkReferencedPipelines.isEmpty()) {
             return false;
         }
@@ -258,22 +256,24 @@ static boolean hasAtLeastOneGeoipProcessor(ClusterState clusterState) {
     }
 
     /**
-     * Retrieve list of pipelines that have at least one geoip processor.
+     * Retrieve the set of pipeline ids that have at least one geoip processor.
      * @param clusterState Cluster state.
      * @param downloadDatabaseOnPipelineCreation Filter the list to include only pipeline with the download_database_on_pipeline_creation
      *                                           matching the param.
-     * @return A list of {@link PipelineConfiguration} matching criteria.
+     * @return A set of pipeline ids matching criteria.
      */
     @SuppressWarnings("unchecked")
-    private static List<PipelineConfiguration> pipelineConfigurationsWithGeoIpProcessor(
-        ClusterState clusterState,
-        boolean downloadDatabaseOnPipelineCreation
-    ) {
-        List<PipelineConfiguration> pipelineDefinitions = IngestService.getPipelines(clusterState);
-        return pipelineDefinitions.stream().filter(pipelineConfig -> {
-            List<Map<String, Object>> processors = (List<Map<String, Object>>) pipelineConfig.getConfigAsMap().get(Pipeline.PROCESSORS_KEY);
-            return hasAtLeastOneGeoipProcessor(processors, downloadDatabaseOnPipelineCreation);
-        }).toList();
+    private static Set<String> pipelinesWithGeoIpProcessor(ClusterState clusterState, boolean downloadDatabaseOnPipelineCreation) {
+        List<PipelineConfiguration> configurations = IngestService.getPipelines(clusterState);
+        Set<String> ids = new HashSet<>();
+        // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph
+        for (PipelineConfiguration configuration : configurations) {
+            List<Map<String, Object>> processors = (List<Map<String, Object>>) configuration.getConfigAsMap().get(Pipeline.PROCESSORS_KEY);
+            if (hasAtLeastOneGeoipProcessor(processors, downloadDatabaseOnPipelineCreation)) {
+                ids.add(configuration.getId());
+            }
+        }
+        return Collections.unmodifiableSet(ids);
     }
 
     /**
@@ -283,7 +283,15 @@ private static List<PipelineConfiguration> pipelineConfigurationsWithGeoIpProces
      * @return true if a geoip processor is found in the processor list.
      */
     private static boolean hasAtLeastOneGeoipProcessor(List<Map<String, Object>> processors, boolean downloadDatabaseOnPipelineCreation) {
-        return processors != null && processors.stream().anyMatch(p -> hasAtLeastOneGeoipProcessor(p, downloadDatabaseOnPipelineCreation));
+        if (processors != null) {
+            // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph
+            for (Map<String, Object> processor : processors) {
+                if (hasAtLeastOneGeoipProcessor(processor, downloadDatabaseOnPipelineCreation)) {
+                    return true;
+                }
+            }
+        }
+        return false;
     }
 
     /**
@@ -317,7 +325,7 @@ private static boolean hasAtLeastOneGeoipProcessor(Map<String, Object> processor
     }
 
     /**
-     * Check if a processor config is has an on_failure clause containing at least a geoip processor.
+     * Check if a processor config has an on_failure clause containing at least a geoip processor.
      * @param processor Processor config.
      * @param downloadDatabaseOnPipelineCreation Should the download_database_on_pipeline_creation of the geoip processor be true or false.
      * @return true if a geoip processor is found in the processor list.
@@ -327,16 +335,17 @@ private static boolean isProcessorWithOnFailureGeoIpProcessor(
         Map<String, Object> processor,
         boolean downloadDatabaseOnPipelineCreation
     ) {
-        return processor != null
-            && processor.values()
-                .stream()
-                .anyMatch(
-                    value -> value instanceof Map
-                        && hasAtLeastOneGeoipProcessor(
-                            ((Map<String, List<Map<String, Object>>>) value).get("on_failure"),
-                            downloadDatabaseOnPipelineCreation
-                        )
-                );
+        // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph
+        for (Object value : processor.values()) {
+            if (value instanceof Map
+                && hasAtLeastOneGeoipProcessor(
+                    ((Map<String, List<Map<String, Object>>>) value).get("on_failure"),
+                    downloadDatabaseOnPipelineCreation
+                )) {
+                return true;
+            }
+        }
+        return false;
     }
 
     /**

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java

@@ -364,8 +364,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
             // MatchOnlyText never has norms, so we have to use the field names field
             BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name());
-            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
-            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup, sourceMode);
+            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup);
         }
 
         @Override

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -319,8 +319,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
             BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
-            return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup, sourceMode);
+            return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup);
         }
 
         @Override