Merge remote-tracking branch 'upstream/main' into include-default-index-sort

Author: jordan-powers

docs/changelog/131485.yaml

@@ -0,0 +1,5 @@
+pr: 131485
+summary: Run single phase aggregation when possible
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/135776.yaml

@@ -0,0 +1,6 @@
+pr: 135776
+summary: Fix KQL case-sensitivity for keyword fields in ES|QL
+area: Search
+type: bug
+issues:
+ - 135772

muted-tests.yml

@@ -594,6 +594,24 @@ tests:
 - class: org.elasticsearch.datastreams.TSDBIndexingIT
   method: testTsdbTemplatesNoKeywordFieldType
   issue: https://github.com/elastic/elasticsearch/issues/135746
+- class: org.elasticsearch.xpack.esql.expression.function.aggregate.IncreaseTests
+  method: testGroupingAggregate {TestCase=<small positive doubles>}
+  issue: https://github.com/elastic/elasticsearch/issues/135752
+- class: org.elasticsearch.xpack.esql.expression.function.aggregate.IncreaseTests
+  method: testGroupingAggregate {TestCase=<positive ints>}
+  issue: https://github.com/elastic/elasticsearch/issues/135753
+- class: org.elasticsearch.xpack.esql.expression.function.aggregate.IncreaseTests
+  method: testGroupingAggregate {TestCase=<positive longs>}
+  issue: https://github.com/elastic/elasticsearch/issues/135754
+- class: org.elasticsearch.xpack.esql.expression.function.aggregate.IncreaseTests
+  method: testGroupingAggregate {TestCase=<big positive doubles>}
+  issue: https://github.com/elastic/elasticsearch/issues/135755
+- class: org.elasticsearch.xpack.esql.ccq.AllSupportedFieldsIT
+  method: testFetchDenseVector {pref=null mode=time_series}
+  issue: https://github.com/elastic/elasticsearch/issues/135762
+- class: org.elasticsearch.xpack.esql.expression.function.aggregate.IrateTests
+  method: testGroupingAggregate {TestCase=<positive longs>}
+  issue: https://github.com/elastic/elasticsearch/issues/135775
 
 # Examples:
 #

rest-api-spec/src/main/resources/rest-api-spec/api/inference.put_contextualai.json

@@ -0,0 +1,41 @@
+{
+  "inference.put_contextualai": {
+    "documentation": {
+      "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put-contextualai",
+      "description": "Create an Contextual AI inference endpoint"
+    },
+    "stability": "stable",
+    "visibility": "public",
+    "headers": {
+      "accept": [
+        "application/json"
+      ],
+      "content_type": [
+        "application/json"
+      ]
+    },
+    "url": {
+      "paths": [
+        {
+          "path": "/_inference/{task_type}/{contextualai_inference_id}",
+          "methods": [
+            "PUT"
+          ],
+          "parts": {
+            "task_type": {
+              "type": "string",
+              "description": "The task type"
+            },
+            "contextualai_inference_id": {
+              "type": "string",
+              "description": "The inference Id"
+            }
+          }
+        }
+      ]
+    },
+    "body": {
+      "description": "The inference endpoint's task and service settings"
+    }
+  }
+}

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -2099,7 +2099,7 @@ public void onResponse(SearchResponse searchResponse) {
                         }
                     }
                 }
-                searchResponseMetrics.incrementResponseCount(responseCountTotalStatus);
+                searchResponseMetrics.incrementResponseCount(responseCountTotalStatus, searchRequestAttributes);
 
                 if (collectCCSTelemetry) {
                     extractCCSTelemetry(searchResponse);
@@ -2115,7 +2115,7 @@ public void onResponse(SearchResponse searchResponse) {
 
         @Override
         public void onFailure(Exception e) {
-            searchResponseMetrics.incrementResponseCount(SearchResponseMetrics.ResponseCountTotalStatus.FAILURE);
+            searchResponseMetrics.incrementResponseCount(SearchResponseMetrics.ResponseCountTotalStatus.FAILURE, searchRequestAttributes);
             if (collectCCSTelemetry) {
                 usageBuilder.setFailure(e);
                 recordTelemetry();

server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
+import org.elasticsearch.cluster.routing.ExpectedShardSizeEstimator;
 import org.elasticsearch.cluster.routing.RecoverySource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -45,7 +46,7 @@
  * <code>InternalClusterInfoService.shardIdentifierFromRouting(String)</code>
  * for the key used in the shardSizes map
  */
-public class ClusterInfo implements ChunkedToXContent, Writeable {
+public class ClusterInfo implements ChunkedToXContent, Writeable, ExpectedShardSizeEstimator.ShardSizeProvider {
 
     public static final ClusterInfo EMPTY = new ClusterInfo();
 
@@ -289,33 +290,11 @@ public Map<ShardId, Double> getShardWriteLoads() {
     /**
      * Returns the shard size for the given shardId or <code>null</code> if that metric is not available.
      */
+    @Override
     public Long getShardSize(ShardId shardId, boolean primary) {
         return shardSizes.get(shardIdentifierFromRouting(shardId, primary));
     }
 
-    /**
-     * Returns the shard size for the given shard routing or <code>null</code> if that metric is not available.
-     */
-    public Long getShardSize(ShardRouting shardRouting) {
-        return getShardSize(shardRouting.shardId(), shardRouting.primary());
-    }
-
-    /**
-     * Returns the shard size for the given shard routing or <code>defaultValue</code> it that metric is not available.
-     */
-    public long getShardSize(ShardRouting shardRouting, long defaultValue) {
-        Long shardSize = getShardSize(shardRouting);
-        return shardSize == null ? defaultValue : shardSize;
-    }
-
-    /**
-     * Returns the shard size for the given shard routing or <code>defaultValue</code> it that metric is not available.
-     */
-    public long getShardSize(ShardId shardId, boolean primary, long defaultValue) {
-        Long shardSize = getShardSize(shardId, primary);
-        return shardSize == null ? defaultValue : shardSize;
-    }
-
     /**
      * Returns the nodes absolute data-path the given shard is allocated on or <code>null</code> if the information is not available.
      */

server/src/main/java/org/elasticsearch/cluster/ClusterInfoSimulator.java

@@ -101,7 +101,7 @@ public void simulateShardStarted(ShardRouting shard) {
         var size = getExpectedShardSize(
             shard,
             shard.getExpectedShardSize(),
-            getClusterInfo(),
+            (shardId, primary) -> shardSizes.get(shardIdentifierFromRouting(shardId, primary)),
             allocation.snapshotShardSizeInfo(),
             project,
             allocation.routingTable(project.id())

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.Assertions;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.core.Tuple;
@@ -818,7 +819,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo
         var templateToValidate = indexTemplate.toBuilder().template(Template.builder(finalTemplate).settings(finalSettings)).build();
 
         validate(name, templateToValidate, additionalSettings);
-        validateDataStreamsStillReferenced(projectMetadata, name, templateToValidate);
+        maybeValidateDataStreamsStillReferenced(projectMetadata, name, templateToValidate);
         validateLifecycle(componentTemplates, name, templateToValidate, globalRetentionSettings.get(false));
         validateDataStreamOptions(componentTemplates, name, templateToValidate, globalRetentionSettings.get(true));
 
@@ -944,6 +945,43 @@ static void validateDataStreamOptions(
         }
     }
 
+    /**
+     * Maybe runs {@link #validateDataStreamsStillReferenced} if it looks like the new composite template could change data stream coverage.
+     */
+    private static void maybeValidateDataStreamsStillReferenced(
+        ProjectMetadata project,
+        String templateName,
+        ComposableIndexTemplate newTemplate
+    ) {
+        final ComposableIndexTemplate existingTemplate = project.templatesV2().get(templateName);
+        final Settings existingSettings = Optional.ofNullable(existingTemplate)
+            .map(ComposableIndexTemplate::template)
+            .map(Template::settings)
+            .orElse(Settings.EMPTY);
+        final Settings newSettings = Optional.ofNullable(newTemplate)
+            .map(ComposableIndexTemplate::template)
+            .map(Template::settings)
+            .orElse(Settings.EMPTY);
+        // We check whether anything relevant has changed that could affect data stream coverage and return early if not.
+        // These checks are based on the implementation of findV2Template and the data stream template check in this method.
+        // If we're adding a new template, we do the full check in case this template's priority changes coverage.
+        if (existingTemplate != null
+            && Objects.equals(existingTemplate.indexPatterns(), newTemplate.indexPatterns())
+            && Objects.equals(existingSettings.get(IndexMetadata.SETTING_INDEX_HIDDEN), newSettings.get(IndexMetadata.SETTING_INDEX_HIDDEN))
+            && Objects.equals(existingTemplate.getDataStreamTemplate() != null, newTemplate.getDataStreamTemplate() != null)
+            && Objects.equals(existingTemplate.priorityOrZero(), newTemplate.priorityOrZero())) {
+            if (Assertions.ENABLED) {
+                try {
+                    validateDataStreamsStillReferenced(project, templateName, newTemplate);
+                } catch (IllegalArgumentException e) {
+                    assert false : "Data stream reference validation took a shortcut but the full check failed: " + e.getMessage();
+                }
+            }
+            return;
+        }
+        validateDataStreamsStillReferenced(project, templateName, newTemplate);
+    }
+
     /**
      * Validate that by changing or adding {@code newTemplate}, there are
      * no unreferenced data streams. Note that this scenario is still possible
@@ -955,18 +993,16 @@ private static void validateDataStreamsStillReferenced(
         String templateName,
         ComposableIndexTemplate newTemplate
     ) {
-        final Set<String> dataStreams = project.dataStreams()
-            .entrySet()
-            .stream()
-            .filter(entry -> entry.getValue().isSystem() == false)
-            .map(Map.Entry::getKey)
-            .collect(Collectors.toSet());
-
         Function<Map<String, ComposableIndexTemplate>, Set<String>> findUnreferencedDataStreams = composableTemplates -> {
             final Set<String> unreferenced = new HashSet<>();
             // For each data stream that we have, see whether it's covered by a different
             // template (which is great), or whether it's now uncovered by any template
-            for (String dataStream : dataStreams) {
+            for (var dataStreamEntry : project.dataStreams().entrySet()) {
+                // Exclude system data streams
+                if (dataStreamEntry.getValue().isSystem()) {
+                    continue;
+                }
+                final String dataStream = dataStreamEntry.getKey();
                 final String matchingTemplate = findV2Template(project, composableTemplates.entrySet(), dataStream, false, false);
                 if (matchingTemplate == null) {
                     unreferenced.add(dataStream);

server/src/main/java/org/elasticsearch/cluster/routing/ExpectedShardSizeEstimator.java

@@ -9,7 +9,6 @@
 
 package org.elasticsearch.cluster.routing;
 
-import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
@@ -22,6 +21,36 @@
 
 public class ExpectedShardSizeEstimator {
 
+    public interface ShardSizeProvider {
+        /**
+         * Returns the shard size for the given shardId or <code>null</code> if that metric is not available.
+         */
+        Long getShardSize(ShardId shardId, boolean primary);
+
+        /**
+         * Returns the shard size for the given shard routing or <code>null</code> if that metric is not available.
+         */
+        default Long getShardSize(ShardRouting shardRouting) {
+            return getShardSize(shardRouting.shardId(), shardRouting.primary());
+        }
+
+        /**
+         * Returns the shard size for the given shard routing or <code>defaultValue</code> it that metric is not available.
+         */
+        default long getShardSize(ShardRouting shardRouting, long defaultValue) {
+            final var shardSize = getShardSize(shardRouting);
+            return shardSize == null ? defaultValue : shardSize;
+        }
+
+        /**
+         * Returns the shard size for the given shard routing or <code>defaultValue</code> it that metric is not available.
+         */
+        default long getShardSize(ShardId shardId, boolean primary, long defaultValue) {
+            final var shardSize = getShardSize(shardId, primary);
+            return shardSize == null ? defaultValue : shardSize;
+        }
+    }
+
     public static boolean shouldReserveSpaceForInitializingShard(ShardRouting shard, RoutingAllocation allocation) {
         return shouldReserveSpaceForInitializingShard(shard, allocation.metadata());
     }
@@ -69,7 +98,7 @@ public static boolean shouldReserveSpaceForInitializingShard(ShardRouting shard,
     public static long getExpectedShardSize(
         ShardRouting shard,
         long defaultValue,
-        ClusterInfo clusterInfo,
+        ShardSizeProvider shardSizeProvider,
         SnapshotShardSizeInfo snapshotShardSizeInfo,
         ProjectMetadata projectMetadata,
         RoutingTable routingTable
@@ -79,15 +108,15 @@ public static long getExpectedShardSize(
             && shard.active() == false
             && shard.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
             assert shard.primary() : "All replica shards are recovering from " + RecoverySource.Type.PEER;
-            return getExpectedSizeOfResizedShard(shard, defaultValue, indexMetadata, clusterInfo, projectMetadata, routingTable);
+            return getExpectedSizeOfResizedShard(shard, defaultValue, indexMetadata, shardSizeProvider, projectMetadata, routingTable);
         } else if (shard.active() == false && shard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT) {
             assert shard.primary() : "All replica shards are recovering from " + RecoverySource.Type.PEER;
             return snapshotShardSizeInfo.getShardSize(shard, defaultValue);
         } else {
-            var shardSize = clusterInfo.getShardSize(shard.shardId(), shard.primary());
+            var shardSize = shardSizeProvider.getShardSize(shard.shardId(), shard.primary());
             if (shardSize == null && shard.primary() == false) {
                 // derive replica size from corresponding primary
-                shardSize = clusterInfo.getShardSize(shard.shardId(), true);
+                shardSize = shardSizeProvider.getShardSize(shard.shardId(), true);
             }
             return shardSize == null ? defaultValue : shardSize;
         }
@@ -97,7 +126,7 @@ private static long getExpectedSizeOfResizedShard(
         ShardRouting shard,
         long defaultValue,
         IndexMetadata indexMetadata,
-        ClusterInfo clusterInfo,
+        ShardSizeProvider shardSizeProvider,
         ProjectMetadata projectMetadata,
         RoutingTable routingTable
     ) {
@@ -120,7 +149,7 @@ private static long getExpectedSizeOfResizedShard(
             for (int i = 0; i < indexRoutingTable.size(); i++) {
                 IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(i);
                 if (shardIds.contains(shardRoutingTable.shardId())) {
-                    targetShardSize += clusterInfo.getShardSize(shardRoutingTable.primaryShard(), 0);
+                    targetShardSize += shardSizeProvider.getShardSize(shardRoutingTable.primaryShard(), 0);
                 }
             }
         }

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java

@@ -43,6 +43,11 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             return Decision.single(Decision.Type.YES, NAME, "Decider is disabled");
         }
 
+        // Never reject allocation of an unassigned shard
+        if (shardRouting.assignedToNode() == false) {
+            return Decision.single(Decision.Type.YES, NAME, "Shard is unassigned. Decider takes no action.");
+        }
+
         // Check whether the shard being relocated has any write load estimate. If it does not, then this decider has no opinion.
         var allShardWriteLoads = allocation.clusterInfo().getShardWriteLoads();
         var shardWriteLoad = allShardWriteLoads.get(shardRouting.shardId());