Merge branch 'main' into reshard_disruption_test

lkts · web-flow · commit 93487b2a1e9b · 2025-07-11T14:20:14.000-07:00
diff --git a/docs/changelog/131056.yaml b/docs/changelog/131056.yaml
@@ -0,0 +1,5 @@
+pr: 131056
+summary: Add existing shards allocator settings to failure store allowed list
+area: Data streams
+type: bug
+issues: []
diff --git a/docs/changelog/131081.yaml b/docs/changelog/131081.yaml
@@ -0,0 +1,6 @@
+pr: 131081
+summary: Fix knn search error when dimensions are not set
+area: Vector Search
+type: bug
+issues:
+ - 129550
diff --git a/docs/changelog/131111.yaml b/docs/changelog/131111.yaml
@@ -0,0 +1,5 @@
+pr: 131111
+summary: Don't allow field caps to use semantic queries as index filters
+area: Search
+type: bug
+issues: []
diff --git a/docs/reference/query-languages/query-dsl/query-dsl-bool-query.md b/docs/reference/query-languages/query-dsl/query-dsl-bool-query.md
@@ -13,8 +13,8 @@ A query that matches documents matching boolean combinations of other queries. T
 | --- | --- |
 | `must` | The clause (query) must appear in matching documents and will contribute to the score. Each query defined under a `must` acts as a logical "AND", returning only documents that match *all* the specified queries. |
 | `should` | The clause (query) should appear in the matching document. Each query defined under a `should` acts as a logical "OR", returning documents that match *any* of the specified queries. |
-| `filter` | The clause (query) must appear in matching documents. However unlike`must` the score of the query will be ignored. Filter clauses are executedin [filter context](/reference/query-languages/query-dsl/query-filter-context.md), meaning that scoring is ignoredand clauses are considered for caching. Each query defined under a `filter` acts as a logical "AND", returning only documents that match *all* the specified queries. |
-| `must_not` | The clause (query) must not appear in the matchingdocuments. Clauses are executed in [filter context](/reference/query-languages/query-dsl/query-filter-context.md) meaningthat scoring is ignored and clauses are considered for caching. Because scoring isignored, a score of `0` for all documents is returned. Each query defined under a `must_not` acts as a logical "NOT", returning only documents that do not match any of the specified queries. |
+| `filter` | The clause (query) must appear in matching documents. However unlike `must` the score of the query will be ignored. Filter clauses are executed in [filter context](/reference/query-languages/query-dsl/query-filter-context.md), meaning that scoring is ignored and clauses are considered for caching. Each query defined under a `filter` acts as a logical "AND", returning only documents that match *all* the specified queries. |
+| `must_not` | The clause (query) must not appear in the matching documents. Clauses are executed in [filter context](/reference/query-languages/query-dsl/query-filter-context.md) meaning that scoring is ignored and clauses are considered for caching. Because scoring is ignored, a score of `0` for all documents is returned. Each query defined under a `must_not` acts as a logical "NOT", returning only documents that do not match any of the specified queries. |
 
 The `must` and `should` clauses function as logical AND, OR operators, contributing to the scoring of results. However, these results are not cached, which means repeated queries won't benefit from faster retrieval. In contrast, the `filter` and `must_not` clauses are used to include or exclude results without impacting the score, unless used within a `constant_score` query.
 
diff --git a/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md b/docs/reference/query-languages/query-dsl/query-dsl-knn-query.md
@@ -165,7 +165,6 @@ POST my-image-index/_search
 
 Knn query can be used as a part of hybrid search, where knn query is combined with other lexical queries. For example, the query below finds documents with `title` matching `mountain lake`, and combines them with the top 10 documents that have the closest image vectors to the `query_vector`. The combined documents are then scored and the top 3 top scored documents are returned.
 
-+
 
 ```console
 POST my-image-index/_search
diff --git a/muted-tests.yml b/muted-tests.yml
@@ -176,9 +176,6 @@ tests:
 - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
   method: test {yaml=snapshot.delete/10_basic/Delete a snapshot asynchronously}
   issue: https://github.com/elastic/elasticsearch/issues/122102
-- class: org.elasticsearch.smoketest.SmokeTestMonitoringWithSecurityIT
-  method: testHTTPExporterWithSSL
-  issue: https://github.com/elastic/elasticsearch/issues/122220
 - class: org.elasticsearch.blocks.SimpleBlocksIT
   method: testConcurrentAddBlock
   issue: https://github.com/elastic/elasticsearch/issues/122324
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml
@@ -670,3 +670,36 @@ setup:
           properties:
             embedding:
               type: dense_vector
+
+
+---
+"Searching with no data dimensions specified":
+  - requires:
+      cluster_features: "search.vectors.no_dimensions_bugfix"
+      reason: "Search with no dimensions bugfix"
+
+  - do:
+      indices.create:
+        index: empty-test
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                index: true
+
+  - do:
+      search:
+        index: empty-test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: vector
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+            k: 3
+            num_candidates: 3
+            rescore_vector:
+              oversample: 1.5
+            similarity: 0.1
+
+  - match: { hits.total.value: 0 }
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java
@@ -18,7 +18,13 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.BoostingQueryBuilder;
+import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
+import org.elasticsearch.index.query.DisMaxQueryBuilder;
+import org.elasticsearch.index.query.NestedQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -268,9 +274,53 @@ public ActionRequestValidationException validate() {
         if (fields == null || fields.length == 0) {
             validationException = ValidateActions.addValidationError("no fields specified", validationException);
         }
+
+        // Band-aid fix for https://github.com/elastic/elasticsearch/issues/116106.
+        // Semantic queries are high-recall queries, making them poor filters and effectively the same as an exists query when used in that
+        // context.
+        if (containsSemanticQuery(indexFilter)) {
+            validationException = ValidateActions.addValidationError(
+                "index filter cannot contain semantic queries. Use an exists query instead.",
+                validationException
+            );
+        }
+
         return validationException;
     }
 
+    /**
+     * Recursively checks if a query builder contains any semantic queries
+     */
+    private static boolean containsSemanticQuery(QueryBuilder queryBuilder) {
+        boolean containsSemanticQuery = false;
+
+        if (queryBuilder == null) {
+            return containsSemanticQuery;
+        }
+
+        if ("semantic".equals(queryBuilder.getWriteableName())) {
+            containsSemanticQuery = true;
+        } else if (queryBuilder instanceof BoolQueryBuilder boolQuery) {
+            containsSemanticQuery = boolQuery.must().stream().anyMatch(FieldCapabilitiesRequest::containsSemanticQuery)
+                || boolQuery.mustNot().stream().anyMatch(FieldCapabilitiesRequest::containsSemanticQuery)
+                || boolQuery.should().stream().anyMatch(FieldCapabilitiesRequest::containsSemanticQuery)
+                || boolQuery.filter().stream().anyMatch(FieldCapabilitiesRequest::containsSemanticQuery);
+        } else if (queryBuilder instanceof DisMaxQueryBuilder disMaxQuery) {
+            containsSemanticQuery = disMaxQuery.innerQueries().stream().anyMatch(FieldCapabilitiesRequest::containsSemanticQuery);
+        } else if (queryBuilder instanceof NestedQueryBuilder nestedQuery) {
+            containsSemanticQuery = containsSemanticQuery(nestedQuery.query());
+        } else if (queryBuilder instanceof BoostingQueryBuilder boostingQuery) {
+            containsSemanticQuery = containsSemanticQuery(boostingQuery.positiveQuery())
+                || containsSemanticQuery(boostingQuery.negativeQuery());
+        } else if (queryBuilder instanceof ConstantScoreQueryBuilder constantScoreQuery) {
+            containsSemanticQuery = containsSemanticQuery(constantScoreQuery.innerQuery());
+        } else if (queryBuilder instanceof FunctionScoreQueryBuilder functionScoreQuery) {
+            containsSemanticQuery = containsSemanticQuery(functionScoreQuery.query());
+        }
+
+        return containsSemanticQuery;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java
@@ -10,6 +10,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.cluster.routing.allocation.DataTier;
+import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -40,7 +41,9 @@ public class DataStreamFailureStoreDefinition {
         IndexMetadata.SETTING_NUMBER_OF_SHARDS,
         IndexMetadata.SETTING_NUMBER_OF_REPLICAS,
         IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS,
-        IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()
+        IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(),
+        // Different recovery implementations may be provided on the index which need to be preserved.
+        ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING.getKey()
     );
     public static final Set<String> SUPPORTED_USER_SETTINGS_PREFIXES = Set.of(
         IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".",
diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
@@ -577,6 +577,7 @@ public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) {
     }
 
     public static class TaskTrackingConfig {
+        // This is a random starting point alpha.
         public static final double DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST = 0.3;
 
         private final boolean trackExecutionTime;
@@ -597,17 +598,6 @@ public static class TaskTrackingConfig {
             DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST
         );
 
-        public TaskTrackingConfig(boolean trackOngoingTasks, double executionTimeEWMAAlpha) {
-            this(true, trackOngoingTasks, false, executionTimeEWMAAlpha);
-        }
-
-        /**
-         * Execution tracking enabled constructor, with extra options to enable further specialized tracking.
-         */
-        public TaskTrackingConfig(boolean trackOngoingTasks, boolean trackMaxQueueLatency, double executionTimeEwmaAlpha) {
-            this(true, trackOngoingTasks, trackMaxQueueLatency, executionTimeEwmaAlpha);
-        }
-
         /**
          * @param trackExecutionTime Whether to track execution stats
          * @param trackOngoingTasks Whether to track ongoing task execution time, not just finished tasks
@@ -641,6 +631,39 @@ public boolean trackMaxQueueLatency() {
         public double getExecutionTimeEwmaAlpha() {
             return executionTimeEwmaAlpha;
         }
+
+        public static Builder builder() {
+            return new Builder();
+        }
+
+        public static class Builder {
+            private boolean trackExecutionTime = false;
+            private boolean trackOngoingTasks = false;
+            private boolean trackMaxQueueLatency = false;
+            private double ewmaAlpha = DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST;
+
+            public Builder() {}
+
+            public Builder trackExecutionTime(double alpha) {
+                trackExecutionTime = true;
+                ewmaAlpha = alpha;
+                return this;
+            }
+
+            public Builder trackOngoingTasks() {
+                trackOngoingTasks = true;
+                return this;
+            }
+
+            public Builder trackMaxQueueLatency() {
+                trackMaxQueueLatency = true;
+                return this;
+            }
+
+            public TaskTrackingConfig build() {
+                return new TaskTrackingConfig(trackExecutionTime, trackOngoingTasks, trackMaxQueueLatency, ewmaAlpha);
+            }
+        }
     }
 
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.KnnByteVectorQuery;
 import org.apache.lucene.search.KnnFloatVectorQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PatienceKnnVectorQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.BitSetProducer;
@@ -2530,6 +2531,9 @@ public Query createKnnQuery(
                     "to perform knn search on field [" + name() + "], its mapping must have [index] set to [true]"
                 );
             }
+            if (dims == null) {
+                return new MatchNoDocsQuery("No data has been indexed for field [" + name() + "]");
+            }
             KnnSearchStrategy knnSearchStrategy = heuristic.getKnnSearchStrategy();
             return switch (getElementType()) {
                 case BYTE -> createKnnByteQuery(
diff --git a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java
@@ -32,6 +32,7 @@ public Set<NodeFeature> getFeatures() {
     public static final NodeFeature INT_SORT_FOR_INT_SHORT_BYTE_FIELDS = new NodeFeature("search.sort.int_sort_for_int_short_byte_fields");
     static final NodeFeature MULTI_MATCH_CHECKS_POSITIONS = new NodeFeature("search.multi.match.checks.positions");
     public static final NodeFeature BBQ_HNSW_DEFAULT_INDEXING = new NodeFeature("search.vectors.mappers.default_bbq_hnsw");
+    public static final NodeFeature SEARCH_WITH_NO_DIMENSIONS_BUGFIX = new NodeFeature("search.vectors.no_dimensions_bugfix");
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
@@ -41,7 +42,8 @@ public Set<NodeFeature> getTestFeatures() {
             RESCORER_MISSING_FIELD_BAD_REQUEST,
             INT_SORT_FOR_INT_SHORT_BYTE_FIELDS,
             MULTI_MATCH_CHECKS_POSITIONS,
-            BBQ_HNSW_DEFAULT_INDEXING
+            BBQ_HNSW_DEFAULT_INDEXING,
+            SEARCH_WITH_NO_DIMENSIONS_BUGFIX
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java b/server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java
@@ -56,7 +56,11 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 allocatedProcessors,
                 // 10,000 for all nodes with 8 cores or fewer. Scale up once we have more than 8 cores.
                 Math.max(allocatedProcessors * 750, 10000),
-                new EsExecutors.TaskTrackingConfig(true, true, indexAutoscalingEWMA)
+                EsExecutors.TaskTrackingConfig.builder()
+                    .trackOngoingTasks()
+                    .trackMaxQueueLatency()
+                    .trackExecutionTime(indexAutoscalingEWMA)
+                    .build()
             )
         );
         int searchOrGetThreadPoolSize = ThreadPool.searchOrGetThreadPoolSize(allocatedProcessors);
@@ -81,7 +85,7 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 ThreadPool.Names.SEARCH,
                 searchOrGetThreadPoolSize,
                 searchOrGetThreadPoolSize * 1000,
-                new EsExecutors.TaskTrackingConfig(true, searchAutoscalingEWMA)
+                EsExecutors.TaskTrackingConfig.builder().trackOngoingTasks().trackExecutionTime(searchAutoscalingEWMA).build()
             )
         );
         result.put(
@@ -91,7 +95,7 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 ThreadPool.Names.SEARCH_COORDINATION,
                 halfProc,
                 1000,
-                new EsExecutors.TaskTrackingConfig(true, searchAutoscalingEWMA)
+                EsExecutors.TaskTrackingConfig.builder().trackOngoingTasks().trackExecutionTime(searchAutoscalingEWMA).build()
             )
         );
         result.put(
@@ -195,7 +199,7 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 ThreadPool.Names.SYSTEM_WRITE,
                 halfProcMaxAt5,
                 1000,
-                new EsExecutors.TaskTrackingConfig(true, indexAutoscalingEWMA),
+                EsExecutors.TaskTrackingConfig.builder().trackOngoingTasks().trackExecutionTime(indexAutoscalingEWMA).build(),
                 true
             )
         );
@@ -228,7 +232,7 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 ThreadPool.Names.SYSTEM_CRITICAL_WRITE,
                 halfProcMaxAt5,
                 1500,
-                new EsExecutors.TaskTrackingConfig(true, indexAutoscalingEWMA),
+                EsExecutors.TaskTrackingConfig.builder().trackOngoingTasks().trackExecutionTime(indexAutoscalingEWMA).build(),
                 true
             )
         );
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -35,6 +35,7 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
+import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
@@ -791,6 +792,55 @@ public boolean overrulesTemplateAndRequestSettings() {
         assertThat(aggregatedIndexSettings.get("other_setting"), equalTo("other_value"));
     }
 
+    /**
+     * When a failure store index is created, we must filter out any unsupported settings from the create request or from the template that
+     * may have been provided by users in the create request or from the original data stream template. An exception to this is any settings
+     * that have been provided by index setting providers which should be considered default values on indices.
+     */
+    public void testAggregateSettingsProviderIsNotFilteredOnFailureStore() {
+        IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> {
+            builder.settings(Settings.builder().put("template_setting", "value1"));
+        });
+        ProjectMetadata projectMetadata = ProjectMetadata.builder(projectId).templates(Map.of("template_1", templateMetadata)).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(projectMetadata).build();
+        var request = new CreateIndexClusterStateUpdateRequest("create index", projectId, "test", "test").settings(
+            Settings.builder().put("request_setting", "value2").build()
+        ).isFailureIndex(true);
+
+        Settings aggregatedIndexSettings = aggregateIndexSettings(
+            clusterState,
+            request,
+            templateMetadata.settings(),
+            null,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Set.of(new IndexSettingProvider() {
+                @Override
+                public Settings getAdditionalIndexSettings(
+                    String indexName,
+                    String dataStreamName,
+                    IndexMode templateIndexMode,
+                    ProjectMetadata projectMetadata,
+                    Instant resolvedAt,
+                    Settings indexTemplateAndCreateRequestSettings,
+                    List<CompressedXContent> combinedTemplateMappings
+                ) {
+                    return Settings.builder().put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING.getKey(), "override").build();
+                }
+
+                @Override
+                public boolean overrulesTemplateAndRequestSettings() {
+                    return true;
+                }
+            })
+        );
+
+        assertThat(aggregatedIndexSettings.get("template_setting"), nullValue());
+        assertThat(aggregatedIndexSettings.get(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING.getKey()), equalTo("override"));
+    }
+
     public void testAggregateSettingsProviderOverrulesNullFromRequest() {
         IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> {
             builder.settings(Settings.builder().put("template_setting", "value1"));
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java
diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/TaskExecutionTimeTrackingEsThreadPoolExecutorTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/TaskExecutionTimeTrackingEsThreadPoolExecutorTests.java
diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java
diff --git a/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java b/x-pack/plugin/blob-cache/src/test/java/org/elasticsearch/blobcache/shared/SharedBlobCacheServiceTests.java
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/StoredFieldsSequentialIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/StoredFieldsSequentialIT.java
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/knn-function.csv-spec
diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestSemanticIndexFilterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestSemanticIndexFilterTests.java
