Merge branch 'main' into fix-failure-store-removing-allocator-settings

Author: jbaiera

docs/changelog/131081.yaml

@@ -0,0 +1,6 @@
+pr: 131081
+summary: Fix knn search error when dimensions are not set
+area: Vector Search
+type: bug
+issues:
+ - 129550

docs/reference/query-languages/query-dsl/query-dsl-bool-query.md

@@ -13,8 +13,8 @@ A query that matches documents matching boolean combinations of other queries. T
 | --- | --- |
 | `must` | The clause (query) must appear in matching documents and will contribute to the score. Each query defined under a `must` acts as a logical "AND", returning only documents that match *all* the specified queries. |
 | `should` | The clause (query) should appear in the matching document. Each query defined under a `should` acts as a logical "OR", returning documents that match *any* of the specified queries. |
-| `filter` | The clause (query) must appear in matching documents. However unlike`must` the score of the query will be ignored. Filter clauses are executedin [filter context](/reference/query-languages/query-dsl/query-filter-context.md), meaning that scoring is ignoredand clauses are considered for caching. Each query defined under a `filter` acts as a logical "AND", returning only documents that match *all* the specified queries. |
-| `must_not` | The clause (query) must not appear in the matchingdocuments. Clauses are executed in [filter context](/reference/query-languages/query-dsl/query-filter-context.md) meaningthat scoring is ignored and clauses are considered for caching. Because scoring isignored, a score of `0` for all documents is returned. Each query defined under a `must_not` acts as a logical "NOT", returning only documents that do not match any of the specified queries. |
+| `filter` | The clause (query) must appear in matching documents. However unlike `must` the score of the query will be ignored. Filter clauses are executed in [filter context](/reference/query-languages/query-dsl/query-filter-context.md), meaning that scoring is ignored and clauses are considered for caching. Each query defined under a `filter` acts as a logical "AND", returning only documents that match *all* the specified queries. |
+| `must_not` | The clause (query) must not appear in the matching documents. Clauses are executed in [filter context](/reference/query-languages/query-dsl/query-filter-context.md) meaning that scoring is ignored and clauses are considered for caching. Because scoring is ignored, a score of `0` for all documents is returned. Each query defined under a `must_not` acts as a logical "NOT", returning only documents that do not match any of the specified queries. |
 
 The `must` and `should` clauses function as logical AND, OR operators, contributing to the scoring of results. However, these results are not cached, which means repeated queries won't benefit from faster retrieval. In contrast, the `filter` and `must_not` clauses are used to include or exclude results without impacting the score, unless used within a `constant_score` query.
 

docs/reference/query-languages/query-dsl/query-dsl-knn-query.md

@@ -165,7 +165,6 @@ POST my-image-index/_search
 
 Knn query can be used as a part of hybrid search, where knn query is combined with other lexical queries. For example, the query below finds documents with `title` matching `mountain lake`, and combines them with the top 10 documents that have the closest image vectors to the `query_vector`. The combined documents are then scored and the top 3 top scored documents are returned.
 
-+
 
 ```console
 POST my-image-index/_search

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml

@@ -670,3 +670,36 @@ setup:
           properties:
             embedding:
               type: dense_vector
+
+
+---
+"Searching with no data dimensions specified":
+  - requires:
+      cluster_features: "search.vectors.no_dimensions_bugfix"
+      reason: "Search with no dimensions bugfix"
+
+  - do:
+      indices.create:
+        index: empty-test
+        body:
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                index: true
+
+  - do:
+      search:
+        index: empty-test
+        body:
+          fields: [ "name" ]
+          knn:
+            field: vector
+            query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+            k: 3
+            num_candidates: 3
+            rescore_vector:
+              oversample: 1.5
+            similarity: 0.1
+
+  - match: { hits.total.value: 0 }

server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java

@@ -577,6 +577,7 @@ public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) {
     }
 
     public static class TaskTrackingConfig {
+        // This is a random starting point alpha.
         public static final double DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST = 0.3;
 
         private final boolean trackExecutionTime;
@@ -597,17 +598,6 @@ public static class TaskTrackingConfig {
             DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST
         );
 
-        public TaskTrackingConfig(boolean trackOngoingTasks, double executionTimeEWMAAlpha) {
-            this(true, trackOngoingTasks, false, executionTimeEWMAAlpha);
-        }
-
-        /**
-         * Execution tracking enabled constructor, with extra options to enable further specialized tracking.
-         */
-        public TaskTrackingConfig(boolean trackOngoingTasks, boolean trackMaxQueueLatency, double executionTimeEwmaAlpha) {
-            this(true, trackOngoingTasks, trackMaxQueueLatency, executionTimeEwmaAlpha);
-        }
-
         /**
          * @param trackExecutionTime Whether to track execution stats
          * @param trackOngoingTasks Whether to track ongoing task execution time, not just finished tasks
@@ -641,6 +631,39 @@ public boolean trackMaxQueueLatency() {
         public double getExecutionTimeEwmaAlpha() {
             return executionTimeEwmaAlpha;
         }
+
+        public static Builder builder() {
+            return new Builder();
+        }
+
+        public static class Builder {
+            private boolean trackExecutionTime = false;
+            private boolean trackOngoingTasks = false;
+            private boolean trackMaxQueueLatency = false;
+            private double ewmaAlpha = DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST;
+
+            public Builder() {}
+
+            public Builder trackExecutionTime(double alpha) {
+                trackExecutionTime = true;
+                ewmaAlpha = alpha;
+                return this;
+            }
+
+            public Builder trackOngoingTasks() {
+                trackOngoingTasks = true;
+                return this;
+            }
+
+            public Builder trackMaxQueueLatency() {
+                trackMaxQueueLatency = true;
+                return this;
+            }
+
+            public TaskTrackingConfig build() {
+                return new TaskTrackingConfig(trackExecutionTime, trackOngoingTasks, trackMaxQueueLatency, ewmaAlpha);
+            }
+        }
     }
 
 }

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -35,6 +35,7 @@
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.KnnByteVectorQuery;
 import org.apache.lucene.search.KnnFloatVectorQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PatienceKnnVectorQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.BitSetProducer;
@@ -2530,6 +2531,9 @@ public Query createKnnQuery(
                     "to perform knn search on field [" + name() + "], its mapping must have [index] set to [true]"
                 );
             }
+            if (dims == null) {
+                return new MatchNoDocsQuery("No data has been indexed for field [" + name() + "]");
+            }
             KnnSearchStrategy knnSearchStrategy = heuristic.getKnnSearchStrategy();
             return switch (getElementType()) {
                 case BYTE -> createKnnByteQuery(

server/src/main/java/org/elasticsearch/search/SearchFeatures.java

@@ -32,6 +32,7 @@ public Set<NodeFeature> getFeatures() {
     public static final NodeFeature INT_SORT_FOR_INT_SHORT_BYTE_FIELDS = new NodeFeature("search.sort.int_sort_for_int_short_byte_fields");
     static final NodeFeature MULTI_MATCH_CHECKS_POSITIONS = new NodeFeature("search.multi.match.checks.positions");
     public static final NodeFeature BBQ_HNSW_DEFAULT_INDEXING = new NodeFeature("search.vectors.mappers.default_bbq_hnsw");
+    public static final NodeFeature SEARCH_WITH_NO_DIMENSIONS_BUGFIX = new NodeFeature("search.vectors.no_dimensions_bugfix");
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
@@ -41,7 +42,8 @@ public Set<NodeFeature> getTestFeatures() {
             RESCORER_MISSING_FIELD_BAD_REQUEST,
             INT_SORT_FOR_INT_SHORT_BYTE_FIELDS,
             MULTI_MATCH_CHECKS_POSITIONS,
-            BBQ_HNSW_DEFAULT_INDEXING
+            BBQ_HNSW_DEFAULT_INDEXING,
+            SEARCH_WITH_NO_DIMENSIONS_BUGFIX
         );
     }
 }

server/src/main/java/org/elasticsearch/threadpool/DefaultBuiltInExecutorBuilders.java

@@ -56,7 +56,11 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 allocatedProcessors,
                 // 10,000 for all nodes with 8 cores or fewer. Scale up once we have more than 8 cores.
                 Math.max(allocatedProcessors * 750, 10000),
-                new EsExecutors.TaskTrackingConfig(true, true, indexAutoscalingEWMA)
+                EsExecutors.TaskTrackingConfig.builder()
+                    .trackOngoingTasks()
+                    .trackMaxQueueLatency()
+                    .trackExecutionTime(indexAutoscalingEWMA)
+                    .build()
             )
         );
         int searchOrGetThreadPoolSize = ThreadPool.searchOrGetThreadPoolSize(allocatedProcessors);
@@ -81,7 +85,7 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 ThreadPool.Names.SEARCH,
                 searchOrGetThreadPoolSize,
                 searchOrGetThreadPoolSize * 1000,
-                new EsExecutors.TaskTrackingConfig(true, searchAutoscalingEWMA)
+                EsExecutors.TaskTrackingConfig.builder().trackOngoingTasks().trackExecutionTime(searchAutoscalingEWMA).build()
             )
         );
         result.put(
@@ -91,7 +95,7 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 ThreadPool.Names.SEARCH_COORDINATION,
                 halfProc,
                 1000,
-                new EsExecutors.TaskTrackingConfig(true, searchAutoscalingEWMA)
+                EsExecutors.TaskTrackingConfig.builder().trackOngoingTasks().trackExecutionTime(searchAutoscalingEWMA).build()
             )
         );
         result.put(
@@ -195,7 +199,7 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 ThreadPool.Names.SYSTEM_WRITE,
                 halfProcMaxAt5,
                 1000,
-                new EsExecutors.TaskTrackingConfig(true, indexAutoscalingEWMA),
+                EsExecutors.TaskTrackingConfig.builder().trackOngoingTasks().trackExecutionTime(indexAutoscalingEWMA).build(),
                 true
             )
         );
@@ -228,7 +232,7 @@ public Map<String, ExecutorBuilder> getBuilders(Settings settings, int allocated
                 ThreadPool.Names.SYSTEM_CRITICAL_WRITE,
                 halfProcMaxAt5,
                 1500,
-                new EsExecutors.TaskTrackingConfig(true, indexAutoscalingEWMA),
+                EsExecutors.TaskTrackingConfig.builder().trackOngoingTasks().trackExecutionTime(indexAutoscalingEWMA).build(),
                 true
             )
         );

server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java

@@ -675,6 +675,7 @@ public void testScalingWithTaskTimeTracking() {
         final int max = between(min + 1, 6);
 
         {
+            var executionTimeEwma = randomDoubleBetween(0.01, 0.1, true);
             ThreadPoolExecutor pool = EsExecutors.newScaling(
                 getClass().getName() + "/" + getTestName(),
                 min,
@@ -684,7 +685,9 @@ public void testScalingWithTaskTimeTracking() {
                 randomBoolean(),
                 EsExecutors.daemonThreadFactory("test"),
                 threadContext,
-                new EsExecutors.TaskTrackingConfig(randomBoolean(), randomDoubleBetween(0.01, 0.1, true))
+                randomBoolean()
+                    ? EsExecutors.TaskTrackingConfig.builder().trackOngoingTasks().trackExecutionTime(executionTimeEwma).build()
+                    : EsExecutors.TaskTrackingConfig.builder().trackExecutionTime(executionTimeEwma).build()
             );
             assertThat(pool, instanceOf(TaskExecutionTimeTrackingEsThreadPoolExecutor.class));
         }

server/src/test/java/org/elasticsearch/common/util/concurrent/TaskExecutionTimeTrackingEsThreadPoolExecutorTests.java

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.common.metrics.ExponentialBucketHistogram;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.EsExecutors.TaskTrackingConfig;
 import org.elasticsearch.telemetry.InstrumentType;
 import org.elasticsearch.telemetry.Measurement;
 import org.elasticsearch.telemetry.RecordingMeterRegistry;
@@ -51,7 +50,12 @@ public void testExecutionEWMACalculation() throws Exception {
             EsExecutors.daemonThreadFactory("queuetest"),
             new EsAbortPolicy(),
             context,
-            new TaskTrackingConfig(randomBoolean(), DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+            randomBoolean()
+                ? EsExecutors.TaskTrackingConfig.builder()
+                    .trackOngoingTasks()
+                    .trackExecutionTime(DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+                    .build()
+                : EsExecutors.TaskTrackingConfig.builder().trackExecutionTime(DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST).build()
         );
         executor.prestartAllCoreThreads();
         logger.info("--> executor: {}", executor);
@@ -109,7 +113,16 @@ public void testMaxQueueLatency() throws Exception {
             EsExecutors.daemonThreadFactory("queue-latency-test"),
             new EsAbortPolicy(),
             context,
-            new TaskTrackingConfig(randomBoolean(), true, DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+            randomBoolean()
+                ? EsExecutors.TaskTrackingConfig.builder()
+                    .trackOngoingTasks()
+                    .trackMaxQueueLatency()
+                    .trackExecutionTime(DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+                    .build()
+                : EsExecutors.TaskTrackingConfig.builder()
+                    .trackMaxQueueLatency()
+                    .trackExecutionTime(DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+                    .build()
         );
         try {
             executor.prestartAllCoreThreads();
@@ -153,7 +166,12 @@ public void testExceptionThrowingTask() throws Exception {
             EsExecutors.daemonThreadFactory("queuetest"),
             new EsAbortPolicy(),
             context,
-            new TaskTrackingConfig(randomBoolean(), DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+            randomBoolean()
+                ? EsExecutors.TaskTrackingConfig.builder()
+                    .trackOngoingTasks()
+                    .trackExecutionTime(DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+                    .build()
+                : EsExecutors.TaskTrackingConfig.builder().trackExecutionTime(DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST).build()
         );
         executor.prestartAllCoreThreads();
         logger.info("--> executor: {}", executor);
@@ -185,7 +203,10 @@ public void testGetOngoingTasks() throws Exception {
             EsExecutors.daemonThreadFactory("queuetest"),
             new EsAbortPolicy(),
             context,
-            new TaskTrackingConfig(true, DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+            EsExecutors.TaskTrackingConfig.builder()
+                .trackOngoingTasks()
+                .trackExecutionTime(DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+                .build()
         );
         var taskRunningLatch = new CountDownLatch(1);
         var exitTaskLatch = new CountDownLatch(1);
@@ -220,7 +241,10 @@ public void testQueueLatencyHistogramMetrics() {
             EsExecutors.daemonThreadFactory("queuetest"),
             new EsAbortPolicy(),
             new ThreadContext(Settings.EMPTY),
-            new TaskTrackingConfig(true, DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+            EsExecutors.TaskTrackingConfig.builder()
+                .trackOngoingTasks()
+                .trackExecutionTime(DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST)
+                .build()
         );
         executor.setupMetrics(meterRegistry, threadPoolName);