switch to dynamic index setting

Author: tteofili

docs/reference/elasticsearch/index-settings/index-modules.md

@@ -259,3 +259,6 @@ $$$index-esql-stored-fields-sequential-proportion$$$
 
 `index.esql.stored_fields_sequential_proportion`
 :   Tuning parameter for deciding when {{esql}} will load [Stored fields](/reference/elasticsearch/rest-apis/retrieve-selected-fields.md#stored-fields) using a strategy tuned for loading dense sequence of documents. Allows values between 0.0 and 1.0 and defaults to 0.2. Indices with documents smaller than 10kb may see speed improvements loading `text` fields by setting this lower.
+
+$$$index-dense-vector-hnsw-filter-heuristic$$$ `index.dense_vector.hnsw_filter_heuristic`
+:   Whether to apply _patience_ based early termination strategy to knn queries over HNSW graphs (see [paper](https://cs.uwaterloo.ca/~jimmylin/publications/Teofili_Lin_ECIR2025.pdf)). This is only applicable to `dense_vector` fields with `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. Defaults to `false`.

docs/reference/elasticsearch/mapping-reference/dense-vector.md

@@ -281,9 +281,6 @@ $$$dense-vector-index-options$$$
     :   In case a knn query specifies a `rescore_vector` parameter, the query `rescore_vector` parameter will be used instead.
     :   See [oversampling and rescoring quantized vectors](docs-content://solutions/search/vector/knn.md#dense-vector-knn-search-rescoring) for details.
 :::::
-
-`early_termination`
-:   (Optional, boolean) Apply _patience_ based early termination strategy to knn queries over HNSW graphs (see [paper](https://cs.uwaterloo.ca/~jimmylin/publications/Teofili_Lin_ECIR2025.pdf)). This is expected to produce Only applicable to `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types.
 ::::
 
 

rest-api-spec/build.gradle

@@ -90,6 +90,4 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("indices.create/21_synthetic_source_stored/field param - keep root array", "Synthetic source keep arrays now stores leaf arrays natively")
   task.skipTest("cluster.info/30_info_thread_pool/Cluster HTTP Info", "The search_throttled thread pool has been removed")
   task.skipTest("synonyms/80_synonyms_from_index/Fail loading synonyms from index if synonyms_set doesn't exist", "Synonyms do no longer fail if the synonyms_set doesn't exist")
-  task.skipTest("search.vectors/80_dense_vector_indexed_by_default/Default index options for dense_vector", "Introduced early_termination option")
-  task.skipTest("search.vectors/80_dense_vector_indexed_by_default/Indexed by default with specified similarity and index options", "Introduced early_termination option")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/240_knn_search_early_termination.yml

@@ -25,7 +25,6 @@ setup:
   - match: { test.mappings.properties.vector.dims: 5 }
   - match: { test.mappings.properties.vector.index: true }
   - match: { test.mappings.properties.vector.similarity: cosine }
-
 ---
 "Indexed by default with specified similarity and index options":
   - do:
@@ -49,48 +48,19 @@ setup:
       indices.get_mapping:
         index: test
 
-  - match: { test.mappings.properties.vector.type: dense_vector }
-  - match: { test.mappings.properties.vector.dims: 5 }
-  - match: { test.mappings.properties.vector.index: true }
-  - match: { test.mappings.properties.vector.similarity: dot_product }
-  - match: { test.mappings.properties.vector.index_options.type: hnsw }
-  - match: { test.mappings.properties.vector.index_options.m: 32 }
-  - match: { test.mappings.properties.vector.index_options.ef_construction: 200 }
-
----
-"Indexed by default with specified similarity and index options with early_termination":
-  - requires:
-      cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ]
-      reason: "requires early termination for hnsw to be exposed"
-  - do:
-      indices.create:
-        index: test
-        body:
-          mappings:
-            properties:
-              vector:
-                type: dense_vector
-                dims: 5
-                similarity: dot_product
-                index_options:
-                  type: hnsw
-                  m: 32
-                  ef_construction: 200
-
-  - match: { acknowledged: true }
-
-  - do:
-      indices.get_mapping:
-        index: test
-
-  - match: { test.mappings.properties.vector.type: dense_vector }
-  - match: { test.mappings.properties.vector.dims: 5 }
-  - match: { test.mappings.properties.vector.index: true }
-  - match: { test.mappings.properties.vector.similarity: dot_product }
-  - match: { test.mappings.properties.vector.index_options.type: hnsw }
-  - match: { test.mappings.properties.vector.index_options.m: 32 }
-  - match: { test.mappings.properties.vector.index_options.ef_construction: 200 }
-  - match: { test.mappings.properties.vector.index_options.early_termination: false }
+  - match:
+      test:
+        mappings:
+          properties:
+            vector:
+              type: dense_vector
+              dims: 5
+              index: true
+              similarity: dot_product
+              index_options:
+                type: hnsw
+                m: 32
+                ef_construction: 200
 
 ---
 "Not indexed vector":
@@ -156,9 +126,6 @@ setup:
   - requires:
       cluster_features: "gte_v8.14.0"
       reason: 'dense_vector indexed as int8_hnsw by default was added in 8.14'
-  - requires:
-      cluster_features: [ "search.vectors.mappers.expose_hnsw_early_termination" ]
-      reason: "requires early termination for hnsw to be exposed"
   - do:
       indices.create:
         index: test_default_index_options
@@ -180,7 +147,6 @@ setup:
   - match: { test_default_index_options.mappings.properties.vector.index: true }
   - match: { test_default_index_options.mappings.properties.vector.similarity: cosine }
   - match: { test_default_index_options.mappings.properties.vector.index_options.type: int8_hnsw }
-  - match: { test_default_index_options.mappings.properties.vector.index_options.early_termination: false }
 ---
 "Default index options for dense_vector element type byte":
   - requires:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/80_dense_vector_indexed_by_default.yml

@@ -127,4 +127,55 @@ public void testFilteredQueryStrategy() {
         });
     }
 
+    public void testHnswEarlyTerminationQuery() {
+        float[] vector = new float[16];
+        randomVector(vector, 25);
+        int upperLimit = 35;
+        var query = new KnnSearchBuilder(VECTOR_FIELD, vector, 1, 1, null, null);
+        assertResponse(client().prepareSearch(INDEX_NAME).setKnnSearch(List.of(query)).setSize(1).setProfile(true), response -> {
+            assertNotEquals(0, response.getHits().getHits().length);
+            var profileResults = response.getProfileResults();
+            long vectorOpsSum = profileResults.values()
+                .stream()
+                .mapToLong(
+                    pr -> pr.getQueryPhase()
+                        .getSearchProfileDfsPhaseResult()
+                        .getQueryProfileShardResult()
+                        .stream()
+                        .mapToLong(qpr -> qpr.getVectorOperationsCount().longValue())
+                        .sum()
+                )
+                .sum();
+            client().admin()
+                .indices()
+                .prepareUpdateSettings(INDEX_NAME)
+                .setSettings(Settings.builder().put(DenseVectorFieldMapper.HNSW_EARLY_TERMINATION.getKey(), true))
+                .get();
+            assertResponse(
+                client().prepareSearch(INDEX_NAME).setKnnSearch(List.of(query)).setSize(1).setProfile(true),
+                earlyTerminationResponse -> {
+                    assertNotEquals(0, earlyTerminationResponse.getHits().getHits().length);
+                    var earlyTerminationResults = earlyTerminationResponse.getProfileResults();
+                    long earlyTerminationVectorOpsSum = earlyTerminationResults.values()
+                        .stream()
+                        .mapToLong(
+                            pr -> pr.getQueryPhase()
+                                .getSearchProfileDfsPhaseResult()
+                                .getQueryProfileShardResult()
+                                .stream()
+                                .mapToLong(qpr -> qpr.getVectorOperationsCount().longValue())
+                                .sum()
+                        )
+                        .sum();
+                    assertTrue(
+                        "earlyTerminationVectorOps [" + earlyTerminationVectorOpsSum + "] is not lt vectorOps [" + vectorOpsSum + "]",
+                        earlyTerminationVectorOpsSum < vectorOpsSum
+                            // if both switch to brute-force due to excessive exploration, they will both equal to upperLimit
+                            || (earlyTerminationVectorOpsSum == vectorOpsSum && vectorOpsSum == upperLimit + 1)
+                    );
+                }
+            );
+        });
+    }
+
 }

server/src/internalClusterTest/java/org/elasticsearch/search/query/VectorIT.java

@@ -159,6 +159,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING,
                 IndexSettings.INDEX_SEARCH_IDLE_AFTER,
                 DenseVectorFieldMapper.HNSW_FILTER_HEURISTIC,
+                DenseVectorFieldMapper.HNSW_EARLY_TERMINATION,
                 IndexFieldDataService.INDEX_FIELDDATA_CACHE_KEY,
                 IndexSettings.IGNORE_ABOVE_SETTING,
                 FieldMapper.IGNORE_MALFORMED_SETTING,

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -916,6 +916,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
     private volatile int maxNgramDiff;
     private volatile int maxShingleDiff;
     private volatile DenseVectorFieldMapper.FilterHeuristic hnswFilterHeuristic;
+    private volatile boolean earlyTermination;
     private volatile TimeValue searchIdleAfter;
     private volatile int maxAnalyzedOffset;
     private volatile boolean weightMatchesEnabled;
@@ -1113,6 +1114,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING);
         skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING);
         hnswFilterHeuristic = scopedSettings.get(DenseVectorFieldMapper.HNSW_FILTER_HEURISTIC);
+        earlyTermination = scopedSettings.get(DenseVectorFieldMapper.HNSW_EARLY_TERMINATION);
         indexMappingSourceMode = scopedSettings.get(INDEX_MAPPER_SOURCE_MODE_SETTING);
         recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings);
         recoverySourceSyntheticEnabled = DiscoveryNode.isStateless(nodeSettings) == false
@@ -1227,6 +1229,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         );
         scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead);
         scopedSettings.addSettingsUpdateConsumer(DenseVectorFieldMapper.HNSW_FILTER_HEURISTIC, this::setHnswFilterHeuristic);
+        scopedSettings.addSettingsUpdateConsumer(DenseVectorFieldMapper.HNSW_EARLY_TERMINATION, this::setHnswEarlyTermination);
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) {
@@ -1858,6 +1861,14 @@ private void setHnswFilterHeuristic(DenseVectorFieldMapper.FilterHeuristic heuri
         this.hnswFilterHeuristic = heuristic;
     }
 
+    public boolean getHnswEarlyTermination() {
+        return this.earlyTermination;
+    }
+
+    private void setHnswEarlyTermination(boolean earlyTermination) {
+        this.earlyTermination = earlyTermination;
+    }
+
     public SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions() {
         return seqNoIndexOptions;
     }

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -178,7 +178,6 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion UPGRADE_TO_LUCENE_10_2_2 = def(9_030_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT = def(9_031_0_00, Version.LUCENE_10_2_2);
     public static final IndexVersion DEFAULT_DENSE_VECTOR_TO_BBQ_HNSW = def(9_032_0_00, Version.LUCENE_10_2_2);
-    public static final IndexVersion EXPOSE_EARLY_TERMINATION = def(9_033_0_00, Version.LUCENE_10_2_2);
 
     /*
      * STOP! READ THIS FIRST! No, really,