Allow zero for rescore_vector.oversample to indicate by-passing oversample and rescoring

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml

@@ -337,3 +337,58 @@ setup:
   - match: { hits.hits.0._score: $rescore_score0 }
   - match: { hits.hits.1._score: $rescore_score1 }
   - match: { hits.hits.2._score: $rescore_score2 }
+---
+"Test index configured rescore vector updateable and settable to 0":
+  - requires:
+      cluster_features: ["mapper.dense_vector.rescore_zero_vector"]
+      reason: Needs rescore_zero_vector feature
+
+  - do:
+      indices.create:
+        index: bbq_rescore_0_hnsw
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                index_options:
+                  type: bbq_hnsw
+                  rescore_vector:
+                    oversample: 0
+
+  - do:
+      indices.create:
+        index: bbq_rescore_update_hnsw
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                index_options:
+                  type: bbq_hnsw
+                  rescore_vector:
+                    oversample: 1
+
+  - do:
+      indices.put_mapping:
+        index: bbq_rescore_update_hnsw
+        body:
+          properties:
+            vector:
+              type: dense_vector
+              index_options:
+                type: bbq_hnsw
+                rescore_vector:
+                  oversample: 0
+
+  - do:
+      indices.get_mapping:
+        index: bbq_rescore_update_hnsw
+
+  - match: { .bbq_rescore_update_hnsw.mappings.properties.vector.index_options.rescore_vector.oversample: 0 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml

@@ -700,3 +700,58 @@ setup:
   - match: { hits.hits.0._score: $rescore_score0 }
   - match: { hits.hits.1._score: $rescore_score1 }
   - match: { hits.hits.2._score: $rescore_score2 }
+---
+"Test index configured rescore vector updateable and settable to 0":
+  - requires:
+      cluster_features: ["mapper.dense_vector.rescore_zero_vector"]
+      reason: Needs rescore_zero_vector feature
+
+  - do:
+      indices.create:
+        index: int8_rescore_0_hnsw
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                index_options:
+                  type: int8_hnsw
+                  rescore_vector:
+                    oversample: 0
+
+  - do:
+      indices.create:
+        index: int8_rescore_update_hnsw
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                index_options:
+                  type: int8_hnsw
+                  rescore_vector:
+                    oversample: 1
+
+  - do:
+      indices.put_mapping:
+        index: int8_rescore_update_hnsw
+        body:
+          properties:
+            vector:
+              type: dense_vector
+              index_options:
+                type: int8_hnsw
+                rescore_vector:
+                  oversample: 0
+
+  - do:
+      indices.get_mapping:
+        index: int8_rescore_update_hnsw
+
+  - match: { .int8_rescore_update_hnsw.mappings.properties.vector.index_options.rescore_vector.oversample: 0 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml

@@ -731,3 +731,58 @@ setup:
   - match: { hits.hits.0._score: $rescore_score0 }
   - match: { hits.hits.1._score: $rescore_score1 }
   - match: { hits.hits.2._score: $rescore_score2 }
+---
+"Test index configured rescore vector updateable and settable to 0":
+  - requires:
+      cluster_features: ["mapper.dense_vector.rescore_zero_vector"]
+      reason: Needs rescore_zero_vector feature
+
+  - do:
+      indices.create:
+        index: int4_rescore_0_hnsw
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                index_options:
+                  type: int4_hnsw
+                  rescore_vector:
+                    oversample: 0
+
+  - do:
+      indices.create:
+        index: int4_rescore_update_hnsw
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                index_options:
+                  type: int4_hnsw
+                  rescore_vector:
+                    oversample: 1
+
+  - do:
+      indices.put_mapping:
+        index: int4_rescore_update_hnsw
+        body:
+          properties:
+            vector:
+              type: dense_vector
+              index_options:
+                type: int4_hnsw
+                rescore_vector:
+                  oversample: 0
+
+  - do:
+      indices.get_mapping:
+        index: int4_rescore_update_hnsw
+
+  - match: { .int4_rescore_update_hnsw.mappings.properties.vector.index_options.rescore_vector.oversample: 0 }

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -15,6 +15,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.RESCORE_VECTOR_QUANTIZED_VECTOR_MAPPING;
+import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.RESCORE_ZERO_VECTOR_QUANTIZED_VECTOR_MAPPING;
 
 /**
  * Spec for mapper-related features.
@@ -61,7 +62,8 @@ public Set<NodeFeature> getTestFeatures() {
             ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX,
             UKNOWN_FIELD_MAPPING_UPDATE_ERROR_MESSAGE,
             DOC_VALUES_SKIPPER,
-            RESCORE_VECTOR_QUANTIZED_VECTOR_MAPPING
+            RESCORE_VECTOR_QUANTIZED_VECTOR_MAPPING,
+            RESCORE_ZERO_VECTOR_QUANTIZED_VECTOR_MAPPING
         );
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -119,6 +119,9 @@ public static boolean isNotUnitVector(float magnitude) {
         IndexVersions.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS;
 
     public static final NodeFeature RESCORE_VECTOR_QUANTIZED_VECTOR_MAPPING = new NodeFeature("mapper.dense_vector.rescore_vector");
+    public static final NodeFeature RESCORE_ZERO_VECTOR_QUANTIZED_VECTOR_MAPPING = new NodeFeature(
+        "mapper.dense_vector.rescore_zero_vector"
+    );
 
     public static final String CONTENT_TYPE = "dense_vector";
     public static final short MAX_DIMS_COUNT = 4096; // maximum allowed number of dimensions
@@ -2007,10 +2010,9 @@ static RescoreVector fromIndexOptions(Map<String, ?> indexOptionsMap, IndexVersi
             if (oversampleValue == 0 && indexVersion.before(RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS)) {
                 throw new IllegalArgumentException("oversample must be greater than 1");
             }
-            if (oversampleValue < 1) {
-                throw new IllegalArgumentException("oversample must be greater than 1");
-            }
-            if (oversampleValue > 10) {
+            if (oversampleValue < 1 && oversampleValue != 0) {
+                throw new IllegalArgumentException("oversample must be greater than 1 or exactly 0");
+            } else if (oversampleValue > 10) {
                 throw new IllegalArgumentException("oversample must be less than or equal to 10");
             }
             return new RescoreVector(oversampleValue);

server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java

@@ -903,7 +903,7 @@ public void testRescoreVectorForNonQuantized() {
         }
     }
 
-    public void tesetRescoreVectorOldIndexVersion() {
+    public void testRescoreVectorOldIndexVersion() {
         IndexVersion incompatibleVersion = IndexVersionUtils.randomVersionBetween(
             random(),
             IndexVersionUtils.getLowestReadCompatibleVersion(),
@@ -927,6 +927,30 @@ public void tesetRescoreVectorOldIndexVersion() {
         }
     }
 
+    public void testRescoreZeroVectorOldIndexVersion() {
+        IndexVersion incompatibleVersion = IndexVersionUtils.randomVersionBetween(
+            random(),
+            IndexVersionUtils.getLowestReadCompatibleVersion(),
+            IndexVersionUtils.getPreviousVersion(DenseVectorFieldMapper.RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS)
+        );
+        for (String indexType : List.of("int8_hnsw", "int8_flat", "int4_hnsw", "int4_flat", "bbq_hnsw", "bbq_flat")) {
+            expectThrows(
+                MapperParsingException.class,
+                () -> createDocumentMapper(
+                    incompatibleVersion,
+                    fieldMapping(
+                        b -> b.field("type", "dense_vector")
+                            .field("index", true)
+                            .startObject("index_options")
+                            .field("type", indexType)
+                            .field(DenseVectorFieldMapper.RescoreVector.NAME, Map.of("oversample", 0f))
+                            .endObject()
+                    )
+                )
+            );
+        }
+    }
+
     public void testInvalidRescoreVector() {
         for (String indexType : List.of("int8_hnsw", "int8_flat", "int4_hnsw", "int4_flat", "bbq_hnsw", "bbq_flat")) {
             Exception e = expectThrows(

server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java

@@ -50,7 +50,7 @@ public DenseVectorFieldTypeTests() {
     }
 
     private static DenseVectorFieldMapper.RescoreVector randomRescoreVector() {
-        return new DenseVectorFieldMapper.RescoreVector(randomFloatBetween(1.0F, 10.0F, false));
+        return new DenseVectorFieldMapper.RescoreVector(randomBoolean() ? 0 : randomFloatBetween(1.0F, 10.0F, false));
     }
 
     private DenseVectorFieldMapper.IndexOptions randomIndexOptionsNonQuantized() {
@@ -94,24 +94,24 @@ private DenseVectorFieldMapper.IndexOptions randomIndexOptionsAll() {
     }
 
     private DenseVectorFieldMapper.IndexOptions randomIndexOptionsHnswQuantized() {
+        return randomIndexOptionsHnswQuantized(randomBoolean() ? null : randomRescoreVector());
+    }
+
+    private DenseVectorFieldMapper.IndexOptions randomIndexOptionsHnswQuantized(DenseVectorFieldMapper.RescoreVector rescoreVector) {
         return randomFrom(
             new DenseVectorFieldMapper.Int8HnswIndexOptions(
                 randomIntBetween(1, 100),
                 randomIntBetween(1, 10_000),
                 randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)),
-                randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
+                rescoreVector
             ),
             new DenseVectorFieldMapper.Int4HnswIndexOptions(
                 randomIntBetween(1, 100),
                 randomIntBetween(1, 10_000),
                 randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)),
-                randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
+                rescoreVector
             ),
-            new DenseVectorFieldMapper.BBQHnswIndexOptions(
-                randomIntBetween(1, 100),
-                randomIntBetween(1, 10_000),
-                randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
-            )
+            new DenseVectorFieldMapper.BBQHnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000), rescoreVector)
         );
     }
 
@@ -492,20 +492,34 @@ public void testRescoreOversampleModifiesNumCandidates() {
         checkRescoreQueryParameters(fieldType, 1000, 1000, 11.0F, OVERSAMPLE_LIMIT, OVERSAMPLE_LIMIT, 1000);
     }
 
-    public void testRescoreOversampleZeroBypassesRescore() {
+    public void testRescoreOversampleQueryOverrides() {
+        // verify we can override to `0`
         DenseVectorFieldType fieldType = new DenseVectorFieldType(
             "f",
             IndexVersion.current(),
             FLOAT,
             3,
             true,
             VectorSimilarity.COSINE,
-            randomIndexOptionsHnswQuantized(),
+            randomIndexOptionsHnswQuantized(new DenseVectorFieldMapper.RescoreVector(randomFloatBetween(1.1f, 9.9f, false))),
             Collections.emptyMap()
         );
-
         Query query = fieldType.createKnnQuery(VectorData.fromFloats(new float[] { 1, 4, 10 }), 10, 100, 0f, null, null, null);
         assertTrue(query instanceof ESKnnFloatVectorQuery);
+
+        // verify we can override a `0` to a positive number
+        fieldType = new DenseVectorFieldType(
+            "f",
+            IndexVersion.current(),
+            FLOAT,
+            3,
+            true,
+            VectorSimilarity.COSINE,
+            randomIndexOptionsHnswQuantized(new DenseVectorFieldMapper.RescoreVector(0)),
+            Collections.emptyMap()
+        );
+        query = fieldType.createKnnQuery(VectorData.fromFloats(new float[] { 1, 4, 10 }), 10, 100, 2f, null, null, null);
+        assertTrue(query instanceof RescoreKnnVectorQuery);
     }
 
     private static void checkRescoreQueryParameters(

server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java

@@ -166,7 +166,7 @@ protected RescoreVectorBuilder randomRescoreVectorBuilder() {
             return null;
         }
 
-        return new RescoreVectorBuilder(randomFloatBetween(1.0f, 10.0f, false));
+        return new RescoreVectorBuilder(randomBoolean() ? 0f : randomFloatBetween(1.0f, 10.0f, false));
     }
 
     @Override
@@ -181,9 +181,13 @@ protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query que
             k = context.requestSize() == null || context.requestSize() < 0 ? DEFAULT_SIZE : context.requestSize();
         }
         if (queryBuilder.rescoreVectorBuilder() != null && isQuantizedElementType()) {
-            RescoreKnnVectorQuery rescoreQuery = (RescoreKnnVectorQuery) query;
-            assertEquals(k.intValue(), (rescoreQuery.k()));
-            query = rescoreQuery.innerQuery();
+            if (queryBuilder.rescoreVectorBuilder().oversample() > 0) {
+                RescoreKnnVectorQuery rescoreQuery = (RescoreKnnVectorQuery) query;
+                assertEquals(k.intValue(), (rescoreQuery.k()));
+                query = rescoreQuery.innerQuery();
+            } else {
+                assertFalse(query instanceof RescoreKnnVectorQuery);
+            }
         }
         switch (elementType()) {
             case FLOAT -> assertTrue(query instanceof ESKnnFloatVectorQuery);