[8.19] New vector_rescore parameter as a quantized index type option (#124581) (#127644)

* New `vector_rescore` parameter as a quantized index type option (#124581)

This adds a new parameter to the quantized index mapping that allows
default oversampling and rescoring to occur.

This doesn't adjust any of the defaults. It allows it to be configured.
When the user provides `rescore_vector: {oversample: <number>}` in the
query it will overwrite it.

For example, here is how to use it with bbq:

```
PUT rescored_bbq
{
  "mappings": {
    "properties": {
      "vector": {
        "type": "dense_vector",
        "index_options": {
          "type": "bbq_hnsw",
          "rescore_vector": {"oversample": 3.0}
        }
      }
    }
  }
}
```

Then, when querying, it will auto oversample the `k` by `3x` and rerank
with the raw vectors.

```
POST _search
{
  "knn": {
    "query_vector": [...],
    "field": "vector"
  }
}
```

(cherry picked from commit b2c1c4e)

* Adds new BWC version for 8.19 backport of (#124581) (#127647)

* [CI] Auto commit changes from spotless

---------

Co-authored-by: elasticsearchmachine <[email protected]>

Author: benwtrent

docs/changelog/124581.yaml

@@ -0,0 +1,5 @@
+pr: 124581
+summary: New `vector_rescore` parameter as a quantized index type option
+area: Vector Search
+type: enhancement
+issues: []

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml

@@ -250,3 +250,93 @@ setup:
         index: dynamic_dim_bbq_hnsw
         body:
           vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0]
+---
+"Test index configured rescore vector":
+  - requires:
+      cluster_features: ["mapper.dense_vector.rescore_vector"]
+      reason: Needs rescore_vector feature
+  - skip:
+      features: "headers"
+  - do:
+      indices.create:
+        index: bbq_rescore_hnsw
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                dims: 64
+                index: true
+                similarity: max_inner_product
+                index_options:
+                  type: bbq_hnsw
+                  rescore_vector:
+                    oversample: 1.5
+
+  - do:
+      bulk:
+        index: bbq_rescore_hnsw
+        refresh: true
+        body: |
+          { "index": {"_id": "1"}}
+          { "vector": [0.077,  0.32 , -0.205,  0.63 ,  0.032,  0.201,  0.167, -0.313, 0.176,  0.531, -0.375,  0.334, -0.046,  0.078, -0.349,  0.272, 0.307, -0.083,  0.504,  0.255, -0.404,  0.289, -0.226, -0.132, -0.216,  0.49 ,  0.039,  0.507, -0.307,  0.107,  0.09 , -0.265, -0.285,  0.336, -0.272,  0.369, -0.282,  0.086, -0.132,  0.475, -0.224,  0.203,  0.439,  0.064,  0.246, -0.396,  0.297,  0.242, -0.028,  0.321, -0.022, -0.009, -0.001  ,  0.031, -0.533,  0.45, -0.683,  1.331,  0.194, -0.157, -0.1  , -0.279, -0.098, -0.176] }
+          { "index": {"_id": "2"}}
+          { "vector": [0.196,  0.514,  0.039,  0.555, -0.042,  0.242,  0.463, -0.348, -0.08 ,  0.442, -0.067, -0.05 , -0.001,  0.298, -0.377,  0.048, 0.307,  0.159,  0.278,  0.119, -0.057,  0.333, -0.289, -0.438, -0.014,  0.361, -0.169,  0.292, -0.229,  0.123,  0.031, -0.138, -0.139,  0.315, -0.216,  0.322, -0.445, -0.059,  0.071,  0.429, -0.602, -0.142,  0.11 ,  0.192,  0.259, -0.241,  0.181, -0.166, 0.082,  0.107, -0.05 ,  0.155,  0.011,  0.161, -0.486,  0.569, -0.489,  0.901,  0.208,  0.011, -0.209, -0.153, -0.27 , -0.013] }
+          { "index": {"_id": "3"}}
+          { "vector": [0.196,  0.514,  0.039,  0.555, -0.042,  0.242,  0.463, -0.348, -0.08 ,  0.442, -0.067, -0.05 , -0.001,  0.298, -0.377,  0.048, 0.307,  0.159,  0.278,  0.119, -0.057,  0.333, -0.289, -0.438, -0.014,  0.361, -0.169,  0.292, -0.229,  0.123,  0.031, -0.138, -0.139,  0.315, -0.216,  0.322, -0.445, -0.059,  0.071,  0.429, -0.602, -0.142,  0.11 ,  0.192,  0.259, -0.241,  0.181, -0.166, 0.082,  0.107, -0.05 ,  0.155,  0.011,  0.161, -0.486,  0.569, -0.489,  0.901,  0.208,  0.011, -0.209, -0.153, -0.27 , -0.013] }
+
+  - do:
+      headers:
+        Content-Type: application/json
+      search:
+        rest_total_hits_as_int: true
+        index: bbq_rescore_hnsw
+        body:
+          knn:
+            field: vector
+            query_vector: [0.128,  0.067, -0.08 ,  0.395, -0.11 , -0.259,  0.473, -0.393,
+                           0.292,  0.571, -0.491,  0.444, -0.288,  0.198, -0.343,  0.015,
+                           0.232,  0.088,  0.228,  0.151, -0.136,  0.236, -0.273, -0.259,
+                           -0.217,  0.359, -0.207,  0.352, -0.142,  0.192, -0.061, -0.17 ,
+                           -0.343,  0.189, -0.221,  0.32 , -0.301, -0.1  ,  0.005,  0.232,
+                           -0.344,  0.136,  0.252,  0.157, -0.13 , -0.244,  0.193, -0.034,
+                           -0.12 , -0.193, -0.102,  0.252, -0.185, -0.167, -0.575,  0.582,
+                           -0.426,  0.983,  0.212,  0.204,  0.03 , -0.276, -0.425, -0.158]
+            k: 3
+            num_candidates: 3
+
+  - match: { hits.total: 3 }
+  - set: { hits.hits.0._score: rescore_score0 }
+  - set: { hits.hits.1._score: rescore_score1 }
+  - set: { hits.hits.2._score: rescore_score2 }
+
+  - do:
+      headers:
+        Content-Type: application/json
+      search:
+        rest_total_hits_as_int: true
+        index: bbq_rescore_hnsw
+        body:
+          query:
+            script_score:
+              query: {match_all: {} }
+              script:
+                source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
+                params:
+                  query_vector: [0.128,  0.067, -0.08 ,  0.395, -0.11 , -0.259,  0.473, -0.393,
+                                 0.292,  0.571, -0.491,  0.444, -0.288,  0.198, -0.343,  0.015,
+                                 0.232,  0.088,  0.228,  0.151, -0.136,  0.236, -0.273, -0.259,
+                                 -0.217,  0.359, -0.207,  0.352, -0.142,  0.192, -0.061, -0.17 ,
+                                 -0.343,  0.189, -0.221,  0.32 , -0.301, -0.1  ,  0.005,  0.232,
+                                 -0.344,  0.136,  0.252,  0.157, -0.13 , -0.244,  0.193, -0.034,
+                                 -0.12 , -0.193, -0.102,  0.252, -0.185, -0.167, -0.575,  0.582,
+                                 -0.426,  0.983,  0.212,  0.204,  0.03 , -0.276, -0.425, -0.158]
+
+  # Compare scores as hit IDs may change depending on how things are distributed
+  - match: { hits.total: 3 }
+  - match: { hits.hits.0._score: $rescore_score0 }
+  - match: { hits.hits.1._score: $rescore_score1 }
+  - match: { hits.hits.2._score: $rescore_score2 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml

@@ -611,3 +611,92 @@ setup:
   - match: { hits.hits.0._id: "1"}
   - match: { hits.hits.1._id: "2"}
   - match: { hits.hits.2._id: "3"}
+---
+"Test index configured rescore vector":
+  - requires:
+      cluster_features: ["mapper.dense_vector.rescore_vector"]
+      reason: Needs rescore_vector feature
+  - skip:
+      features: "headers"
+  - do:
+      indices.create:
+        index: int8_rescore_hnsw
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                dims: 64
+                index: true
+                similarity: max_inner_product
+                index_options:
+                  type: int8_hnsw
+                  rescore_vector:
+                    oversample: 1.5
+
+  - do:
+      bulk:
+        index: int8_rescore_hnsw
+        refresh: true
+        body: |
+          { "index": {"_id": "1"}}
+          { "vector": [0.077,  0.32 , -0.205,  0.63 ,  0.032,  0.201,  0.167, -0.313, 0.176,  0.531, -0.375,  0.334, -0.046,  0.078, -0.349,  0.272, 0.307, -0.083,  0.504,  0.255, -0.404,  0.289, -0.226, -0.132, -0.216,  0.49 ,  0.039,  0.507, -0.307,  0.107,  0.09 , -0.265, -0.285,  0.336, -0.272,  0.369, -0.282,  0.086, -0.132,  0.475, -0.224,  0.203,  0.439,  0.064,  0.246, -0.396,  0.297,  0.242, -0.028,  0.321, -0.022, -0.009, -0.001  ,  0.031, -0.533,  0.45, -0.683,  1.331,  0.194, -0.157, -0.1  , -0.279, -0.098, -0.176] }
+          { "index": {"_id": "2"}}
+          { "vector": [0.196,  0.514,  0.039,  0.555, -0.042,  0.242,  0.463, -0.348, -0.08 ,  0.442, -0.067, -0.05 , -0.001,  0.298, -0.377,  0.048, 0.307,  0.159,  0.278,  0.119, -0.057,  0.333, -0.289, -0.438, -0.014,  0.361, -0.169,  0.292, -0.229,  0.123,  0.031, -0.138, -0.139,  0.315, -0.216,  0.322, -0.445, -0.059,  0.071,  0.429, -0.602, -0.142,  0.11 ,  0.192,  0.259, -0.241,  0.181, -0.166, 0.082,  0.107, -0.05 ,  0.155,  0.011,  0.161, -0.486,  0.569, -0.489,  0.901,  0.208,  0.011, -0.209, -0.153, -0.27 , -0.013] }
+          { "index": {"_id": "3"}}
+          { "vector": [0.196,  0.514,  0.039,  0.555, -0.042,  0.242,  0.463, -0.348, -0.08 ,  0.442, -0.067, -0.05 , -0.001,  0.298, -0.377,  0.048, 0.307,  0.159,  0.278,  0.119, -0.057,  0.333, -0.289, -0.438, -0.014,  0.361, -0.169,  0.292, -0.229,  0.123,  0.031, -0.138, -0.139,  0.315, -0.216,  0.322, -0.445, -0.059,  0.071,  0.429, -0.602, -0.142,  0.11 ,  0.192,  0.259, -0.241,  0.181, -0.166, 0.082,  0.107, -0.05 ,  0.155,  0.011,  0.161, -0.486,  0.569, -0.489,  0.901,  0.208,  0.011, -0.209, -0.153, -0.27 , -0.013] }
+
+  - do:
+      headers:
+        Content-Type: application/json
+      search:
+        rest_total_hits_as_int: true
+        index: int8_rescore_hnsw
+        body:
+          knn:
+            field: vector
+            query_vector: [0.128,  0.067, -0.08 ,  0.395, -0.11 , -0.259,  0.473, -0.393,
+                           0.292,  0.571, -0.491,  0.444, -0.288,  0.198, -0.343,  0.015,
+                           0.232,  0.088,  0.228,  0.151, -0.136,  0.236, -0.273, -0.259,
+                           -0.217,  0.359, -0.207,  0.352, -0.142,  0.192, -0.061, -0.17 ,
+                           -0.343,  0.189, -0.221,  0.32 , -0.301, -0.1  ,  0.005,  0.232,
+                           -0.344,  0.136,  0.252,  0.157, -0.13 , -0.244,  0.193, -0.034,
+                           -0.12 , -0.193, -0.102,  0.252, -0.185, -0.167, -0.575,  0.582,
+                           -0.426,  0.983,  0.212,  0.204,  0.03 , -0.276, -0.425, -0.158]
+            k: 3
+            num_candidates: 3
+
+  - match: { hits.total: 3 }
+  - set: { hits.hits.0._score: rescore_score0 }
+  - set: { hits.hits.1._score: rescore_score1 }
+  - set: { hits.hits.2._score: rescore_score2 }
+
+  - do:
+      headers:
+        Content-Type: application/json
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query:
+            script_score:
+              query: {match_all: {} }
+              script:
+                source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
+                params:
+                  query_vector: [0.128,  0.067, -0.08 ,  0.395, -0.11 , -0.259,  0.473, -0.393,
+                                 0.292,  0.571, -0.491,  0.444, -0.288,  0.198, -0.343,  0.015,
+                                 0.232,  0.088,  0.228,  0.151, -0.136,  0.236, -0.273, -0.259,
+                                 -0.217,  0.359, -0.207,  0.352, -0.142,  0.192, -0.061, -0.17 ,
+                                 -0.343,  0.189, -0.221,  0.32 , -0.301, -0.1  ,  0.005,  0.232,
+                                 -0.344,  0.136,  0.252,  0.157, -0.13 , -0.244,  0.193, -0.034,
+                                 -0.12 , -0.193, -0.102,  0.252, -0.185, -0.167, -0.575,  0.582,
+                                 -0.426,  0.983,  0.212,  0.204,  0.03 , -0.276, -0.425, -0.158]
+
+  # Compare scores as hit IDs may change depending on how things are distributed
+  - match: { hits.total: 3 }
+  - match: { hits.hits.0._score: $rescore_score0 }
+  - match: { hits.hits.1._score: $rescore_score1 }
+  - match: { hits.hits.2._score: $rescore_score2 }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml

@@ -645,3 +645,92 @@ setup:
         index: dynamic_dim_hnsw_quantized
         body:
           vector: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]
+---
+"Test index configured rescore vector":
+  - requires:
+      cluster_features: ["mapper.dense_vector.rescore_vector"]
+      reason: Needs rescore_vector feature
+  - skip:
+      features: "headers"
+  - do:
+      indices.create:
+        index: int4_rescore_hnsw
+        body:
+          settings:
+            index:
+              number_of_shards: 1
+          mappings:
+            properties:
+              vector:
+                type: dense_vector
+                dims: 64
+                index: true
+                similarity: max_inner_product
+                index_options:
+                  type: int4_hnsw
+                  rescore_vector:
+                    oversample: 1.5
+
+  - do:
+      bulk:
+        index: int4_rescore_hnsw
+        refresh: true
+        body: |
+          { "index": {"_id": "1"}}
+          { "vector": [0.077,  0.32 , -0.205,  0.63 ,  0.032,  0.201,  0.167, -0.313, 0.176,  0.531, -0.375,  0.334, -0.046,  0.078, -0.349,  0.272, 0.307, -0.083,  0.504,  0.255, -0.404,  0.289, -0.226, -0.132, -0.216,  0.49 ,  0.039,  0.507, -0.307,  0.107,  0.09 , -0.265, -0.285,  0.336, -0.272,  0.369, -0.282,  0.086, -0.132,  0.475, -0.224,  0.203,  0.439,  0.064,  0.246, -0.396,  0.297,  0.242, -0.028,  0.321, -0.022, -0.009, -0.001  ,  0.031, -0.533,  0.45, -0.683,  1.331,  0.194, -0.157, -0.1  , -0.279, -0.098, -0.176] }
+          { "index": {"_id": "2"}}
+          { "vector": [0.196,  0.514,  0.039,  0.555, -0.042,  0.242,  0.463, -0.348, -0.08 ,  0.442, -0.067, -0.05 , -0.001,  0.298, -0.377,  0.048, 0.307,  0.159,  0.278,  0.119, -0.057,  0.333, -0.289, -0.438, -0.014,  0.361, -0.169,  0.292, -0.229,  0.123,  0.031, -0.138, -0.139,  0.315, -0.216,  0.322, -0.445, -0.059,  0.071,  0.429, -0.602, -0.142,  0.11 ,  0.192,  0.259, -0.241,  0.181, -0.166, 0.082,  0.107, -0.05 ,  0.155,  0.011,  0.161, -0.486,  0.569, -0.489,  0.901,  0.208,  0.011, -0.209, -0.153, -0.27 , -0.013] }
+          { "index": {"_id": "3"}}
+          { "vector": [0.196,  0.514,  0.039,  0.555, -0.042,  0.242,  0.463, -0.348, -0.08 ,  0.442, -0.067, -0.05 , -0.001,  0.298, -0.377,  0.048, 0.307,  0.159,  0.278,  0.119, -0.057,  0.333, -0.289, -0.438, -0.014,  0.361, -0.169,  0.292, -0.229,  0.123,  0.031, -0.138, -0.139,  0.315, -0.216,  0.322, -0.445, -0.059,  0.071,  0.429, -0.602, -0.142,  0.11 ,  0.192,  0.259, -0.241,  0.181, -0.166, 0.082,  0.107, -0.05 ,  0.155,  0.011,  0.161, -0.486,  0.569, -0.489,  0.901,  0.208,  0.011, -0.209, -0.153, -0.27 , -0.013] }
+
+  - do:
+      headers:
+        Content-Type: application/json
+      search:
+        rest_total_hits_as_int: true
+        index: int4_rescore_hnsw
+        body:
+          knn:
+            field: vector
+            query_vector: [0.128,  0.067, -0.08 ,  0.395, -0.11 , -0.259,  0.473, -0.393,
+                           0.292,  0.571, -0.491,  0.444, -0.288,  0.198, -0.343,  0.015,
+                           0.232,  0.088,  0.228,  0.151, -0.136,  0.236, -0.273, -0.259,
+                           -0.217,  0.359, -0.207,  0.352, -0.142,  0.192, -0.061, -0.17 ,
+                           -0.343,  0.189, -0.221,  0.32 , -0.301, -0.1  ,  0.005,  0.232,
+                           -0.344,  0.136,  0.252,  0.157, -0.13 , -0.244,  0.193, -0.034,
+                           -0.12 , -0.193, -0.102,  0.252, -0.185, -0.167, -0.575,  0.582,
+                           -0.426,  0.983,  0.212,  0.204,  0.03 , -0.276, -0.425, -0.158]
+            k: 3
+            num_candidates: 3
+
+  - match: { hits.total: 3 }
+  - set: { hits.hits.0._score: rescore_score0 }
+  - set: { hits.hits.1._score: rescore_score1 }
+  - set: { hits.hits.2._score: rescore_score2 }
+
+  - do:
+      headers:
+        Content-Type: application/json
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query:
+            script_score:
+              query: {match_all: {} }
+              script:
+                source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
+                params:
+                  query_vector: [0.128,  0.067, -0.08 ,  0.395, -0.11 , -0.259,  0.473, -0.393,
+                                 0.292,  0.571, -0.491,  0.444, -0.288,  0.198, -0.343,  0.015,
+                                 0.232,  0.088,  0.228,  0.151, -0.136,  0.236, -0.273, -0.259,
+                                 -0.217,  0.359, -0.207,  0.352, -0.142,  0.192, -0.061, -0.17 ,
+                                 -0.343,  0.189, -0.221,  0.32 , -0.301, -0.1  ,  0.005,  0.232,
+                                 -0.344,  0.136,  0.252,  0.157, -0.13 , -0.244,  0.193, -0.034,
+                                 -0.12 , -0.193, -0.102,  0.252, -0.185, -0.167, -0.575,  0.582,
+                                 -0.426,  0.983,  0.212,  0.204,  0.03 , -0.276, -0.425, -0.158]
+
+  # Compare scores as hit IDs may change depending on how things are distributed
+  - match: { hits.total: 3 }
+  - match: { hits.hits.0._score: $rescore_score0 }
+  - match: { hits.hits.1._score: $rescore_score1 }
+  - match: { hits.hits.2._score: $rescore_score2 }