From d260dd6310c69d24446767805e4ab619d2e2b59a Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 16 Jun 2025 17:15:59 +0200 Subject: [PATCH 1/3] Make dense_vector fields updatable to bbq_flat/bbq_hnsw (#128291) (cherry picked from commit 629a366baa18ab81d3bf132f31370268207aa27c) --- docs/changelog/128291.yaml | 5 + .../mapping/types/dense-vector.asciidoc | 4 +- .../upgrades/DenseVectorMappingUpdateIT.java | 25 +- .../180_update_dense_vector_type.yml | 391 ++++++++++++++++ .../vectors/DenseVectorFieldMapper.java | 24 +- .../action/search/SearchCapabilities.java | 2 + .../vectors/DenseVectorFieldMapperTests.java | 437 ++++++++++++++++++ 7 files changed, 873 insertions(+), 15 deletions(-) create mode 100644 docs/changelog/128291.yaml diff --git a/docs/changelog/128291.yaml b/docs/changelog/128291.yaml new file mode 100644 index 0000000000000..097bd8a44a4c6 --- /dev/null +++ b/docs/changelog/128291.yaml @@ -0,0 +1,5 @@ +pr: 128291 +summary: Make `dense_vector` fields updatable to bbq_flat/bbq_hnsw +area: Vector Search +type: enhancement +issues: [] diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index b1c1aff245760..cf194b48adcdc 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -476,10 +476,10 @@ To better accommodate scaling and performance needs, updating the `type` setting [source,txt] ---- -flat --> int8_flat --> int4_flat --> hnsw --> int8_hnsw --> int4_hnsw +flat --> int8_flat --> int4_flat --> bbq_flat --> hnsw --> int8_hnsw --> int4_hnsw --> bbq_hnsw ---- -For updating all HNSW types (`hnsw`, `int8_hnsw`, `int4_hnsw`) the number of connections `m` must either stay the same or increase. For scalar quantized formats (`int8_flat`, `int4_flat`, `int8_hnsw`, `int4_hnsw`) the `confidence_interval` must always be consistent (once defined, it cannot change). +For updating all HNSW types (`hnsw`, `int8_hnsw`, `int4_hnsw`, `bbq_hnsw`) the number of connections `m` must either stay the same or increase. For the scalar quantized formats `int8_flat`, `int4_flat`, `int8_hnsw` and `int4_hnsw` the `confidence_interval` must always be consistent (once defined, it cannot change). Updating `type` in `index_options` will fail in all other scenarios. diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java index b8fc3b15636fb..0276e5a1558a0 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -31,6 +32,8 @@ */ public class DenseVectorMappingUpdateIT extends AbstractRollingUpgradeTestCase { + private static final String SYNTHETIC_SOURCE_FEATURE = "gte_v8.12.0"; + private static final String BULK1 = """ {"index": {"_id": "1"}} {"embedding": [1, 1, 1, 1]} @@ -86,10 +89,22 @@ public void testDenseVectorMappingUpdateOnOldCluster() throws IOException { String indexName = "test_index"; if (isOldCluster()) { Request createIndex = new Request("PUT", "/" + indexName); - XContentBuilder mappings = XContentBuilder.builder(XContentType.JSON.xContent()) - .startObject() - .startObject("mappings") - .startObject("properties") + boolean useSyntheticSource = randomBoolean() && oldClusterHasFeature(SYNTHETIC_SOURCE_FEATURE); + + boolean useIndexSetting = SourceFieldMapper.onOrAfterDeprecateModeVersion(getOldClusterIndexVersion()); + XContentBuilder payload = XContentBuilder.builder(XContentType.JSON.xContent()).startObject(); + if (useSyntheticSource) { + if (useIndexSetting) { + payload.startObject("settings").field("index.mapping.source.mode", "synthetic").endObject(); + } + } + payload.startObject("mappings"); + if (useIndexSetting == false) { + payload.startObject("_source"); + payload.field("mode", "synthetic"); + payload.endObject(); + } + payload.startObject("properties") .startObject("embedding") .field("type", "dense_vector") .field("index", "true") @@ -104,7 +119,7 @@ public void testDenseVectorMappingUpdateOnOldCluster() throws IOException { .endObject() .endObject() .endObject(); - createIndex.setJsonEntity(Strings.toString(mappings)); + createIndex.setJsonEntity(Strings.toString(payload)); client().performRequest(createIndex); Request index = new Request("POST", "/" + indexName + "/_bulk/"); index.addParameter("refresh", "true"); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml index c9afc7c9b3948..3bc2fff001077 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml @@ -2116,3 +2116,394 @@ setup: - contains: {hits.hits: {_id: "2"}} - contains: {hits.hits: {_id: "21"}} - contains: {hits.hits: {_id: "31"}} + +--- +"Test update flat --> bbq_flat --> bbq_hnsw": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ optimized_scalar_quantization_bbq ] + test_runner_features: capabilities + reason: "BBQ is required to test upgrading to bbq_flat and bbq_hnsw" + - requires: + reason: 'dense_vector updatable to bbq capability required' + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path : /_mapping + capabilities: [ dense_vector_updatable_bbq ] + - do: + indices.create: + index: vectors_64 + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: flat + + - do: + index: + index: vectors_64 + id: "1" + body: + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + - do: + indices.flush: + index: vectors_64 + - do: + indices.put_mapping: + index: vectors_64 + body: + properties: + embedding: + type: dense_vector + dims: 64 + index_options: + type: bbq_flat + + - do: + indices.get_mapping: + index: vectors_64 + + - match: { vectors_64.mappings.properties.embedding.type: dense_vector } + - match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_flat } + + - do: + index: + index: vectors_64 + id: "2" + body: + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] + - do: + indices.flush: + index: vectors_64 + - do: + indices.put_mapping: + index: vectors_64 + body: + properties: + embedding: + type: dense_vector + dims: 64 + index_options: + type: bbq_hnsw + + - do: + indices.get_mapping: + index: vectors_64 + + - match: { vectors_64.mappings.properties.embedding.type: dense_vector } + - match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_hnsw } + + - do: + index: + index: vectors_64 + id: "3" + body: + name: rabbit.jpg + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + - do: + indices.flush: + index: vectors_64 + + - do: + indices.forcemerge: + index: vectors_64 + max_num_segments: 1 + + - do: + indices.refresh: { } + - do: + search: + index: vectors_64 + body: + knn: + field: vector + query_vector: [ 0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158 ] + k: 3 + num_candidates: 3 + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } +--- +"Test update int8_hnsw --> bbq_flat": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ optimized_scalar_quantization_bbq ] + test_runner_features: capabilities + reason: "BBQ is required to test upgrading to bbq_flat and bbq_hnsw" + - requires: + reason: 'dense_vector updatable to bbq capability required' + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path : /_mapping + capabilities: [ dense_vector_updatable_bbq ] + - do: + indices.create: + index: vectors_64 + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: int8_hnsw + + - do: + index: + index: vectors_64 + id: "1" + body: + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + - do: + indices.flush: + index: vectors_64 + - do: + indices.put_mapping: + index: vectors_64 + body: + properties: + embedding: + type: dense_vector + dims: 64 + index_options: + type: bbq_flat + + - do: + indices.get_mapping: + index: vectors_64 + + - match: { vectors_64.mappings.properties.embedding.type: dense_vector } + - match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_flat } + + - do: + index: + index: vectors_64 + id: "2" + body: + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] + + - do: + index: + index: vectors_64 + id: "3" + body: + name: rabbit.jpg + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + - do: + indices.flush: + index: vectors_64 + + - do: + indices.refresh: { } + - do: + search: + index: vectors_64 + body: + knn: + field: vector + query_vector: [ 0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158 ] + k: 3 + num_candidates: 3 + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } +--- +"Test update int8_hnsw --> bbq_hnsw": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ optimized_scalar_quantization_bbq ] + test_runner_features: capabilities + reason: "BBQ is required to test upgrading to bbq_flat and bbq_hnsw" + - requires: + reason: 'dense_vector updatable to bbq capability required' + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path : /_mapping + capabilities: [ dense_vector_updatable_bbq ] + - do: + indices.create: + index: vectors_64 + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: int8_hnsw + + - do: + index: + index: vectors_64 + id: "1" + body: + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + - do: + indices.flush: + index: vectors_64 + + - do: + index: + index: vectors_64 + id: "2" + body: + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] + - do: + indices.flush: + index: vectors_64 + - do: + indices.put_mapping: + index: vectors_64 + body: + properties: + embedding: + type: dense_vector + dims: 64 + index_options: + type: bbq_hnsw + + - do: + indices.get_mapping: + index: vectors_64 + + - match: { vectors_64.mappings.properties.embedding.type: dense_vector } + - match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_hnsw } + + - do: + index: + index: vectors_64 + id: "3" + body: + name: rabbit.jpg + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + - do: + indices.flush: + index: vectors_64 + - do: + indices.refresh: { } + - do: + search: + index: vectors_64 + body: + knn: + field: vector + query_vector: [ 0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158 ] + k: 3 + num_candidates: 3 + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 61fc655cd10f2..a66824d77f8d9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -1588,7 +1588,9 @@ boolean updatableTo(IndexOptions update) { || update.type.equals(VectorIndexType.HNSW) || update.type.equals(VectorIndexType.INT8_HNSW) || update.type.equals(VectorIndexType.INT4_HNSW) - || update.type.equals(VectorIndexType.INT4_FLAT); + || update.type.equals(VectorIndexType.BBQ_HNSW) + || update.type.equals(VectorIndexType.INT4_FLAT) + || update.type.equals(VectorIndexType.BBQ_FLAT); } } @@ -1695,12 +1697,14 @@ public String toString() { @Override boolean updatableTo(IndexOptions update) { - boolean updatable = update.type.equals(this.type); - if (updatable) { + boolean updatable = false; + if (update.type.equals(VectorIndexType.INT4_HNSW)) { Int4HnswIndexOptions int4HnswIndexOptions = (Int4HnswIndexOptions) update; // fewer connections would break assumptions on max number of connections (based on largest previous graph) during merge // quantization could not behave as expected with different confidence intervals (and quantiles) to be created updatable = int4HnswIndexOptions.m >= this.m && confidenceInterval == int4HnswIndexOptions.confidenceInterval; + } else if (update.type.equals(VectorIndexType.BBQ_HNSW)) { + updatable = ((BBQHnswIndexOptions) update).m >= m; } return updatable; } @@ -1758,7 +1762,9 @@ boolean updatableTo(IndexOptions update) { return update.type.equals(this.type) || update.type.equals(VectorIndexType.HNSW) || update.type.equals(VectorIndexType.INT8_HNSW) - || update.type.equals(VectorIndexType.INT4_HNSW); + || update.type.equals(VectorIndexType.INT4_HNSW) + || update.type.equals(VectorIndexType.BBQ_HNSW) + || update.type.equals(VectorIndexType.BBQ_FLAT); } } @@ -1840,7 +1846,8 @@ boolean updatableTo(IndexOptions update) { || int8HnswIndexOptions.confidenceInterval != null && confidenceInterval.equals(int8HnswIndexOptions.confidenceInterval); } else { - updatable = update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= this.m; + updatable = update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= this.m + || (update.type.equals(VectorIndexType.BBQ_HNSW) && ((BBQHnswIndexOptions) update).m >= m); } return updatable; } @@ -1874,7 +1881,8 @@ boolean updatableTo(IndexOptions update) { } return updatable || (update.type.equals(VectorIndexType.INT8_HNSW) && ((Int8HnswIndexOptions) update).m >= m) - || (update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= m); + || (update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= m) + || (update.type.equals(VectorIndexType.BBQ_HNSW) && ((BBQHnswIndexOptions) update).m >= m); } @Override @@ -1924,7 +1932,7 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) { @Override boolean updatableTo(IndexOptions update) { - return update.type.equals(this.type); + return update.type.equals(this.type) && ((BBQHnswIndexOptions) update).m >= this.m; } @Override @@ -1978,7 +1986,7 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) { @Override boolean updatableTo(IndexOptions update) { - return update.type.equals(this.type); + return update.type.equals(this.type) || update.type.equals(VectorIndexType.BBQ_HNSW); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 9dc25fa59f7d6..af2ff0a857a74 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -48,6 +48,7 @@ private SearchCapabilities() {} private static final String SIGNIFICANT_TERMS_BACKGROUND_FILTER_AS_SUB = "significant_terms_background_filter_as_sub"; private static final String SIGNIFICANT_TERMS_ON_NESTED_FIELDS = "significant_terms_on_nested_fields"; private static final String EXCLUDE_VECTORS_PARAM = "exclude_vectors_param"; + private static final String DENSE_VECTOR_UPDATABLE_BBQ = "dense_vector_updatable_bbq"; public static final Set CAPABILITIES; static { @@ -68,6 +69,7 @@ private SearchCapabilities() {} capabilities.add(SIGNIFICANT_TERMS_BACKGROUND_FILTER_AS_SUB); capabilities.add(SIGNIFICANT_TERMS_ON_NESTED_FIELDS); capabilities.add(EXCLUDE_VECTORS_PARAM); + capabilities.add(DENSE_VECTOR_UPDATABLE_BBQ); CAPABILITIES = Set.copyOf(capabilities); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 0619fa95a4cab..5cc9d590f0f05 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -275,6 +275,36 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject(), m -> assertTrue(m.toString().contains("\"type\":\"int4_hnsw\"")) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"bbq_flat\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"bbq_hnsw\"")) + ); // update for int8_flat checker.registerUpdateCheck( b -> b.field("type", "dense_vector") @@ -355,6 +385,36 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject() ) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"bbq_flat\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"bbq_hnsw\"")) + ); // update for hnsw checker.registerUpdateCheck( b -> b.field("type", "dense_vector") @@ -480,6 +540,40 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject() ) ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject() + ) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"bbq_hnsw\"")) + ); // update for int8_hnsw checker.registerUpdateCheck( b -> b.field("type", "dense_vector") @@ -591,6 +685,40 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject() ) ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject() + ) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"bbq_hnsw\"")) + ); // update for int4_flat checker.registerUpdateCheck( b -> b.field("type", "dense_vector") @@ -677,6 +805,36 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject() ) ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"bbq_flat\"")) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"bbq_hnsw\"")) + ); // update for int4_hnsw checker.registerUpdateCheck( b -> b.field("type", "dense_vector") @@ -853,6 +1011,285 @@ protected void registerParameters(ParameterChecker checker) throws IOException { .endObject() ) ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject() + ) + ); + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"bbq_hnsw\"")) + ); + // update for bbq_flat + checker.registerUpdateCheck( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject(), + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject(), + m -> assertTrue(m.toString().contains("\"type\":\"bbq_hnsw\"")) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_flat") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ) + ); + // update for bbq_hnsw + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int8_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int4_flat") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "hnsw") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int8_hnsw") + .endObject() + ) + ); + checker.registerConflictCheck( + "index_options", + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "bbq_hnsw") + .endObject() + ), + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", dims * 16) + .field("index", true) + .startObject("index_options") + .field("type", "int4_hnsw") + .endObject() + ) + ); } @Override From 7a780a3fe6c2b45b5242c90cda23691e9c15ec8f Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Tue, 17 Jun 2025 10:53:05 +0200 Subject: [PATCH 2/3] IT fix --- .../elasticsearch/upgrades/DenseVectorMappingUpdateIT.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java index 0276e5a1558a0..00bf9b31c625f 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java @@ -15,6 +15,8 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -91,7 +93,8 @@ public void testDenseVectorMappingUpdateOnOldCluster() throws IOException { Request createIndex = new Request("PUT", "/" + indexName); boolean useSyntheticSource = randomBoolean() && oldClusterHasFeature(SYNTHETIC_SOURCE_FEATURE); - boolean useIndexSetting = SourceFieldMapper.onOrAfterDeprecateModeVersion(getOldClusterIndexVersion()); + IndexVersion version = getOldClusterIndexVersion(); + boolean useIndexSetting = version.onOrAfter(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER); XContentBuilder payload = XContentBuilder.builder(XContentType.JSON.xContent()).startObject(); if (useSyntheticSource) { if (useIndexSetting) { From 683fbba47bd68fbc4f13607a4f1abf0905d8a9d3 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 17 Jun 2025 09:01:26 +0000 Subject: [PATCH 3/3] [CI] Auto commit changes from spotless --- .../org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java index 00bf9b31c625f..60081c0d7812d 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType;