Skip to content

Commit 3212ec2

Browse files
carlosdelestcarlosdelest
authored
Allow update any index type to bbq_disk (#131760)
1 parent 079355a commit 3212ec2

File tree

4 files changed

+233
-37
lines changed

4 files changed

+233
-37
lines changed

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml‎

Lines changed: 97 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -701,6 +701,14 @@ setup:
701701

702702
---
703703
"Index, update and merge":
704+
- requires:
705+
capabilities:
706+
- method: POST
707+
path: /_search
708+
capabilities: [ dense_vector_updatable_bbq ]
709+
test_runner_features: capabilities
710+
reason: "BBQ disk is required to test upgrading to bbq_disk"
711+
704712
- do:
705713
indices.create:
706714
index: test_index
@@ -2118,21 +2126,15 @@ setup:
21182126
- contains: {hits.hits: {_id: "31"}}
21192127

21202128
---
2121-
"Test update flat --> bbq_flat --> bbq_hnsw":
2129+
"Test update flat --> bbq_flat --> bbq_hnsw --> bbq_disk":
21222130
- requires:
21232131
capabilities:
21242132
- method: POST
21252133
path: /_search
2126-
capabilities: [ optimized_scalar_quantization_bbq ]
2134+
capabilities: [ update_field_to_bbq_disk, dense_vector_updatable_bbq ]
21272135
test_runner_features: capabilities
2128-
reason: "BBQ is required to test upgrading to bbq_flat and bbq_hnsw"
2129-
- requires:
2130-
reason: 'dense_vector updatable to bbq capability required'
2131-
test_runner_features: [ capabilities ]
2132-
capabilities:
2133-
- method: PUT
2134-
path : /_mapping
2135-
capabilities: [ dense_vector_updatable_bbq ]
2136+
reason: "BBQ disk is required to test upgrading to bbq_disk"
2137+
21362138
- do:
21372139
indices.create:
21382140
index: vectors_64
@@ -2236,6 +2238,41 @@ setup:
22362238
indices.flush:
22372239
index: vectors_64
22382240

2241+
- do:
2242+
indices.put_mapping:
2243+
index: vectors_64
2244+
body:
2245+
properties:
2246+
embedding:
2247+
type: dense_vector
2248+
dims: 64
2249+
index_options:
2250+
type: bbq_disk
2251+
2252+
- do:
2253+
indices.get_mapping:
2254+
index: vectors_64
2255+
2256+
- match: { vectors_64.mappings.properties.embedding.type: dense_vector }
2257+
- match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_disk }
2258+
2259+
- do:
2260+
index:
2261+
index: vectors_64
2262+
id: "4"
2263+
body:
2264+
vector: [0.011, 0.052, 0.099, 0.138, 0.040, -0.121, 0.215, -0.021,
2265+
0.003, 0.064, -0.195, -0.010, 0.080, 0.072, -0.016, 0.016,
2266+
0.006, 0.168, 0.063, 0.057, -0.036, 0.042, -0.011, -0.007,
2267+
-0.004, 0.010, 0.017, 0.031, -0.025, 0.014, -0.058, -0.001,
2268+
-0.079, 0.026, -0.016, 0.087, -0.014, 0.019, 0.018, 0.107,
2269+
-0.022, -0.003, 0.047, 0.041, 0.027, -0.031, 0.015, 0.017,
2270+
0.021, -0.023, -0.018, 0.015, -0.007, 0.037, -0.044, 0.115,
2271+
-0.068, 0.132, 0.019, -0.017, -0.010, -0.029, -0.008, -0.016]
2272+
- do:
2273+
indices.flush:
2274+
index: vectors_64
2275+
22392276
- do:
22402277
indices.forcemerge:
22412278
index: vectors_64
@@ -2257,28 +2294,23 @@ setup:
22572294
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
22582295
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
22592296
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158 ]
2260-
k: 3
2261-
num_candidates: 3
2297+
k: 4
2298+
num_candidates: 4
22622299

22632300
- match: { hits.hits.0._id: "1" }
22642301
- match: { hits.hits.1._id: "3" }
22652302
- match: { hits.hits.2._id: "2" }
2303+
- match: { hits.hits.3._id: "4" }
22662304
---
2267-
"Test update int8_hnsw --> bbq_flat":
2305+
"Test update int8_hnsw --> bbq_flat --> bbq_disk":
22682306
- requires:
22692307
capabilities:
22702308
- method: POST
22712309
path: /_search
2272-
capabilities: [ optimized_scalar_quantization_bbq ]
2310+
capabilities: [ optimized_scalar_quantization_bbq, dense_vector_updatable_bbq, update_field_to_bbq_disk ]
22732311
test_runner_features: capabilities
22742312
reason: "BBQ is required to test upgrading to bbq_flat and bbq_hnsw"
2275-
- requires:
2276-
reason: 'dense_vector updatable to bbq capability required'
2277-
test_runner_features: [ capabilities ]
2278-
capabilities:
2279-
- method: PUT
2280-
path : /_mapping
2281-
capabilities: [ dense_vector_updatable_bbq ]
2313+
22822314
- do:
22832315
indices.create:
22842316
index: vectors_64
@@ -2344,6 +2376,27 @@ setup:
23442376
0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569,
23452377
-0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013]
23462378

2379+
- do:
2380+
indices.flush:
2381+
index: vectors_64
2382+
- do:
2383+
indices.put_mapping:
2384+
index: vectors_64
2385+
body:
2386+
properties:
2387+
embedding:
2388+
type: dense_vector
2389+
dims: 64
2390+
index_options:
2391+
type: bbq_disk
2392+
2393+
- do:
2394+
indices.get_mapping:
2395+
index: vectors_64
2396+
2397+
- match: { vectors_64.mappings.properties.embedding.type: dense_vector }
2398+
- match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_disk }
2399+
23472400
- do:
23482401
index:
23492402
index: vectors_64
@@ -2385,21 +2438,15 @@ setup:
23852438
- match: { hits.hits.1._id: "3" }
23862439
- match: { hits.hits.2._id: "2" }
23872440
---
2388-
"Test update int8_hnsw --> bbq_hnsw":
2441+
"Test update int8_hnsw --> bbq_hnsw --> bbq_disk":
23892442
- requires:
23902443
capabilities:
23912444
- method: POST
23922445
path: /_search
2393-
capabilities: [ optimized_scalar_quantization_bbq ]
2446+
capabilities: [ optimized_scalar_quantization_bbq, dense_vector_updatable_bbq, update_field_to_bbq_disk ]
23942447
test_runner_features: capabilities
23952448
reason: "BBQ is required to test upgrading to bbq_flat and bbq_hnsw"
2396-
- requires:
2397-
reason: 'dense_vector updatable to bbq capability required'
2398-
test_runner_features: [ capabilities ]
2399-
capabilities:
2400-
- method: PUT
2401-
path : /_mapping
2402-
capabilities: [ dense_vector_updatable_bbq ]
2449+
24032450
- do:
24042451
indices.create:
24052452
index: vectors_64
@@ -2434,6 +2481,24 @@ setup:
24342481
indices.flush:
24352482
index: vectors_64
24362483

2484+
- do:
2485+
indices.put_mapping:
2486+
index: vectors_64
2487+
body:
2488+
properties:
2489+
embedding:
2490+
type: dense_vector
2491+
dims: 64
2492+
index_options:
2493+
type: bbq_hnsw
2494+
2495+
- do:
2496+
indices.get_mapping:
2497+
index: vectors_64
2498+
2499+
- match: { vectors_64.mappings.properties.embedding.type: dense_vector }
2500+
- match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_hnsw }
2501+
24372502
- do:
24382503
index:
24392504
index: vectors_64
@@ -2459,14 +2524,14 @@ setup:
24592524
type: dense_vector
24602525
dims: 64
24612526
index_options:
2462-
type: bbq_hnsw
2527+
type: bbq_disk
24632528

24642529
- do:
24652530
indices.get_mapping:
24662531
index: vectors_64
24672532

24682533
- match: { vectors_64.mappings.properties.embedding.type: dense_vector }
2469-
- match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_hnsw }
2534+
- match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_disk }
24702535

24712536
- do:
24722537
index:

‎server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1822,7 +1822,8 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
18221822
|| update.type.equals(VectorIndexType.INT4_HNSW)
18231823
|| update.type.equals(VectorIndexType.BBQ_HNSW)
18241824
|| update.type.equals(VectorIndexType.INT4_FLAT)
1825-
|| update.type.equals(VectorIndexType.BBQ_FLAT);
1825+
|| update.type.equals(VectorIndexType.BBQ_FLAT)
1826+
|| update.type.equals(VectorIndexType.BBQ_DISK);
18261827
}
18271828
}
18281829

@@ -1947,6 +1948,8 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
19471948
updatable = int4HnswIndexOptions.m >= this.m && confidenceInterval == int4HnswIndexOptions.confidenceInterval;
19481949
} else if (update.type.equals(VectorIndexType.BBQ_HNSW)) {
19491950
updatable = ((BBQHnswIndexOptions) update).m >= m;
1951+
} else {
1952+
updatable = update.type.equals(VectorIndexType.BBQ_DISK);
19501953
}
19511954
return updatable;
19521955
}
@@ -2011,7 +2014,8 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
20112014
|| update.type.equals(VectorIndexType.INT8_HNSW)
20122015
|| update.type.equals(VectorIndexType.INT4_HNSW)
20132016
|| update.type.equals(VectorIndexType.BBQ_HNSW)
2014-
|| update.type.equals(VectorIndexType.BBQ_FLAT);
2017+
|| update.type.equals(VectorIndexType.BBQ_FLAT)
2018+
|| update.type.equals(VectorIndexType.BBQ_DISK);
20152019
}
20162020

20172021
}
@@ -2098,7 +2102,8 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
20982102
|| int8HnswIndexOptions.confidenceInterval != null
20992103
&& confidenceInterval.equals(int8HnswIndexOptions.confidenceInterval);
21002104
} else {
2101-
updatable = update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= this.m
2105+
updatable = update.type.equals(VectorIndexType.BBQ_DISK)
2106+
|| update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= this.m
21022107
|| (update.type.equals(VectorIndexType.BBQ_HNSW) && ((BBQHnswIndexOptions) update).m >= m);
21032108
}
21042109
return updatable;
@@ -2132,6 +2137,7 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
21322137
updatable = hnswIndexOptions.m >= this.m;
21332138
}
21342139
return updatable
2140+
|| update.type.equals(VectorIndexType.BBQ_DISK)
21352141
|| (update.type.equals(VectorIndexType.INT8_HNSW) && ((Int8HnswIndexOptions) update).m >= m)
21362142
|| (update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= m)
21372143
|| (update.type.equals(VectorIndexType.BBQ_HNSW) && ((BBQHnswIndexOptions) update).m >= m);
@@ -2189,7 +2195,8 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) {
21892195

21902196
@Override
21912197
public boolean updatableTo(DenseVectorIndexOptions update) {
2192-
return update.type.equals(this.type) && ((BBQHnswIndexOptions) update).m >= this.m;
2198+
return (update.type.equals(this.type) && ((BBQHnswIndexOptions) update).m >= this.m)
2199+
|| update.type.equals(VectorIndexType.BBQ_DISK);
21932200
}
21942201

21952202
@Override
@@ -2248,7 +2255,9 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) {
22482255

22492256
@Override
22502257
public boolean updatableTo(DenseVectorIndexOptions update) {
2251-
return update.type.equals(this.type) || update.type.equals(VectorIndexType.BBQ_HNSW);
2258+
return update.type.equals(this.type)
2259+
|| update.type.equals(VectorIndexType.BBQ_HNSW)
2260+
|| update.type.equals(VectorIndexType.BBQ_DISK);
22522261
}
22532262

22542263
@Override

‎server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ private SearchCapabilities() {}
5656
private static final String DENSE_VECTOR_UPDATABLE_BBQ = "dense_vector_updatable_bbq";
5757
private static final String FIELD_EXISTS_QUERY_FOR_TEXT_FIELDS_NO_INDEX_OR_DV = "field_exists_query_for_text_fields_no_index_or_dv";
5858
private static final String SYNTHETIC_VECTORS_SETTING = "synthetic_vectors_setting";
59+
private static final String UPDATE_FIELD_TO_BBQ_DISK = "update_field_to_bbq_disk";
5960

6061
public static final Set<String> CAPABILITIES;
6162
static {
@@ -80,6 +81,7 @@ private SearchCapabilities() {}
8081
capabilities.add(EXCLUDE_VECTORS_PARAM);
8182
capabilities.add(DENSE_VECTOR_UPDATABLE_BBQ);
8283
capabilities.add(FIELD_EXISTS_QUERY_FOR_TEXT_FIELDS_NO_INDEX_OR_DV);
84+
capabilities.add(UPDATE_FIELD_TO_BBQ_DISK);
8385
if (SYNTHETIC_VECTORS) {
8486
capabilities.add(SYNTHETIC_VECTORS_SETTING);
8587
}

0 commit comments

Comments
 (0)