Skip to content

Commit a999178

Browse files
benwtrentbenwtrent
authored
Disallow index types updates to bbq_disk, revert (elastic#131760) (elastic#139061) (elastic#139086)
We planned and worked on `131760` from before our changes to the query execution path for bbq_disk indices. But, with all the changes, its less amenable to have hnsw & bbq_disk indices within the same shard. Maybe this is something we can do again in the future (I am frankly not sure), but allowing it naively just breaks too many expectations. (cherry picked from commit 604ca48)
1 parent 227bbac commit a999178

File tree

6 files changed

+22
-241
lines changed

6 files changed

+22
-241
lines changed

‎docs/changelog/139061.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 139061
2+
summary: "Disallow index types updates to bbq_disk, revert"
3+
area: Vector Search
4+
type: bug
5+
issues: []

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml‎

Lines changed: 11 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -701,14 +701,6 @@ setup:
701701

702702
---
703703
"Index, update and merge":
704-
- requires:
705-
capabilities:
706-
- method: POST
707-
path: /_search
708-
capabilities: [ dense_vector_updatable_bbq ]
709-
test_runner_features: capabilities
710-
reason: "BBQ disk is required to test upgrading to bbq_disk"
711-
712704
- do:
713705
indices.create:
714706
index: test_index
@@ -2126,14 +2118,14 @@ setup:
21262118
- contains: {hits.hits: {_id: "31"}}
21272119

21282120
---
2129-
"Test update flat --> bbq_flat --> bbq_hnsw --> bbq_disk":
2121+
"Test update flat --> bbq_flat --> bbq_hnsw":
21302122
- requires:
21312123
capabilities:
21322124
- method: POST
21332125
path: /_search
2134-
capabilities: [ update_field_to_bbq_disk, dense_vector_updatable_bbq ]
2126+
capabilities: [ optimized_scalar_quantization_bbq, dense_vector_updatable_bbq ]
21352127
test_runner_features: capabilities
2136-
reason: "BBQ disk is required to test upgrading to bbq_disk"
2128+
reason: "BBQ is required to test upgrading to bbq_flat and bbq_hnsw"
21372129

21382130
- do:
21392131
indices.create:
@@ -2238,41 +2230,6 @@ setup:
22382230
indices.flush:
22392231
index: vectors_64
22402232

2241-
- do:
2242-
indices.put_mapping:
2243-
index: vectors_64
2244-
body:
2245-
properties:
2246-
embedding:
2247-
type: dense_vector
2248-
dims: 64
2249-
index_options:
2250-
type: bbq_disk
2251-
2252-
- do:
2253-
indices.get_mapping:
2254-
index: vectors_64
2255-
2256-
- match: { vectors_64.mappings.properties.embedding.type: dense_vector }
2257-
- match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_disk }
2258-
2259-
- do:
2260-
index:
2261-
index: vectors_64
2262-
id: "4"
2263-
body:
2264-
vector: [0.011, 0.052, 0.099, 0.138, 0.040, -0.121, 0.215, -0.021,
2265-
0.003, 0.064, -0.195, -0.010, 0.080, 0.072, -0.016, 0.016,
2266-
0.006, 0.168, 0.063, 0.057, -0.036, 0.042, -0.011, -0.007,
2267-
-0.004, 0.010, 0.017, 0.031, -0.025, 0.014, -0.058, -0.001,
2268-
-0.079, 0.026, -0.016, 0.087, -0.014, 0.019, 0.018, 0.107,
2269-
-0.022, -0.003, 0.047, 0.041, 0.027, -0.031, 0.015, 0.017,
2270-
0.021, -0.023, -0.018, 0.015, -0.007, 0.037, -0.044, 0.115,
2271-
-0.068, 0.132, 0.019, -0.017, -0.010, -0.029, -0.008, -0.016]
2272-
- do:
2273-
indices.flush:
2274-
index: vectors_64
2275-
22762233
- do:
22772234
indices.forcemerge:
22782235
index: vectors_64
@@ -2294,20 +2251,19 @@ setup:
22942251
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
22952252
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
22962253
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158 ]
2297-
k: 4
2298-
num_candidates: 4
2254+
k: 3
2255+
num_candidates: 3
22992256

23002257
- match: { hits.hits.0._id: "1" }
23012258
- match: { hits.hits.1._id: "3" }
23022259
- match: { hits.hits.2._id: "2" }
2303-
- match: { hits.hits.3._id: "4" }
23042260
---
2305-
"Test update int8_hnsw --> bbq_flat --> bbq_disk":
2261+
"Test update int8_hnsw --> bbq_flat":
23062262
- requires:
23072263
capabilities:
23082264
- method: POST
23092265
path: /_search
2310-
capabilities: [ optimized_scalar_quantization_bbq, dense_vector_updatable_bbq, update_field_to_bbq_disk ]
2266+
capabilities: [ optimized_scalar_quantization_bbq, dense_vector_updatable_bbq ]
23112267
test_runner_features: capabilities
23122268
reason: "BBQ is required to test upgrading to bbq_flat and bbq_hnsw"
23132269

@@ -2376,27 +2332,6 @@ setup:
23762332
0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569,
23772333
-0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013]
23782334

2379-
- do:
2380-
indices.flush:
2381-
index: vectors_64
2382-
- do:
2383-
indices.put_mapping:
2384-
index: vectors_64
2385-
body:
2386-
properties:
2387-
embedding:
2388-
type: dense_vector
2389-
dims: 64
2390-
index_options:
2391-
type: bbq_disk
2392-
2393-
- do:
2394-
indices.get_mapping:
2395-
index: vectors_64
2396-
2397-
- match: { vectors_64.mappings.properties.embedding.type: dense_vector }
2398-
- match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_disk }
2399-
24002335
- do:
24012336
index:
24022337
index: vectors_64
@@ -2438,12 +2373,12 @@ setup:
24382373
- match: { hits.hits.1._id: "3" }
24392374
- match: { hits.hits.2._id: "2" }
24402375
---
2441-
"Test update int8_hnsw --> bbq_hnsw --> bbq_disk":
2376+
"Test update int8_hnsw --> bbq_hnsw":
24422377
- requires:
24432378
capabilities:
24442379
- method: POST
24452380
path: /_search
2446-
capabilities: [ optimized_scalar_quantization_bbq, dense_vector_updatable_bbq, update_field_to_bbq_disk ]
2381+
capabilities: [ optimized_scalar_quantization_bbq, dense_vector_updatable_bbq ]
24472382
test_runner_features: capabilities
24482383
reason: "BBQ is required to test upgrading to bbq_flat and bbq_hnsw"
24492384

@@ -2481,24 +2416,6 @@ setup:
24812416
indices.flush:
24822417
index: vectors_64
24832418

2484-
- do:
2485-
indices.put_mapping:
2486-
index: vectors_64
2487-
body:
2488-
properties:
2489-
embedding:
2490-
type: dense_vector
2491-
dims: 64
2492-
index_options:
2493-
type: bbq_hnsw
2494-
2495-
- do:
2496-
indices.get_mapping:
2497-
index: vectors_64
2498-
2499-
- match: { vectors_64.mappings.properties.embedding.type: dense_vector }
2500-
- match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_hnsw }
2501-
25022419
- do:
25032420
index:
25042421
index: vectors_64
@@ -2524,14 +2441,14 @@ setup:
25242441
type: dense_vector
25252442
dims: 64
25262443
index_options:
2527-
type: bbq_disk
2444+
type: bbq_hnsw
25282445

25292446
- do:
25302447
indices.get_mapping:
25312448
index: vectors_64
25322449

25332450
- match: { vectors_64.mappings.properties.embedding.type: dense_vector }
2534-
- match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_disk }
2451+
- match: { vectors_64.mappings.properties.embedding.index_options.type: bbq_hnsw }
25352452

25362453
- do:
25372454
index:

‎server/src/internalClusterTest/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldIndexTypeUpdateIT.java‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -50,17 +50,7 @@ public DenseVectorFieldIndexTypeUpdateIT(@Name("initialType") String initialType
5050

5151
@ParametersFactory
5252
public static Collection<Object[]> params() {
53-
List<String> types = List.of(
54-
"flat",
55-
"int8_flat",
56-
"int4_flat",
57-
"bbq_flat",
58-
"hnsw",
59-
"int8_hnsw",
60-
"int4_hnsw",
61-
"bbq_hnsw",
62-
"bbq_disk"
63-
);
53+
List<String> types = List.of("flat", "int8_flat", "int4_flat", "bbq_flat", "hnsw", "int8_hnsw", "int4_hnsw", "bbq_hnsw");
6454

6555
// A type can be upgraded to types that follow in the list...
6656
List<Object[]> params = new java.util.ArrayList<>();

‎server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1642,8 +1642,7 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
16421642
|| update.type.equals(VectorIndexType.INT4_HNSW)
16431643
|| update.type.equals(VectorIndexType.BBQ_HNSW)
16441644
|| update.type.equals(VectorIndexType.INT4_FLAT)
1645-
|| update.type.equals(VectorIndexType.BBQ_FLAT)
1646-
|| update.type.equals(VectorIndexType.BBQ_DISK);
1645+
|| update.type.equals(VectorIndexType.BBQ_FLAT);
16471646
}
16481647
}
16491648

@@ -1768,8 +1767,6 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
17681767
updatable = int4HnswIndexOptions.m >= this.m && confidenceInterval == int4HnswIndexOptions.confidenceInterval;
17691768
} else if (update.type.equals(VectorIndexType.BBQ_HNSW)) {
17701769
updatable = ((BBQHnswIndexOptions) update).m >= m;
1771-
} else {
1772-
updatable = update.type.equals(VectorIndexType.BBQ_DISK);
17731770
}
17741771
return updatable;
17751772
}
@@ -1834,8 +1831,7 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
18341831
|| update.type.equals(VectorIndexType.INT8_HNSW)
18351832
|| update.type.equals(VectorIndexType.INT4_HNSW)
18361833
|| update.type.equals(VectorIndexType.BBQ_HNSW)
1837-
|| update.type.equals(VectorIndexType.BBQ_FLAT)
1838-
|| update.type.equals(VectorIndexType.BBQ_DISK);
1834+
|| update.type.equals(VectorIndexType.BBQ_FLAT);
18391835
}
18401836

18411837
}
@@ -1934,8 +1930,7 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
19341930
|| int8HnswIndexOptions.confidenceInterval != null
19351931
&& confidenceInterval.equals(int8HnswIndexOptions.confidenceInterval);
19361932
} else {
1937-
updatable = update.type.equals(VectorIndexType.BBQ_DISK)
1938-
|| update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= this.m
1933+
updatable = update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= this.m
19391934
|| (update.type.equals(VectorIndexType.BBQ_HNSW) && ((BBQHnswIndexOptions) update).m >= m);
19401935
}
19411936
return updatable;
@@ -1969,7 +1964,6 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
19691964
updatable = hnswIndexOptions.m >= this.m;
19701965
}
19711966
return updatable
1972-
|| update.type.equals(VectorIndexType.BBQ_DISK)
19731967
|| (update.type.equals(VectorIndexType.INT8_HNSW) && ((Int8HnswIndexOptions) update).m >= m)
19741968
|| (update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= m)
19751969
|| (update.type.equals(VectorIndexType.BBQ_HNSW) && ((BBQHnswIndexOptions) update).m >= m);
@@ -2035,8 +2029,7 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) {
20352029

20362030
@Override
20372031
public boolean updatableTo(DenseVectorIndexOptions update) {
2038-
return (update.type.equals(this.type) && ((BBQHnswIndexOptions) update).m >= this.m)
2039-
|| update.type.equals(VectorIndexType.BBQ_DISK);
2032+
return update.type.equals(this.type) && ((BBQHnswIndexOptions) update).m >= this.m;
20402033
}
20412034

20422035
@Override
@@ -2095,9 +2088,7 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) {
20952088

20962089
@Override
20972090
public boolean updatableTo(DenseVectorIndexOptions update) {
2098-
return update.type.equals(this.type)
2099-
|| update.type.equals(VectorIndexType.BBQ_HNSW)
2100-
|| update.type.equals(VectorIndexType.BBQ_DISK);
2091+
return update.type.equals(this.type) || update.type.equals(VectorIndexType.BBQ_HNSW);
21012092
}
21022093

21032094
@Override

‎server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@ private SearchCapabilities() {}
5555
private static final String EXCLUDE_VECTORS_PARAM = "exclude_vectors_param";
5656
private static final String DENSE_VECTOR_UPDATABLE_BBQ = "dense_vector_updatable_bbq";
5757
private static final String FIELD_EXISTS_QUERY_FOR_TEXT_FIELDS_NO_INDEX_OR_DV = "field_exists_query_for_text_fields_no_index_or_dv";
58-
private static final String UPDATE_FIELD_TO_BBQ_DISK = "update_field_to_bbq_disk";
5958
private static final String KNN_FILTER_ON_NESTED_FIELDS_CAPABILITY = "knn_filter_on_nested_fields";
6059
private static final String BUCKET_SCRIPT_PARENT_MULTI_BUCKET_ERROR = "bucket_script_parent_multi_bucket_error";
6160
private static final String EXCLUDE_SOURCE_VECTORS_SETTING = "exclude_source_vectors_setting";
@@ -86,7 +85,6 @@ private SearchCapabilities() {}
8685
capabilities.add(EXCLUDE_VECTORS_PARAM);
8786
capabilities.add(DENSE_VECTOR_UPDATABLE_BBQ);
8887
capabilities.add(FIELD_EXISTS_QUERY_FOR_TEXT_FIELDS_NO_INDEX_OR_DV);
89-
capabilities.add(UPDATE_FIELD_TO_BBQ_DISK);
9088
capabilities.add(KNN_FILTER_ON_NESTED_FIELDS_CAPABILITY);
9189
capabilities.add(BUCKET_SCRIPT_PARENT_MULTI_BUCKET_ERROR);
9290
capabilities.add(EXCLUDE_SOURCE_VECTORS_SETTING);

0 commit comments

Comments
 (0)