Skip to content

Commit 65d8b48

Browse files
benwtrentbenwtrent
authored
Disk bbq license enforcement (elastic#139087)
This enforces enterprise license for bbq_disk (DiskBBQ) indices. - 9.2 indices are able to continue to use bbq_disk - In indices created in 9.3+ writes are licensed, reads are not. - License check is only applied on format write instantiation. Not on field & index creation.
1 parent 58b9a59 commit 65d8b48

File tree

24 files changed

+849
-253
lines changed

24 files changed

+849
-253
lines changed

‎docs/changelog/139087.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 139087
2+
summary: Disk bbq license enforcement
3+
area: Vector Search
4+
type: bug
5+
issues: []

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/180_update_dense_vector_type.yml‎

Lines changed: 0 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -1674,109 +1674,6 @@ setup:
16741674
- match: { test_index.mappings.properties.embedding.index_options.confidence_interval: 0.3 }
16751675

16761676
---
1677-
"Allowed dense vector updates on same type but different other index_options, bbq_disk":
1678-
- requires:
1679-
cluster_features: "mapper.vectors.diskbbq_on_disk_rescoring"
1680-
reason: 'diskbbq needs to support on-disk rescoring'
1681-
- requires:
1682-
test_runner_features: [ contains ]
1683-
- do:
1684-
indices.create:
1685-
index: test_index
1686-
1687-
- do:
1688-
indices.put_mapping:
1689-
index: test_index
1690-
body:
1691-
properties:
1692-
embedding:
1693-
type: dense_vector
1694-
dims: 64
1695-
index_options:
1696-
type: bbq_disk
1697-
1698-
- do:
1699-
indices.get_mapping:
1700-
index: test_index
1701-
1702-
- match: { test_index.mappings.properties.embedding.type: dense_vector }
1703-
- match: { test_index.mappings.properties.embedding.index_options.type: bbq_disk }
1704-
1705-
- do:
1706-
index:
1707-
index: test_index
1708-
id: "1"
1709-
body:
1710-
embedding: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313,
1711-
0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272,
1712-
0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132,
1713-
-0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265,
1714-
-0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475,
1715-
-0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242,
1716-
-0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45,
1717-
-0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176]
1718-
- do:
1719-
indices.flush:
1720-
index: test_index
1721-
1722-
- do:
1723-
indices.put_mapping:
1724-
index: test_index
1725-
body:
1726-
properties:
1727-
embedding:
1728-
type: dense_vector
1729-
dims: 64
1730-
index_options:
1731-
type: bbq_disk
1732-
on_disk_rescore: true
1733-
1734-
- do:
1735-
indices.get_mapping:
1736-
index: test_index
1737-
1738-
- match: { test_index.mappings.properties.embedding.type: dense_vector }
1739-
- match: { test_index.mappings.properties.embedding.index_options.type: bbq_disk }
1740-
- match: { test_index.mappings.properties.embedding.index_options.on_disk_rescore: true }
1741-
1742-
- do:
1743-
index:
1744-
index: test_index
1745-
id: "2"
1746-
body:
1747-
embedding: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348,
1748-
-0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048,
1749-
0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438,
1750-
-0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138,
1751-
-0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429,
1752-
-0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166,
1753-
0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569,
1754-
-0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013]
1755-
- do:
1756-
indices.flush:
1757-
index: test_index
1758-
- do:
1759-
indices.refresh: { }
1760-
- do:
1761-
search:
1762-
index: test_index
1763-
body:
1764-
knn:
1765-
field: embedding
1766-
query_vector: [ 0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
1767-
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
1768-
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
1769-
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
1770-
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
1771-
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
1772-
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
1773-
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158 ]
1774-
k: 2
1775-
num_candidates: 2
1776-
1777-
- match: { hits.hits.0._id: "1" }
1778-
- match: { hits.hits.1._id: "2" }
1779-
---
17801677
"Test create and update dense vector mapping to int4 with per-doc indexing and flush":
17811678
- requires:
17821679
cluster_features: "gte_v8.16.0"

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/220_dense_vector_node_index_stats.yml‎

Lines changed: 0 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -458,115 +458,6 @@
458458
- is_false: indices.bbq_quantized.primaries.dense_vector.off_heap.fielddata.vector2.cenivf_size_bytes
459459
- is_false: indices.bbq_quantized.primaries.dense_vector.off_heap.fielddata.vector2.clivf_size_bytes
460460

461-
---
462-
"index node stats bbq_disk quantized":
463-
- requires:
464-
capabilities:
465-
- method: GET
466-
path: /_nodes/stats
467-
capabilities: [ dense_vector_off_heap_stats ]
468-
test_runner_features: [ capabilities ]
469-
reason: Capability required to run test
470-
- requires:
471-
capabilities:
472-
- method: POST
473-
path: /_search
474-
capabilities: [ optimized_scalar_quantization_bbq ]
475-
test_runner_features: capabilities
476-
reason: "Uses bbq"
477-
- requires:
478-
cluster_features: ["mapper.bbq_disk_support"]
479-
reason: Needs mapper.bbq_disk_support feature
480-
- requires:
481-
cluster_features: ["mapper.bbq_disk_stats_support"]
482-
reason: Needs bbq_disk DenseVectorStats support
483-
484-
- do:
485-
indices.create:
486-
index: bbq_quantized
487-
body:
488-
settings:
489-
index:
490-
number_of_shards: 1
491-
number_of_replicas: 0
492-
mappings:
493-
properties:
494-
vector3:
495-
type: dense_vector
496-
dims: 64
497-
index: true
498-
similarity: l2_norm
499-
index_options:
500-
type: bbq_disk
501-
502-
- do:
503-
index:
504-
index: bbq_quantized
505-
id: "3"
506-
body:
507-
vector3: [ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
508-
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
509-
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
510-
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 ]
511-
512-
- do:
513-
indices.refresh: {}
514-
515-
- do:
516-
cat.shards:
517-
format: "json"
518-
h: [ id ]
519-
index: "bbq_quantized"
520-
521-
- do:
522-
nodes.stats:
523-
metric: [ indices ]
524-
node_id: $body.0.id
525-
526-
- set:
527-
nodes._arbitrary_key_: node_id
528-
529-
- is_true: cluster_name
530-
- is_true: nodes
531-
- is_true: nodes.$node_id.name
532-
- is_true: nodes.$node_id.indices.dense_vector
533-
- match: { nodes.$node_id.indices.dense_vector.value_count: 1 }
534-
- is_true: nodes.$node_id.indices.dense_vector.off_heap
535-
- gt: { nodes.$node_id.indices.dense_vector.off_heap.total_size_bytes: 0 }
536-
- match: { nodes.$node_id.indices.dense_vector.off_heap.total_veb_size_bytes: 0 }
537-
- gt: { nodes.$node_id.indices.dense_vector.off_heap.total_vec_size_bytes: 0 }
538-
- match: { nodes.$node_id.indices.dense_vector.off_heap.total_veq_size_bytes: 0 }
539-
- match: { nodes.$node_id.indices.dense_vector.off_heap.total_vex_size_bytes: 0 }
540-
- gt: { nodes.$node_id.indices.dense_vector.off_heap.total_cenivf_size_bytes: 0 }
541-
- gt: { nodes.$node_id.indices.dense_vector.off_heap.total_clivf_size_bytes: 0 }
542-
- is_false: nodes.$node_id.indices.dense_vector.off_heap.fielddata
543-
544-
- do:
545-
indices.stats: { index: _all }
546-
547-
- is_true: _all.primaries.dense_vector.off_heap
548-
- is_false: _all.primaries.dense_vector.off_heap.fielddata
549-
- is_true: _all.total.dense_vector.off_heap
550-
- is_false: _all.total.dense_vector.off_heap.fielddata
551-
552-
- is_true: indices.bbq_quantized.primaries.dense_vector
553-
- gt: { indices.bbq_quantized.primaries.dense_vector.off_heap.total_size_bytes: 0 }
554-
- match: { indices.bbq_quantized.primaries.dense_vector.off_heap.total_veb_size_bytes: 0 }
555-
- gt: { indices.bbq_quantized.primaries.dense_vector.off_heap.total_vec_size_bytes: 0 }
556-
- match: { indices.bbq_quantized.primaries.dense_vector.off_heap.total_veq_size_bytes: 0 }
557-
- match: { indices.bbq_quantized.primaries.dense_vector.off_heap.total_vex_size_bytes: 0 }
558-
- gt: { indices.bbq_quantized.primaries.dense_vector.off_heap.total_cenivf_size_bytes: 0 }
559-
- gt: { indices.bbq_quantized.primaries.dense_vector.off_heap.total_clivf_size_bytes: 0 }
560-
# vector3, bbq_disk
561-
- is_true: indices.bbq_quantized.primaries.dense_vector.off_heap.fielddata.vector3
562-
- gt: { indices.bbq_quantized.primaries.dense_vector.off_heap.fielddata.vector3.vec_size_bytes: 0 }
563-
- gt: { indices.bbq_quantized.primaries.dense_vector.off_heap.fielddata.vector3.cenivf_size_bytes: 0 }
564-
- gt: { indices.bbq_quantized.primaries.dense_vector.off_heap.fielddata.vector3.clivf_size_bytes: 0 }
565-
- is_false: indices.bbq_quantized.primaries.dense_vector.off_heap.fielddata.vector3.veb_size_bytes
566-
- is_false: indices.bbq_quantized.primaries.dense_vector.off_heap.fielddata.vector3.veq_size_bytes
567-
- is_false: indices.bbq_quantized.primaries.dense_vector.off_heap.fielddata.vector3.vex_size_bytes
568-
569-
570461
---
571462
"index node stats bit_vectors":
572463
- requires:

‎server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ protected boolean useDirectIO(String name, IOContext context, OptionalLong fileL
7474

7575
@ParametersFactory
7676
public static Iterable<Object[]> parameters() {
77-
return Stream.of("int4_hnsw", "int8_hnsw", "bbq_hnsw", "bbq_disk").map(s -> new Object[] { s }).toList();
77+
return Stream.of("int4_hnsw", "int8_hnsw", "bbq_hnsw").map(s -> new Object[] { s }).toList();
7878
}
7979

8080
public DirectIOIT(String type) {

‎server/src/main/java/module-info.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,9 @@
401401
org.elasticsearch.serverless.apifiltering,
402402
org.elasticsearch.serverless.stateless,
403403
org.elasticsearch.internal.security,
404-
org.elasticsearch.xpack.gpu;
404+
org.elasticsearch.xpack.core,
405+
org.elasticsearch.xpack.gpu,
406+
org.elasticsearch.xpack.diskbbq;
405407

406408
exports org.elasticsearch.telemetry.tracing;
407409
exports org.elasticsearch.telemetry;
@@ -500,8 +502,8 @@
500502
exports org.elasticsearch.index.codec.vectors.reflect; // to org.elasticsearch.gpu;
501503
exports org.elasticsearch.index.codec.vectors.es818 to org.elasticsearch.test.knn;
502504
exports org.elasticsearch.inference.telemetry;
503-
exports org.elasticsearch.index.codec.vectors.diskbbq to org.elasticsearch.test.knn;
504-
exports org.elasticsearch.index.codec.vectors.diskbbq.next to org.elasticsearch.test.knn;
505+
exports org.elasticsearch.index.codec.vectors.diskbbq to org.elasticsearch.test.knn, org.elasticsearch.xpack.diskbbq;
506+
exports org.elasticsearch.index.codec.vectors.diskbbq.next to org.elasticsearch.test.knn, org.elasticsearch.xpack.diskbbq;
505507
exports org.elasticsearch.index.codec.vectors.cluster to org.elasticsearch.test.knn;
506508
exports org.elasticsearch.index.codec.vectors.es93 to org.elasticsearch.test.knn;
507509
exports org.elasticsearch.search.crossproject;

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,7 @@ private static Version parseUnchecked(String version) {
201201
public static final IndexVersion NESTED_PATH_LIMIT = def(9_050_0_00, Version.LUCENE_10_3_2);
202202
public static final IndexVersion GENERIC_DENSE_VECTOR_FORMAT = def(9_051_0_00, Version.LUCENE_10_3_2);
203203
public static final IndexVersion SKIPPER_DEFAULTS_ONLY_ON_TSDB = def(9_052_0_00, Version.LUCENE_10_3_2);
204+
public static final IndexVersion DISK_BBQ_LICENSE_ENFORCEMENT = def(9_053_0_00, Version.LUCENE_10_3_2);
204205

205206
/*
206207
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.apache.lucene.util.BytesRef;
4242
import org.apache.lucene.util.VectorUtil;
4343
import org.elasticsearch.Build;
44+
import org.elasticsearch.ElasticsearchSecurityException;
4445
import org.elasticsearch.common.ParsingException;
4546
import org.elasticsearch.common.settings.Setting;
4647
import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -81,6 +82,7 @@
8182
import org.elasticsearch.index.mapper.blockloader.docvalues.DenseVectorBlockLoaderProcessor;
8283
import org.elasticsearch.index.mapper.blockloader.docvalues.DenseVectorFromBinaryBlockLoader;
8384
import org.elasticsearch.index.query.SearchExecutionContext;
85+
import org.elasticsearch.rest.RestStatus;
8486
import org.elasticsearch.search.DocValueFormat;
8587
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
8688
import org.elasticsearch.search.lookup.Source;
@@ -1732,7 +1734,7 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map<String, ?
17321734
boolean onDiskRescore = XContentMapValues.nodeBooleanValue(onDiskRescoreNode, false);
17331735

17341736
MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
1735-
return new BBQIVFIndexOptions(clusterSize, visitPercentage, onDiskRescore, rescoreVector);
1737+
return new BBQIVFIndexOptions(clusterSize, visitPercentage, onDiskRescore, rescoreVector, indexVersion);
17361738
}
17371739

17381740
@Override
@@ -2354,21 +2356,37 @@ public boolean validateDimension(int dim, boolean throwOnError) {
23542356

23552357
}
23562358

2357-
static class BBQIVFIndexOptions extends QuantizedIndexOptions {
2359+
public static class BBQIVFIndexOptions extends QuantizedIndexOptions {
23582360
final int clusterSize;
23592361
final double defaultVisitPercentage;
23602362
final boolean onDiskRescore;
2363+
final IndexVersion indexVersionCreated;
23612364

2362-
BBQIVFIndexOptions(int clusterSize, double defaultVisitPercentage, boolean onDiskRescore, RescoreVector rescoreVector) {
2365+
BBQIVFIndexOptions(
2366+
int clusterSize,
2367+
double defaultVisitPercentage,
2368+
boolean onDiskRescore,
2369+
RescoreVector rescoreVector,
2370+
IndexVersion indexVersionCreated
2371+
) {
23632372
super(VectorIndexType.BBQ_DISK, rescoreVector);
23642373
this.clusterSize = clusterSize;
23652374
this.defaultVisitPercentage = defaultVisitPercentage;
23662375
this.onDiskRescore = onDiskRescore;
2376+
this.indexVersionCreated = indexVersionCreated;
23672377
}
23682378

23692379
@Override
23702380
KnnVectorsFormat getVectorsFormat(ElementType elementType) {
23712381
assert elementType == ElementType.FLOAT || elementType == ElementType.BFLOAT16;
2382+
if (indexVersionCreated.onOrAfter(IndexVersions.DISK_BBQ_LICENSE_ENFORCEMENT)) {
2383+
// if we got here, this means we didn't get the plugin installed, so we should throw an exception
2384+
throw new ElasticsearchSecurityException(
2385+
"current license is non-compliant for [{}]",
2386+
RestStatus.FORBIDDEN,
2387+
VectorIndexType.BBQ_DISK.name
2388+
);
2389+
}
23722390
if (Build.current().isSnapshot()) {
23732391
return new ESNextDiskBBQVectorsFormat(
23742392
ESNextDiskBBQVectorsFormat.QuantEncoding.ONE_BIT_4BIT_QUERY,
@@ -2425,6 +2443,18 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
24252443
builder.endObject();
24262444
return builder;
24272445
}
2446+
2447+
public int getClusterSize() {
2448+
return clusterSize;
2449+
}
2450+
2451+
public double getDefaultVisitPercentage() {
2452+
return defaultVisitPercentage;
2453+
}
2454+
2455+
public boolean isOnDiskRescore() {
2456+
return onDiskRescore;
2457+
}
24282458
}
24292459

24302460
public record RescoreVector(float oversample) implements ToXContentObject {

‎server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.apache.lucene.util.BytesRef;
2525
import org.apache.lucene.util.IOConsumer;
2626
import org.apache.lucene.util.VectorUtil;
27-
import org.elasticsearch.Build;
2827
import org.elasticsearch.common.bytes.BytesReference;
2928
import org.elasticsearch.common.util.BigArrays;
3029
import org.elasticsearch.common.xcontent.XContentHelper;
@@ -2064,36 +2063,6 @@ public void testKnnBBQHNSWVectorsFormat() throws IOException {
20642063
assertThat(knnVectorsFormat, hasToString(startsWith(expectedString)));
20652064
}
20662065

2067-
public void testKnnBBQIVFVectorsFormat() throws IOException {
2068-
final int dims = randomIntBetween(64, 4096);
2069-
MapperService mapperService = createMapperService(fieldMapping(b -> {
2070-
b.field("type", "dense_vector");
2071-
b.field("dims", dims);
2072-
b.field("index", true);
2073-
b.field("similarity", "dot_product");
2074-
b.startObject("index_options");
2075-
b.field("type", "bbq_disk");
2076-
b.endObject();
2077-
}));
2078-
CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE);
2079-
Codec codec = codecService.codec("default");
2080-
KnnVectorsFormat knnVectorsFormat;
2081-
if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG) {
2082-
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
2083-
knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
2084-
} else {
2085-
if (codec instanceof CodecService.DeduplicateFieldInfosCodec deduplicateFieldInfosCodec) {
2086-
codec = deduplicateFieldInfosCodec.delegate();
2087-
}
2088-
assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class));
2089-
knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
2090-
}
2091-
String expectedString = Build.current().isSnapshot()
2092-
? "ESNextDiskBBQVectorsFormat(vectorPerCluster=384)"
2093-
: "ES920DiskBBQVectorsFormat(vectorPerCluster=384)";
2094-
assertEquals(expectedString, knnVectorsFormat.toString());
2095-
}
2096-
20972066
public void testInvalidVectorDimensionsBBQ() {
20982067
for (String quantizedFlatFormat : new String[] { "bbq_hnsw", "bbq_flat" }) {
20992068
MapperParsingException e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> {

0 commit comments

Comments
 (0)