Skip to content

Commit 30fb01b

Browse files
thecoopthecoop
committed
Maintain confidence_interval option for now
1 parent f5cd1d3 commit 30fb01b

File tree

4 files changed

+78
-23
lines changed

4 files changed

+78
-23
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java‎

Lines changed: 58 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,7 @@ private DenseVectorIndexOptions defaultIndexOptions(boolean defaultInt8Hnsw, boo
398398
return new Int8HnswIndexOptions(
399399
Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
400400
Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
401+
null,
401402
false,
402403
null
403404
);
@@ -1349,20 +1350,23 @@ public boolean supportsDimension(int dims) {
13491350
public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOptionsMap, IndexVersion indexVersion) {
13501351
Object mNode = indexOptionsMap.remove("m");
13511352
Object efConstructionNode = indexOptionsMap.remove("ef_construction");
1352-
// BWC option, ignore
1353-
indexOptionsMap.remove("confidence_interval");
1353+
Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval");
13541354
Object onDiskRescoreNode = indexOptionsMap.remove("on_disk_rescore");
13551355

13561356
int m = XContentMapValues.nodeIntegerValue(mNode, Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN);
13571357
int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH);
13581358
boolean onDiskRescore = XContentMapValues.nodeBooleanValue(onDiskRescoreNode, false);
1359+
Float confidenceInterval = null;
1360+
if (confidenceIntervalNode != null) {
1361+
confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode);
1362+
}
13591363
RescoreVector rescoreVector = null;
13601364
if (hasRescoreIndexVersion(indexVersion)) {
13611365
rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
13621366
}
13631367
MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
13641368

1365-
return new Int8HnswIndexOptions(m, efConstruction, onDiskRescore, rescoreVector);
1369+
return new Int8HnswIndexOptions(m, efConstruction, confidenceInterval, onDiskRescore, rescoreVector);
13661370
}
13671371

13681372
@Override
@@ -1379,20 +1383,23 @@ public boolean supportsDimension(int dims) {
13791383
public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOptionsMap, IndexVersion indexVersion) {
13801384
Object mNode = indexOptionsMap.remove("m");
13811385
Object efConstructionNode = indexOptionsMap.remove("ef_construction");
1382-
// BWC option, ignore
1383-
indexOptionsMap.remove("confidence_interval");
1386+
Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval");
13841387
Object onDiskRescoreNode = indexOptionsMap.remove("on_disk_rescore");
13851388

13861389
int m = XContentMapValues.nodeIntegerValue(mNode, Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN);
13871390
int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH);
1391+
Float confidenceInterval = null;
1392+
if (confidenceIntervalNode != null) {
1393+
confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode);
1394+
}
13881395
boolean onDiskRescore = XContentMapValues.nodeBooleanValue(onDiskRescoreNode, false);
13891396
RescoreVector rescoreVector = null;
13901397
if (hasRescoreIndexVersion(indexVersion)) {
13911398
rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
13921399
}
13931400
MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
13941401

1395-
return new Int4HnswIndexOptions(m, efConstruction, onDiskRescore, rescoreVector);
1402+
return new Int4HnswIndexOptions(m, efConstruction, confidenceInterval, onDiskRescore, rescoreVector);
13961403
}
13971404

13981405
@Override
@@ -1425,14 +1432,17 @@ public boolean supportsDimension(int dims) {
14251432
INT8_FLAT("int8_flat", true) {
14261433
@Override
14271434
public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOptionsMap, IndexVersion indexVersion) {
1428-
// BWC option, ignore
1429-
indexOptionsMap.remove("confidence_interval");
1435+
Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval");
1436+
Float confidenceInterval = null;
1437+
if (confidenceIntervalNode != null) {
1438+
confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode);
1439+
}
14301440
RescoreVector rescoreVector = null;
14311441
if (hasRescoreIndexVersion(indexVersion)) {
14321442
rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
14331443
}
14341444
MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
1435-
return new Int8FlatIndexOptions(rescoreVector);
1445+
return new Int8FlatIndexOptions(confidenceInterval, rescoreVector);
14361446
}
14371447

14381448
@Override
@@ -1448,14 +1458,17 @@ public boolean supportsDimension(int dims) {
14481458
INT4_FLAT("int4_flat", true) {
14491459
@Override
14501460
public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOptionsMap, IndexVersion indexVersion) {
1451-
// BWC option, ignore
1452-
indexOptionsMap.remove("confidence_interval");
1461+
Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval");
1462+
Float confidenceInterval = null;
1463+
if (confidenceIntervalNode != null) {
1464+
confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode);
1465+
}
14531466
RescoreVector rescoreVector = null;
14541467
if (hasRescoreIndexVersion(indexVersion)) {
14551468
rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
14561469
}
14571470
MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
1458-
return new Int4FlatIndexOptions(rescoreVector);
1471+
return new Int4FlatIndexOptions(confidenceInterval, rescoreVector);
14591472
}
14601473

14611474
@Override
@@ -1619,15 +1632,20 @@ public String toString() {
16191632
}
16201633

16211634
static class Int8FlatIndexOptions extends QuantizedIndexOptions {
1635+
private final Float confidenceInterval;
16221636

1623-
Int8FlatIndexOptions(RescoreVector rescoreVector) {
1637+
Int8FlatIndexOptions(Float confidenceInterval, RescoreVector rescoreVector) {
16241638
super(VectorIndexType.INT8_FLAT, rescoreVector);
1639+
this.confidenceInterval = confidenceInterval;
16251640
}
16261641

16271642
@Override
16281643
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
16291644
builder.startObject();
16301645
builder.field("type", type);
1646+
if (confidenceInterval != null) {
1647+
builder.field("confidence_interval", confidenceInterval);
1648+
}
16311649
if (rescoreVector != null) {
16321650
rescoreVector.toXContent(builder, params);
16331651
}
@@ -1716,12 +1734,20 @@ public boolean isFlat() {
17161734
public static class Int4HnswIndexOptions extends QuantizedIndexOptions {
17171735
private final int m;
17181736
private final int efConstruction;
1737+
private final float confidenceInterval;
17191738
private final boolean onDiskRescore;
17201739

1721-
public Int4HnswIndexOptions(int m, int efConstruction, boolean onDiskRescore, RescoreVector rescoreVector) {
1740+
public Int4HnswIndexOptions(
1741+
int m,
1742+
int efConstruction,
1743+
Float confidenceInterval,
1744+
boolean onDiskRescore,
1745+
RescoreVector rescoreVector
1746+
) {
17221747
super(VectorIndexType.INT4_HNSW, rescoreVector);
17231748
this.m = m;
17241749
this.efConstruction = efConstruction;
1750+
this.confidenceInterval = confidenceInterval == null ? 0f : confidenceInterval;
17251751
this.onDiskRescore = onDiskRescore;
17261752
}
17271753

@@ -1743,6 +1769,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
17431769
builder.field("type", type);
17441770
builder.field("m", m);
17451771
builder.field("ef_construction", efConstruction);
1772+
builder.field("confidence_interval", confidenceInterval);
17461773
if (onDiskRescore) {
17471774
builder.field("on_disk_rescore", true);
17481775
}
@@ -1802,8 +1829,11 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
18021829
}
18031830

18041831
static class Int4FlatIndexOptions extends QuantizedIndexOptions {
1805-
Int4FlatIndexOptions(RescoreVector rescoreVector) {
1832+
private final float confidenceInterval;
1833+
1834+
Int4FlatIndexOptions(Float confidenceInterval, RescoreVector rescoreVector) {
18061835
super(VectorIndexType.INT4_FLAT, rescoreVector);
1836+
this.confidenceInterval = confidenceInterval == null ? 0f : confidenceInterval;
18071837
}
18081838

18091839
@Override
@@ -1816,6 +1846,7 @@ public KnnVectorsFormat getVectorsFormat(ElementType elementType) {
18161846
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
18171847
builder.startObject();
18181848
builder.field("type", type);
1849+
builder.field("confidence_interval", confidenceInterval);
18191850
if (rescoreVector != null) {
18201851
rescoreVector.toXContent(builder, params);
18211852
}
@@ -1862,12 +1893,20 @@ public boolean updatableTo(DenseVectorIndexOptions update) {
18621893
public static class Int8HnswIndexOptions extends QuantizedIndexOptions {
18631894
private final int m;
18641895
private final int efConstruction;
1896+
private final Float confidenceInterval;
18651897
private final boolean onDiskRescore;
18661898

1867-
public Int8HnswIndexOptions(int m, int efConstruction, boolean onDiskRescore, RescoreVector rescoreVector) {
1899+
public Int8HnswIndexOptions(
1900+
int m,
1901+
int efConstruction,
1902+
Float confidenceInterval,
1903+
boolean onDiskRescore,
1904+
RescoreVector rescoreVector
1905+
) {
18681906
super(VectorIndexType.INT8_HNSW, rescoreVector);
18691907
this.m = m;
18701908
this.efConstruction = efConstruction;
1909+
this.confidenceInterval = confidenceInterval;
18711910
this.onDiskRescore = onDiskRescore;
18721911
}
18731912

@@ -1889,6 +1928,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
18891928
builder.field("type", type);
18901929
builder.field("m", m);
18911930
builder.field("ef_construction", efConstruction);
1931+
if (confidenceInterval != null) {
1932+
builder.field("confidence_interval", confidenceInterval);
1933+
}
18921934
if (onDiskRescore) {
18931935
builder.field("on_disk_rescore", true);
18941936
}

‎server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,14 @@ private static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptions
7474
public static DenseVectorFieldMapper.DenseVectorIndexOptions randomFlatIndexOptions() {
7575
return randomFrom(
7676
new DenseVectorFieldMapper.FlatIndexOptions(),
77-
new DenseVectorFieldMapper.Int8FlatIndexOptions(randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())),
78-
new DenseVectorFieldMapper.Int4FlatIndexOptions(randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()))
77+
new DenseVectorFieldMapper.Int8FlatIndexOptions(
78+
null,
79+
randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
80+
),
81+
new DenseVectorFieldMapper.Int4FlatIndexOptions(
82+
null,
83+
randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
84+
)
7985
);
8086
}
8187

@@ -85,6 +91,7 @@ public static DenseVectorFieldMapper.DenseVectorIndexOptions randomGpuSupportedI
8591
new DenseVectorFieldMapper.Int8HnswIndexOptions(
8692
randomIntBetween(1, 100),
8793
randomIntBetween(1, 3199),
94+
null,
8895
randomBoolean(),
8996
randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
9097
)
@@ -107,20 +114,24 @@ public static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsA
107114
new DenseVectorFieldMapper.Int8HnswIndexOptions(
108115
randomIntBetween(1, 100),
109116
randomIntBetween(1, 10_000),
117+
null,
110118
randomBoolean(),
111119
randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
112120
),
113121
new DenseVectorFieldMapper.Int4HnswIndexOptions(
114122
randomIntBetween(1, 100),
115123
randomIntBetween(1, 10_000),
124+
null,
116125
randomBoolean(),
117126
randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
118127
),
119128
new DenseVectorFieldMapper.FlatIndexOptions(),
120129
new DenseVectorFieldMapper.Int8FlatIndexOptions(
130+
null,
121131
randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
122132
),
123133
new DenseVectorFieldMapper.Int4FlatIndexOptions(
134+
null,
124135
randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector())
125136
),
126137
new DenseVectorFieldMapper.BBQHnswIndexOptions(
@@ -158,12 +169,14 @@ private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsHnswQua
158169
new DenseVectorFieldMapper.Int8HnswIndexOptions(
159170
randomIntBetween(1, 100),
160171
randomIntBetween(1, 10_000),
172+
null,
161173
randomBoolean(),
162174
rescoreVector
163175
),
164176
new DenseVectorFieldMapper.Int4HnswIndexOptions(
165177
randomIntBetween(1, 100),
166178
randomIntBetween(1, 10_000),
179+
null,
167180
randomBoolean(),
168181
rescoreVector
169182
),

‎x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextIndexOptionsIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ public void testValidateIndexOptionsWithBasicLicense() throws Exception {
118118

119119
IndexOptions indexOptions = new DenseVectorFieldMapper.Int8HnswIndexOptions(
120120
randomIntBetween(1, 100),
121-
randomIntBetween(1, 10_000),
121+
randomIntBetween(1, 10_000),null,
122122
false,
123123
null
124124
);

‎x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1850,7 +1850,7 @@ private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultDenseVector
18501850
// These are the default index options for dense_vector fields, and used for semantic_text fields incompatible with BBQ.
18511851
int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
18521852
int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
1853-
return new DenseVectorFieldMapper.Int8HnswIndexOptions(m, efConstruction, false, null);
1853+
return new DenseVectorFieldMapper.Int8HnswIndexOptions(m, efConstruction,null, false, null);
18541854
}
18551855

18561856
private static SemanticTextIndexOptions defaultDenseVectorSemanticIndexOptions() {
@@ -1950,7 +1950,7 @@ public void testDefaultIndexOptions() throws IOException {
19501950
null,
19511951
new SemanticTextIndexOptions(
19521952
SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR,
1953-
new DenseVectorFieldMapper.Int4HnswIndexOptions(25, 100, false, null)
1953+
new DenseVectorFieldMapper.Int4HnswIndexOptions(25, 100, null, false, null)
19541954
)
19551955
);
19561956

@@ -2070,7 +2070,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException {
20702070
null,
20712071
new SemanticTextIndexOptions(
20722072
SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR,
2073-
new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, true, null)
2073+
new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, null, true, null)
20742074
)
20752075
);
20762076

@@ -2097,7 +2097,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException {
20972097
null,
20982098
new SemanticTextIndexOptions(
20992099
SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR,
2100-
new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, false, null)
2100+
new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, null, false, null)
21012101
)
21022102
);
21032103

0 commit comments

Comments
 (0)