Skip to content

Commit 6a8bc77

Browse files
markjhoymarkjhoy
committed
some cleanups; still needs a few more tests
1 parent a687eb8 commit 6a8bc77

File tree

4 files changed

+235
-122
lines changed

4 files changed

+235
-122
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java‎

Lines changed: 11 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ protected Parameter<?>[] getParameters() {
129129
public SparseVectorFieldMapper build(MapperBuilderContext context) {
130130
SparseVectorIndexOptions builderIndexOptions = indexOptions.getValue();
131131
if (builderIndexOptions == null) {
132-
builderIndexOptions = getDefaultIndexOptions(indexVersionCreated);
132+
builderIndexOptions = SparseVectorIndexOptions.getDefaultIndexOptions(indexVersionCreated);
133133
}
134134

135135
final boolean syntheticVectorFinal = context.isSourceSynthetic() == false && isSyntheticVector;
@@ -148,13 +148,6 @@ public SparseVectorFieldMapper build(MapperBuilderContext context) {
148148
);
149149
}
150150

151-
private SparseVectorIndexOptions getDefaultIndexOptions(IndexVersion indexVersion) {
152-
return (indexVersion.onOrAfter(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION)
153-
|| indexVersion.between(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION_8_X, IndexVersions.UPGRADE_TO_LUCENE_10_0_0))
154-
? SparseVectorIndexOptions.DEFAULT_PRUNING_INDEX_OPTIONS
155-
: null;
156-
}
157-
158151
private boolean indexOptionsSerializerCheck(boolean includeDefaults, boolean isConfigured, SparseVectorIndexOptions value) {
159152
return includeDefaults || (SparseVectorIndexOptions.isDefaultOptions(value, indexVersionCreated) == false);
160153
}
@@ -608,24 +601,18 @@ public static SparseVectorIndexOptions getDefaultIndexOptions(IndexVersion index
608601
}
609602

610603
public static SparseVectorIndexOptions parseFromMap(Map<String, Object> map) {
611-
Boolean prune = null;
612-
TokenPruningConfig pruningConfig = null;
613-
614-
for (Map.Entry<String, Object> entry : map.entrySet()) {
615-
if (entry.getKey().equals(PRUNE_FIELD_NAME.getPreferredName())) {
616-
prune = XContentMapValues.nodeBooleanValue(entry.getValue());
617-
} else if (entry.getKey().equals(PRUNING_CONFIG_FIELD_NAME.getPreferredName())) {
618-
Map<String, Object> pruningConfigMap = XContentMapValues.nodeMapValue(
619-
entry.getValue(),
620-
PRUNING_CONFIG_FIELD_NAME.getPreferredName()
621-
);
622-
pruningConfig = TokenPruningConfig.parseFromMap(pruningConfigMap);
623-
} else {
624-
throw new IllegalArgumentException("Unsupported index option field for sparse_vector: " + entry.getKey());
625-
}
604+
if (map == null) {
605+
return null;
626606
}
627607

628-
return new SparseVectorIndexOptions(prune, pruningConfig);
608+
try {
609+
XContentParser parser =
610+
new MapXContentParser(NamedXContentRegistry.EMPTY, DeprecationHandler.IGNORE_DEPRECATIONS, map, XContentType.JSON);
611+
612+
return INDEX_OPTIONS_PARSER.parse(parser, null);
613+
} catch (IOException ioEx) {
614+
throw new UncheckedIOException(ioEx);
615+
}
629616
}
630617

631618
public Boolean getPrune() {

‎server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -42,23 +42,21 @@ public void testIsNotAggregatable() {
4242
}
4343

4444
public static SparseVectorFieldMapper.SparseVectorIndexOptions randomSparseVectorIndexOptions() {
45-
if (randomBoolean() == false) {
46-
return new SparseVectorFieldMapper.SparseVectorIndexOptions(false, null);
47-
}
48-
49-
return new SparseVectorFieldMapper.SparseVectorIndexOptions(
50-
true,
51-
new TokenPruningConfig(randomFloatBetween(1.0f, 100.0f, true), randomFloatBetween(0.0f, 1.0f, true), randomBoolean())
52-
);
53-
}
54-
55-
public static SparseVectorFieldMapper.SparseVectorIndexOptions randomSparseVectorIndexOptionsAll() {
5645
return randomFrom(
5746
new SparseVectorFieldMapper.SparseVectorIndexOptions(null, null),
5847
new SparseVectorFieldMapper.SparseVectorIndexOptions(false, null),
48+
new SparseVectorFieldMapper.SparseVectorIndexOptions(true, null),
49+
new SparseVectorFieldMapper.SparseVectorIndexOptions(
50+
true,
51+
new TokenPruningConfig(randomFloatBetween(1.0f, 100.0f, true), randomFloatBetween(0.0f, 1.0f, true), randomBoolean())
52+
),
53+
new SparseVectorFieldMapper.SparseVectorIndexOptions(
54+
true,
55+
new TokenPruningConfig(randomFloatBetween(1.0f, 100.0f, true), randomFloatBetween(0.0f, 1.0f, true), randomBoolean())
56+
),
5957
new SparseVectorFieldMapper.SparseVectorIndexOptions(
6058
true,
61-
new TokenPruningConfig(randomFloatBetween(1.0f, 100.0f, true), randomFloatBetween(0.0f, 1.0f, true), false)
59+
new TokenPruningConfig(randomFloatBetween(1.0f, 100.0f, true), randomFloatBetween(0.0f, 1.0f, true), randomBoolean())
6260
)
6361
);
6462
}

‎x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java‎

Lines changed: 41 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -452,17 +452,25 @@ private void validateIndexOptions(SemanticTextIndexOptions indexOptions, String
452452
return;
453453
}
454454

455-
if (indexOptions.type() == SemanticTextIndexOptions.SupportedIndexOptions.SPARSE_VECTOR) {
456-
// sparse vector index options are validated in the ctor when created
457-
return;
458-
}
459-
460455
if (modelSettings == null) {
461456
throw new IllegalArgumentException(
462457
"Model settings must be set to validate index options for inference ID [" + inferenceId + "]"
463458
);
464459
}
465460

461+
if (indexOptions.type() == SemanticTextIndexOptions.SupportedIndexOptions.SPARSE_VECTOR) {
462+
if (modelSettings.taskType() != SPARSE_EMBEDDING) {
463+
throw new IllegalArgumentException(
464+
"Invalid task type for index options, required ["
465+
+ SPARSE_EMBEDDING
466+
+ "] but was ["
467+
+ modelSettings.taskType()
468+
+ "]"
469+
);
470+
}
471+
return;
472+
}
473+
466474
if (indexOptions.type() == SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR) {
467475
if (modelSettings.taskType() != TEXT_EMBEDDING) {
468476
throw new IllegalArgumentException(
@@ -1179,18 +1187,7 @@ private static Mapper.Builder createEmbeddingsField(
11791187
false
11801188
).setStored(useLegacyFormat == false);
11811189

1182-
if (indexOptions != null) {
1183-
SparseVectorFieldMapper.SparseVectorIndexOptions sparseVectorIndexOptions =
1184-
(SparseVectorFieldMapper.SparseVectorIndexOptions) indexOptions.indexOptions();
1185-
1186-
sparseVectorMapperBuilder.setIndexOptions(sparseVectorIndexOptions);
1187-
} else {
1188-
SparseVectorFieldMapper.SparseVectorIndexOptions defaultIndexOptions = SparseVectorFieldMapper.SparseVectorIndexOptions
1189-
.getDefaultIndexOptions(indexVersionCreated);
1190-
if (defaultIndexOptions != null) {
1191-
sparseVectorMapperBuilder.setIndexOptions(defaultIndexOptions);
1192-
}
1193-
}
1190+
configureSparseVectorMapperBuilder(indexVersionCreated, sparseVectorMapperBuilder, indexOptions);
11941191

11951192
yield sparseVectorMapperBuilder;
11961193
}
@@ -1201,15 +1198,34 @@ private static Mapper.Builder createEmbeddingsField(
12011198
false
12021199
);
12031200

1204-
setDenseVectorMapperBuilderForEmbeddings(indexVersionCreated, denseVectorMapperBuilder, modelSettings, indexOptions);
1201+
configureDenseVectorMapperBuilder(indexVersionCreated, denseVectorMapperBuilder, modelSettings, indexOptions);
12051202

12061203
yield denseVectorMapperBuilder;
12071204
}
12081205
default -> throw new IllegalArgumentException("Invalid task_type in model_settings [" + modelSettings.taskType().name() + "]");
12091206
};
12101207
}
12111208

1212-
private static void setDenseVectorMapperBuilderForEmbeddings(
1209+
private static void configureSparseVectorMapperBuilder(
1210+
IndexVersion indexVersionCreated,
1211+
SparseVectorFieldMapper.Builder sparseVectorMapperBuilder,
1212+
SemanticTextIndexOptions indexOptions
1213+
) {
1214+
if (indexOptions != null) {
1215+
SparseVectorFieldMapper.SparseVectorIndexOptions sparseVectorIndexOptions =
1216+
(SparseVectorFieldMapper.SparseVectorIndexOptions) indexOptions.indexOptions();
1217+
1218+
sparseVectorMapperBuilder.setIndexOptions(sparseVectorIndexOptions);
1219+
} else {
1220+
SparseVectorFieldMapper.SparseVectorIndexOptions defaultIndexOptions = SparseVectorFieldMapper.SparseVectorIndexOptions
1221+
.getDefaultIndexOptions(indexVersionCreated);
1222+
if (defaultIndexOptions != null) {
1223+
sparseVectorMapperBuilder.setIndexOptions(defaultIndexOptions);
1224+
}
1225+
}
1226+
}
1227+
1228+
private static void configureDenseVectorMapperBuilder(
12131229
IndexVersion indexVersionCreated,
12141230
DenseVectorFieldMapper.Builder denseVectorMapperBuilder,
12151231
MinimalServiceSettings modelSettings,
@@ -1295,10 +1311,12 @@ static SemanticTextIndexOptions defaultIndexOptions(IndexVersion indexVersionCre
12951311
}
12961312

12971313
if (modelSettings.taskType() == SPARSE_EMBEDDING) {
1298-
return new SemanticTextIndexOptions(
1299-
SemanticTextIndexOptions.SupportedIndexOptions.SPARSE_VECTOR,
1300-
SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersionCreated)
1301-
);
1314+
SparseVectorFieldMapper.SparseVectorIndexOptions sparseVectorIndexOptions = SparseVectorFieldMapper.SparseVectorIndexOptions
1315+
.getDefaultIndexOptions(indexVersionCreated);
1316+
1317+
return sparseVectorIndexOptions == null
1318+
? null
1319+
: new SemanticTextIndexOptions(SemanticTextIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, sparseVectorIndexOptions);
13021320
}
13031321

13041322
return null;

0 commit comments

Comments
 (0)