Skip to content

Commit 84bf8db

Browse files
kderussokderusso
committed
Support include_defaults for semantic_text index_options
1 parent 813814b commit 84bf8db

File tree

6 files changed

+136
-25
lines changed

6 files changed

+136
-25
lines changed

‎x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java‎

Lines changed: 47 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import org.elasticsearch.inference.InferenceResults;
7070
import org.elasticsearch.inference.MinimalServiceSettings;
7171
import org.elasticsearch.inference.SimilarityMeasure;
72+
import org.elasticsearch.inference.TaskType;
7273
import org.elasticsearch.search.fetch.StoredFieldsSpec;
7374
import org.elasticsearch.search.lookup.Source;
7475
import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
@@ -166,19 +167,9 @@ public static BiConsumer<String, MappingParserContext> validateParserContext(Str
166167
public static class Builder extends FieldMapper.Builder {
167168
private final ModelRegistry modelRegistry;
168169
private final boolean useLegacyFormat;
170+
private final IndexVersion indexVersionCreated;
169171

170-
private final Parameter<String> inferenceId = Parameter.stringParam(
171-
INFERENCE_ID_FIELD,
172-
false,
173-
mapper -> ((SemanticTextFieldType) mapper.fieldType()).inferenceId,
174-
DEFAULT_ELSER_2_INFERENCE_ID
175-
).addValidator(v -> {
176-
if (Strings.isEmpty(v)) {
177-
throw new IllegalArgumentException(
178-
"[" + INFERENCE_ID_FIELD + "] on mapper [" + leafName() + "] of type [" + CONTENT_TYPE + "] must not be empty"
179-
);
180-
}
181-
}).alwaysSerialize();
172+
private final Parameter<String> inferenceId;
182173

183174
private final Parameter<String> searchInferenceId = Parameter.stringParam(
184175
SEARCH_INFERENCE_ID_FIELD,
@@ -203,15 +194,7 @@ public static class Builder extends FieldMapper.Builder {
203194
Objects::toString
204195
).acceptsNull().setMergeValidator(SemanticTextFieldMapper::canMergeModelSettings);
205196

206-
private final Parameter<SemanticTextIndexOptions> indexOptions = new Parameter<>(
207-
INDEX_OPTIONS_FIELD,
208-
true,
209-
() -> null,
210-
(n, c, o) -> parseIndexOptionsFromMap(n, o, c.indexVersionCreated()),
211-
mapper -> ((SemanticTextFieldType) mapper.fieldType()).indexOptions,
212-
XContentBuilder::field,
213-
Objects::toString
214-
).acceptsNull();
197+
private final Parameter<SemanticTextIndexOptions> indexOptions;
215198

216199
@SuppressWarnings("unchecked")
217200
private final Parameter<ChunkingSettings> chunkingSettings = new Parameter<>(
@@ -246,8 +229,34 @@ public Builder(
246229
ModelRegistry modelRegistry
247230
) {
248231
super(name);
232+
233+
this.indexVersionCreated = indexSettings.getIndexVersionCreated();
249234
this.modelRegistry = modelRegistry;
250235
this.useLegacyFormat = InferenceMetadataFieldsMapper.isEnabled(indexSettings.getSettings()) == false;
236+
237+
this.inferenceId = Parameter.stringParam(
238+
INFERENCE_ID_FIELD,
239+
false,
240+
mapper -> ((SemanticTextFieldType) mapper.fieldType()).inferenceId,
241+
DEFAULT_ELSER_2_INFERENCE_ID
242+
).addValidator(v -> {
243+
if (Strings.isEmpty(v)) {
244+
throw new IllegalArgumentException(
245+
"[" + INFERENCE_ID_FIELD + "] on mapper [" + leafName() + "] of type [" + CONTENT_TYPE + "] must not be empty"
246+
);
247+
}
248+
}).alwaysSerialize();
249+
250+
this.indexOptions = new Parameter<>(
251+
INDEX_OPTIONS_FIELD,
252+
true,
253+
() -> defaultIndexOptions(indexVersionCreated, modelRegistry.getMinimalServiceSettings(inferenceId.get())),
254+
(n, c, o) -> parseIndexOptionsFromMap(n, o, c.indexVersionCreated()),
255+
mapper -> ((SemanticTextFieldType) mapper.fieldType()).indexOptions,
256+
XContentBuilder::field,
257+
Objects::toString
258+
).acceptsNull(); // .setSerializerCheck(this::indexOptionsSerializerCheck);
259+
251260
this.inferenceFieldBuilder = c -> {
252261
// Resolve the model setting from the registry if it has not been set yet.
253262
var resolvedModelSettings = modelSettings.get() != null ? modelSettings.get() : getResolvedModelSettings(c, false);
@@ -263,6 +272,10 @@ public Builder(
263272
};
264273
}
265274

275+
// private boolean indexOptionsSerializerCheck(boolean includeDefaults, boolean isConfigured, SemanticTextIndexOptions value) {
276+
// return includeDefaults || Objects.equals(value, defaultIndexOptions(indexVersionCreated, modelSettings.get())) == false;
277+
// }
278+
266279
public Builder setInferenceId(String id) {
267280
this.inferenceId.setValue(id);
268281
return this;
@@ -1235,6 +1248,19 @@ static SemanticTextIndexOptions defaultBbqHnswSemanticTextIndexOptions() {
12351248
);
12361249
}
12371250

1251+
static SemanticTextIndexOptions defaultIndexOptions(IndexVersion indexVersionCreated, MinimalServiceSettings modelSettings) {
1252+
1253+
if (modelSettings == null) {
1254+
return null;
1255+
}
1256+
1257+
if (modelSettings.taskType() == TaskType.TEXT_EMBEDDING && indexVersionDefaultsToBbqHnsw(indexVersionCreated)) {
1258+
return defaultBbqHnswSemanticTextIndexOptions();
1259+
}
1260+
1261+
return null;
1262+
}
1263+
12381264
private static boolean canMergeModelSettings(MinimalServiceSettings previous, MinimalServiceSettings current, Conflicts conflicts) {
12391265
if (previous != null && current != null && previous.canMergeWith(current)) {
12401266
return true;

‎x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/LocalStateInferencePlugin.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,24 @@
1818
import org.elasticsearch.xpack.core.ssl.SSLService;
1919
import org.elasticsearch.xpack.inference.mock.TestDenseInferenceServiceExtension;
2020
import org.elasticsearch.xpack.inference.mock.TestSparseInferenceServiceExtension;
21+
import org.elasticsearch.xpack.inference.registry.ModelRegistry;
2122

2223
import java.nio.file.Path;
2324
import java.util.Collection;
2425
import java.util.List;
2526
import java.util.Map;
27+
import java.util.function.Supplier;
2628

2729
import static java.util.stream.Collectors.toList;
30+
import static org.mockito.Mockito.mock;
2831

2932
public class LocalStateInferencePlugin extends LocalStateCompositeXPackPlugin {
3033
private final InferencePlugin inferencePlugin;
3134

3235
public LocalStateInferencePlugin(final Settings settings, final Path configPath) throws Exception {
3336
super(settings, configPath);
3437
LocalStateInferencePlugin thisVar = this;
38+
3539
this.inferencePlugin = new InferencePlugin(settings) {
3640
@Override
3741
protected SSLService getSslService() {
@@ -50,7 +54,13 @@ public List<InferenceServiceExtension.Factory> getInferenceServiceFactories() {
5054
TestDenseInferenceServiceExtension.TestInferenceService::new
5155
);
5256
}
57+
58+
@Override
59+
protected Supplier<ModelRegistry> getModelRegistry() {
60+
return () -> mock(ModelRegistry.class);
61+
}
5362
};
63+
5464
plugins.add(inferencePlugin);
5565
}
5666

‎x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/Utils.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import java.util.HashMap;
4444
import java.util.List;
4545
import java.util.Map;
46+
import java.util.function.Supplier;
4647
import java.util.stream.Collectors;
4748
import java.util.stream.Stream;
4849

@@ -239,4 +240,23 @@ public static void assertJsonEquals(String actual, String expected) throws IOExc
239240
public static <K, V> Map<K, V> modifiableMap(Map<K, V> aMap) {
240241
return new HashMap<>(aMap);
241242
}
243+
244+
public static class InferencePluginWithModelRegistry extends InferencePlugin {
245+
246+
private final ModelRegistry modelRegistry;
247+
248+
public InferencePluginWithModelRegistry(Settings settings, ModelRegistry modelRegistry) {
249+
super(settings);
250+
this.modelRegistry = modelRegistry;
251+
}
252+
253+
public InferencePluginWithModelRegistry(Settings settings) {
254+
this(settings, mock(ModelRegistry.class));
255+
}
256+
257+
@Override
258+
protected Supplier<ModelRegistry> getModelRegistry() {
259+
return () -> modelRegistry;
260+
}
261+
}
242262
}

‎x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/highlight/SemanticTextHighlighterTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
5454
import org.elasticsearch.xcontent.XContentType;
5555
import org.elasticsearch.xpack.core.ml.search.SparseVectorQueryBuilder;
56-
import org.elasticsearch.xpack.inference.InferencePlugin;
56+
import org.elasticsearch.xpack.inference.Utils;
5757
import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
5858
import org.mockito.Mockito;
5959

@@ -90,7 +90,7 @@ public static Iterable<Object[]> parameters() throws Exception {
9090

9191
@Override
9292
protected Collection<? extends Plugin> getPlugins() {
93-
return List.of(new InferencePlugin(Settings.EMPTY));
93+
return List.of(new Utils.InferencePluginWithModelRegistry(Settings.EMPTY));
9494
}
9595

9696
@SuppressWarnings("unchecked")

‎x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
import org.elasticsearch.xcontent.XContentBuilder;
3737
import org.elasticsearch.xcontent.XContentType;
3838
import org.elasticsearch.xcontent.json.JsonXContent;
39-
import org.elasticsearch.xpack.inference.InferencePlugin;
39+
import org.elasticsearch.xpack.inference.Utils;
4040
import org.elasticsearch.xpack.inference.model.TestModel;
4141

4242
import java.io.IOException;
@@ -72,7 +72,7 @@ public static Iterable<Object[]> parameters() throws Exception {
7272

7373
@Override
7474
protected List<MapperPlugin> extraMappers() {
75-
return List.of(new InferencePlugin(Settings.EMPTY));
75+
return List.of(new Utils.InferencePluginWithModelRegistry(Settings.EMPTY));
7676
}
7777

7878
@Override

‎x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -833,3 +833,58 @@ setup:
833833
type: int8_flat
834834

835835
- match: { status: 400 }
836+
837+
838+
---
839+
"Users can see defaults for index options":
840+
- requires:
841+
cluster_features: "semantic_text.index_options"
842+
reason: Index options introduced in 8.19.0
843+
844+
- do:
845+
indices.create:
846+
index: test-index-options-dense
847+
body:
848+
settings:
849+
index:
850+
mapping:
851+
semantic_text:
852+
use_legacy_format: false
853+
mappings:
854+
properties:
855+
semantic_field:
856+
type: semantic_text
857+
inference_id: dense-inference-id-compatible-with-bbq
858+
859+
- do:
860+
indices.get_mapping:
861+
index: test-index-options-dense
862+
include_defaults: true
863+
864+
- match: { "test-index-options-dense.mappings.properties.semantic_field.index_options.dense_vector.type": "bbq_hnsw" }
865+
- match: { "test-index-options-dense.mappings.properties.semantic_field.index_options.dense_vector.m": 20 }
866+
- match: { "test-index-options-dense.mappings.properties.semantic_field.index_options.dense_vector.ef_construction": 100 }
867+
- match: { "test-index-options-dense.mappings.properties.semantic_field.index_options.dense_vector.confidence_interval": 1.0 }
868+
869+
870+
- do:
871+
indices.create:
872+
index: test-index-options-sparse
873+
body:
874+
settings:
875+
index:
876+
mapping:
877+
semantic_text:
878+
use_legacy_format: false
879+
mappings:
880+
properties:
881+
semantic_field:
882+
type: semantic_text
883+
inference_id: sparse-inference-id
884+
885+
- do:
886+
indices.get_mapping:
887+
index: test-index-options-sparse
888+
include_defaults: true
889+
890+
- match: { "test-index-options-dense.mappings.properties.semantic_field.index_options": null }

0 commit comments

Comments
 (0)