Serialize default index_options for semantic_text fields

Author: kderusso

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java

@@ -69,6 +69,7 @@
 import org.elasticsearch.inference.InferenceResults;
 import org.elasticsearch.inference.MinimalServiceSettings;
 import org.elasticsearch.inference.SimilarityMeasure;
+import org.elasticsearch.inference.TaskType;
 import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.vectors.KnnVectorQueryBuilder;
@@ -166,6 +167,7 @@ public static BiConsumer<String, MappingParserContext> validateParserContext(Str
     public static class Builder extends FieldMapper.Builder {
         private final ModelRegistry modelRegistry;
         private final boolean useLegacyFormat;
+        private final IndexVersion indexVersionCreated;
 
         private final Parameter<String> inferenceId = Parameter.stringParam(
             INFERENCE_ID_FIELD,
@@ -203,15 +205,7 @@ public static class Builder extends FieldMapper.Builder {
             Objects::toString
         ).acceptsNull().setMergeValidator(SemanticTextFieldMapper::canMergeModelSettings);
 
-        private final Parameter<SemanticTextIndexOptions> indexOptions = new Parameter<>(
-            INDEX_OPTIONS_FIELD,
-            true,
-            () -> null,
-            (n, c, o) -> parseIndexOptionsFromMap(n, o, c.indexVersionCreated()),
-            mapper -> ((SemanticTextFieldType) mapper.fieldType()).indexOptions,
-            XContentBuilder::field,
-            Objects::toString
-        ).acceptsNull();
+        private final Parameter<SemanticTextIndexOptions> indexOptions;
 
         @SuppressWarnings("unchecked")
         private final Parameter<ChunkingSettings> chunkingSettings = new Parameter<>(
@@ -248,6 +242,18 @@ public Builder(
             super(name);
             this.modelRegistry = modelRegistry;
             this.useLegacyFormat = InferenceMetadataFieldsMapper.isEnabled(indexSettings.getSettings()) == false;
+            this.indexVersionCreated = indexSettings.getIndexVersionCreated();
+
+            indexOptions = new Parameter<>(
+                INDEX_OPTIONS_FIELD,
+                true,
+                () -> null,
+                (n, c, o) -> parseIndexOptionsFromMap(n, o, c.indexVersionCreated()),
+                mapper -> ((SemanticTextFieldType) mapper.fieldType()).indexOptions,
+                XContentBuilder::field,
+                Objects::toString
+            ).acceptsNull().setSerializerCheck(this::indexOptionsSerializerCheck);
+
             this.inferenceFieldBuilder = c -> {
                 // Resolve the model setting from the registry if it has not been set yet.
                 var resolvedModelSettings = modelSettings.get() != null ? modelSettings.get() : getResolvedModelSettings(c, false);
@@ -263,6 +269,10 @@ public Builder(
             };
         }
 
+        private boolean indexOptionsSerializerCheck(boolean includeDefaults, boolean isConfigured, SemanticTextIndexOptions value) {
+            return includeDefaults || Objects.equals(defaultIndexOptions(indexVersionCreated, modelSettings.get()), value) == false;
+        }
+
         public Builder setInferenceId(String id) {
             this.inferenceId.setValue(id);
             return this;
@@ -365,8 +375,13 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) {
                 validateServiceSettings(modelSettings.get(), resolvedModelSettings);
             }
 
-            if (context.getMergeReason() != MapperService.MergeReason.MAPPING_RECOVERY && indexOptions.get() != null) {
-                validateIndexOptions(indexOptions.get(), inferenceId.getValue(), resolvedModelSettings);
+            SemanticTextIndexOptions builderIndexOptions = indexOptions.get();
+            if (context.getMergeReason() != MapperService.MergeReason.MAPPING_RECOVERY && builderIndexOptions != null) {
+                validateIndexOptions(builderIndexOptions, inferenceId.getValue(), resolvedModelSettings);
+            }
+            if (builderIndexOptions == null) {
+                // If no index options are set, use the default options based on the model settings
+                builderIndexOptions = defaultIndexOptions(indexVersionCreated, resolvedModelSettings);
             }
 
             final String fullName = context.buildFullName(leafName());
@@ -383,9 +398,9 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) {
                     fullName,
                     inferenceId.getValue(),
                     searchInferenceId.getValue(),
-                    modelSettings.getValue(),
+                    resolvedModelSettings,
                     chunkingSettings.getValue(),
-                    indexOptions.getValue(),
+                    builderIndexOptions,
                     inferenceField,
                     useLegacyFormat,
                     meta.getValue()
@@ -434,7 +449,6 @@ private void validateIndexOptions(SemanticTextIndexOptions indexOptions, String
             }
 
             if (indexOptions.type() == SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR) {
-
                 if (modelSettings.taskType() != TEXT_EMBEDDING) {
                     throw new IllegalArgumentException(
                         "Invalid task type for index options, required [" + TEXT_EMBEDDING + "] but was [" + modelSettings.taskType() + "]"
@@ -1221,20 +1235,33 @@ static boolean indexVersionDefaultsToBbqHnsw(IndexVersion indexVersion) {
             || indexVersion.between(SEMANTIC_TEXT_DEFAULTS_TO_BBQ_BACKPORT_8_X, IndexVersions.UPGRADE_TO_LUCENE_10_0_0);
     }
 
+    static SemanticTextIndexOptions defaultIndexOptions(IndexVersion indexVersionCreated, MinimalServiceSettings modelSettings) {
+
+        if (modelSettings == null) {
+            return null;
+        }
+
+        SemanticTextIndexOptions defaultIndexOptions = null;
+        if (modelSettings.taskType() == TaskType.TEXT_EMBEDDING) {
+            DenseVectorFieldMapper.DenseVectorIndexOptions denseVectorIndexOptions = defaultDenseVectorIndexOptions(
+                indexVersionCreated,
+                modelSettings
+            );
+            defaultIndexOptions = denseVectorIndexOptions == null
+                ? null
+                : new SemanticTextIndexOptions(SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, denseVectorIndexOptions);
+        }
+
+        return defaultIndexOptions;
+    }
+
     static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDenseVectorIndexOptions() {
         int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
         int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
         DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector(DEFAULT_RESCORE_OVERSAMPLE);
         return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector);
     }
 
-    static SemanticTextIndexOptions defaultBbqHnswSemanticTextIndexOptions() {
-        return new SemanticTextIndexOptions(
-            SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR,
-            defaultBbqHnswDenseVectorIndexOptions()
-        );
-    }
-
     private static boolean canMergeModelSettings(MinimalServiceSettings previous, MinimalServiceSettings current, Conflicts conflicts) {
         if (previous != null && current != null && previous.canMergeWith(current)) {
             return true;

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextIndexOptions.java

@@ -20,6 +20,7 @@
 import java.util.Arrays;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * Represents index options for a semantic_text field.
@@ -50,6 +51,18 @@ public IndexOptions indexOptions() {
         return indexOptions;
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (o instanceof SemanticTextIndexOptions == false) return false;
+        SemanticTextIndexOptions that = (SemanticTextIndexOptions) o;
+        return type == that.type && Objects.equals(indexOptions, that.indexOptions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(type, indexOptions);
+    }
+
     public enum SupportedIndexOptions {
         DENSE_VECTOR("dense_vector") {
             @Override