Add Sparse Vector Index Options Settings to Semantic Text Field

docs/changelog/131058.yaml

@@ -0,0 +1,5 @@
+pr: 131058
+summary: Adds sparse vector index options settings to semantic_text field
+area: Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -156,9 +156,11 @@ to create the endpoint. If not specified, the {{infer}} endpoint defined by
 
 `index_options` {applies_to}`stack: ga 9.1`
 :   (Optional, object) Specifies the index options to override default values
-for the field. Currently, `dense_vector` index options are supported.
-For text embeddings, `index_options` may match any allowed
-[dense_vector index options](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-index-options).
+for the field. Currently, `dense_vector` and `sparse_vector` index options are supported.
+For text embeddings, `index_options` may match any allowed.
+
+* [dense_vector index options](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-index-options).
+* [sparse_vector index options](/reference/elasticsearch/mapping-reference/sparse-vector.md#sparse-vectors-params). {applies_to}`stack: ga 9.2`
 
 `chunking_settings` {applies_to}`stack: ga 9.1`
 :   (Optional, object) Settings for chunking text into smaller passages.
@@ -410,7 +412,7 @@ stack: ga 9.0
 In case you want to customize data indexing, use the
 [`sparse_vector`](/reference/elasticsearch/mapping-reference/sparse-vector.md)
 or [`dense_vector`](/reference/elasticsearch/mapping-reference/dense-vector.md)
-field types and create an ingest pipeline with an 
+field types and create an ingest pipeline with an
 [{{infer}} processor](/reference/enrich-processor/inference-processor.md) to
 generate the embeddings.
 [This tutorial](docs-content://solutions/search/semantic-search/semantic-search-inference.md)

server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java

@@ -48,7 +48,6 @@
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParser.Token;
@@ -98,7 +97,7 @@ public static class Builder extends FieldMapper.Builder {
 
         private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).fieldType().isStored(), false);
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
-        private final Parameter<IndexOptions> indexOptions = new Parameter<>(
+        private final Parameter<SparseVectorIndexOptions> indexOptions = new Parameter<>(
             SPARSE_VECTOR_INDEX_OPTIONS,
             true,
             () -> null,
@@ -128,9 +127,9 @@ protected Parameter<?>[] getParameters() {
 
         @Override
         public SparseVectorFieldMapper build(MapperBuilderContext context) {
-            IndexOptions builderIndexOptions = indexOptions.getValue();
+            SparseVectorIndexOptions builderIndexOptions = indexOptions.getValue();
             if (builderIndexOptions == null) {
-                builderIndexOptions = getDefaultIndexOptions(indexVersionCreated);
+                builderIndexOptions = SparseVectorIndexOptions.getDefaultIndexOptions(indexVersionCreated);
             }
 
             final boolean syntheticVectorFinal = context.isSourceSynthetic() == false && isSyntheticVector;
@@ -149,33 +148,34 @@ public SparseVectorFieldMapper build(MapperBuilderContext context) {
             );
         }
 
-        private IndexOptions getDefaultIndexOptions(IndexVersion indexVersion) {
-            return (indexVersion.onOrAfter(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION)
-                || indexVersion.between(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION_8_X, IndexVersions.UPGRADE_TO_LUCENE_10_0_0))
-                    ? IndexOptions.DEFAULT_PRUNING_INDEX_OPTIONS
-                    : null;
+        private boolean indexOptionsSerializerCheck(boolean includeDefaults, boolean isConfigured, SparseVectorIndexOptions value) {
+            return includeDefaults || (SparseVectorIndexOptions.isDefaultOptions(value, indexVersionCreated) == false);
         }
 
-        private boolean indexOptionsSerializerCheck(boolean includeDefaults, boolean isConfigured, IndexOptions value) {
-            return includeDefaults || (IndexOptions.isDefaultOptions(value, indexVersionCreated) == false);
+        public void setIndexOptions(SparseVectorIndexOptions sparseVectorIndexOptions) {
+            indexOptions.setValue(sparseVectorIndexOptions);
         }
     }
 
-    public IndexOptions getIndexOptions() {
+    public SparseVectorIndexOptions getIndexOptions() {
         return fieldType().getIndexOptions();
     }
 
-    private static final ConstructingObjectParser<IndexOptions, Void> INDEX_OPTIONS_PARSER = new ConstructingObjectParser<>(
+    private static final ConstructingObjectParser<SparseVectorIndexOptions, Void> INDEX_OPTIONS_PARSER = new ConstructingObjectParser<>(
         SPARSE_VECTOR_INDEX_OPTIONS,
-        args -> new IndexOptions((Boolean) args[0], (TokenPruningConfig) args[1])
+        args -> new SparseVectorIndexOptions((Boolean) args[0], (TokenPruningConfig) args[1])
     );
 
     static {
-        INDEX_OPTIONS_PARSER.declareBoolean(optionalConstructorArg(), IndexOptions.PRUNE_FIELD_NAME);
-        INDEX_OPTIONS_PARSER.declareObject(optionalConstructorArg(), TokenPruningConfig.PARSER, IndexOptions.PRUNING_CONFIG_FIELD_NAME);
+        INDEX_OPTIONS_PARSER.declareBoolean(optionalConstructorArg(), SparseVectorIndexOptions.PRUNE_FIELD_NAME);
+        INDEX_OPTIONS_PARSER.declareObject(
+            optionalConstructorArg(),
+            TokenPruningConfig.PARSER,
+            SparseVectorIndexOptions.PRUNING_CONFIG_FIELD_NAME
+        );
     }
 
-    private static SparseVectorFieldMapper.IndexOptions parseIndexOptions(MappingParserContext context, Object propNode) {
+    private static SparseVectorIndexOptions parseIndexOptions(MappingParserContext context, Object propNode) {
         if (propNode == null) {
             return null;
         }
@@ -212,7 +212,7 @@ private static SparseVectorFieldMapper.IndexOptions parseIndexOptions(MappingPar
 
     public static final class SparseVectorFieldType extends MappedFieldType {
         private final IndexVersion indexVersionCreated;
-        private final IndexOptions indexOptions;
+        private final SparseVectorIndexOptions indexOptions;
 
         public SparseVectorFieldType(IndexVersion indexVersionCreated, String name, boolean isStored, Map<String, String> meta) {
             this(indexVersionCreated, name, isStored, meta, null);
@@ -223,14 +223,14 @@ public SparseVectorFieldType(
             String name,
             boolean isStored,
             Map<String, String> meta,
-            @Nullable SparseVectorFieldMapper.IndexOptions indexOptions
+            @Nullable SparseVectorIndexOptions indexOptions
         ) {
             super(name, true, isStored, false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             this.indexVersionCreated = indexVersionCreated;
             this.indexOptions = indexOptions;
         }
 
-        public IndexOptions getIndexOptions() {
+        public SparseVectorIndexOptions getIndexOptions() {
             return indexOptions;
         }
 
@@ -560,15 +560,18 @@ public void reset() {
         }
     }
 
-    public static class IndexOptions implements ToXContent {
+    public static class SparseVectorIndexOptions implements IndexOptions {
         public static final ParseField PRUNE_FIELD_NAME = new ParseField("prune");
         public static final ParseField PRUNING_CONFIG_FIELD_NAME = new ParseField("pruning_config");
-        public static final IndexOptions DEFAULT_PRUNING_INDEX_OPTIONS = new IndexOptions(true, new TokenPruningConfig());
+        public static final SparseVectorIndexOptions DEFAULT_PRUNING_INDEX_OPTIONS = new SparseVectorIndexOptions(
+            true,
+            new TokenPruningConfig()
+        );
 
         final Boolean prune;
         final TokenPruningConfig pruningConfig;
 
-        IndexOptions(@Nullable Boolean prune, @Nullable TokenPruningConfig pruningConfig) {
+        public SparseVectorIndexOptions(@Nullable Boolean prune, @Nullable TokenPruningConfig pruningConfig) {
             if (pruningConfig != null && (prune == null || prune == false)) {
                 throw new IllegalArgumentException(
                     "["
@@ -585,14 +588,37 @@ public static class IndexOptions implements ToXContent {
             this.pruningConfig = pruningConfig;
         }
 
-        public static boolean isDefaultOptions(IndexOptions indexOptions, IndexVersion indexVersion) {
-            IndexOptions defaultIndexOptions = indexVersionSupportsDefaultPruningConfig(indexVersion)
+        public static boolean isDefaultOptions(SparseVectorIndexOptions indexOptions, IndexVersion indexVersion) {
+            SparseVectorIndexOptions defaultIndexOptions = indexVersionSupportsDefaultPruningConfig(indexVersion)
                 ? DEFAULT_PRUNING_INDEX_OPTIONS
                 : null;
 
             return Objects.equals(indexOptions, defaultIndexOptions);
         }
 
+        public static SparseVectorIndexOptions getDefaultIndexOptions(IndexVersion indexVersion) {
+            return indexVersionSupportsDefaultPruningConfig(indexVersion) ? DEFAULT_PRUNING_INDEX_OPTIONS : null;
+        }
+
+        public static SparseVectorIndexOptions parseFromMap(Map<String, Object> map) {
+            if (map == null) {
+                return null;
+            }
+
+            try {
+                XContentParser parser = new MapXContentParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.IGNORE_DEPRECATIONS,
+                    map,
+                    XContentType.JSON
+                );
+
+                return INDEX_OPTIONS_PARSER.parse(parser, null);
+            } catch (IOException ioEx) {
+                throw new UncheckedIOException(ioEx);
+            }
+        }
+
         public Boolean getPrune() {
             return prune;
         }
@@ -626,7 +652,7 @@ public final boolean equals(Object other) {
                 return false;
             }
 
-            IndexOptions otherAsIndexOptions = (IndexOptions) other;
+            SparseVectorIndexOptions otherAsIndexOptions = (SparseVectorIndexOptions) other;
             return Objects.equals(prune, otherAsIndexOptions.prune) && Objects.equals(pruningConfig, otherAsIndexOptions.pruningConfig);
         }
 

server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java

@@ -906,4 +906,8 @@ private Map<String, Float> toFloats(Map<String, ?> value) {
         }
         return result;
     }
+
+    public static IndexVersion getIndexOptionsCompatibleIndexVersion() {
+        return IndexVersionUtils.randomVersionBetween(random(), SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, IndexVersion.current());
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java

@@ -40,4 +40,28 @@ public void testIsNotAggregatable() {
         MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType(indexVersion, "field", false, Collections.emptyMap());
         assertFalse(fieldType.isAggregatable());
     }
+
+    public static SparseVectorFieldMapper.SparseVectorIndexOptions randomSparseVectorIndexOptions() {
+        return randomSparseVectorIndexOptions(true);
+    }
+
+    public static SparseVectorFieldMapper.SparseVectorIndexOptions randomSparseVectorIndexOptions(boolean includeNull) {
+        if (includeNull && randomBoolean()) {
+            return null;
+        }
+
+        Boolean prune = randomBoolean() ? null : randomBoolean();
+        if (prune == null) {
+            new SparseVectorFieldMapper.SparseVectorIndexOptions(null, null);
+        }
+
+        if (prune == Boolean.FALSE) {
+            new SparseVectorFieldMapper.SparseVectorIndexOptions(false, null);
+        }
+
+        return new SparseVectorFieldMapper.SparseVectorIndexOptions(
+            true,
+            new TokenPruningConfig(randomFloatBetween(1.0f, 100.0f, true), randomFloatBetween(0.0f, 1.0f, true), randomBoolean())
+        );
+    }
 }

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java

@@ -19,6 +19,7 @@
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_INDEX_OPTIONS;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_INDEX_OPTIONS_WITH_DEFAULTS;
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_SPARSE_VECTOR_INDEX_OPTIONS;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_SUPPORT_CHUNKING_CONFIG;
 import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_FILTER_FIX;
 import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED;
@@ -78,7 +79,8 @@ public Set<NodeFeature> getTestFeatures() {
                 COHERE_V2_API,
                 SEMANTIC_TEXT_INDEX_OPTIONS_WITH_DEFAULTS,
                 SEMANTIC_QUERY_REWRITE_INTERCEPTORS_PROPAGATE_BOOST_AND_QUERY_NAME_FIX,
-                SEMANTIC_TEXT_HIGHLIGHTING_FLAT
+                SEMANTIC_TEXT_HIGHLIGHTING_FLAT,
+                SEMANTIC_TEXT_SPARSE_VECTOR_INDEX_OPTIONS
             )
         );
         if (RERANK_SNIPPETS.isEnabled()) {