Merge branch 'main' into 2025/07/28/write-load-decider

Author: DiannaHohensee

docs/changelog/131058.yaml

@@ -0,0 +1,5 @@
+pr: 131058
+summary: Adds sparse vector index options settings to semantic_text field
+area: Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -156,9 +156,11 @@ to create the endpoint. If not specified, the {{infer}} endpoint defined by
 
 `index_options` {applies_to}`stack: ga 9.1`
 :   (Optional, object) Specifies the index options to override default values
-for the field. Currently, `dense_vector` index options are supported.
-For text embeddings, `index_options` may match any allowed
-[dense_vector index options](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-index-options).
+for the field. Currently, `dense_vector` and `sparse_vector` index options are supported.
+For text embeddings, `index_options` may match any allowed.
+
+* [dense_vector index options](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-index-options).
+* [sparse_vector index options](/reference/elasticsearch/mapping-reference/sparse-vector.md#sparse-vectors-params). {applies_to}`stack: ga 9.2`
 
 `chunking_settings` {applies_to}`stack: ga 9.1`
 :   (Optional, object) Settings for chunking text into smaller passages.
@@ -410,7 +412,7 @@ stack: ga 9.0
 In case you want to customize data indexing, use the
 [`sparse_vector`](/reference/elasticsearch/mapping-reference/sparse-vector.md)
 or [`dense_vector`](/reference/elasticsearch/mapping-reference/dense-vector.md)
-field types and create an ingest pipeline with an 
+field types and create an ingest pipeline with an
 [{{infer}} processor](/reference/enrich-processor/inference-processor.md) to
 generate the embeddings.
 [This tutorial](docs-content://solutions/search/semantic-search/semantic-search-inference.md)

docs/reference/text-analysis/analysis-lowercase-tokenfilter.md

@@ -58,15 +58,15 @@ PUT lowercase_example
 :   (Optional, string) Language-specific lowercase token filter to use. Valid values include:
 
 `greek`
-:   Uses Lucene’s [GreekLowerCaseFilter](https://lucene.apache.org/core/10_0_0/analysis/common/org/apache/lucene/analysis/el/GreekLowerCaseFilter.md)
+:   Uses Lucene’s [GreekLowerCaseFilter](https://lucene.apache.org/core/10_0_0/analysis/common/org/apache/lucene/analysis/el/GreekLowerCaseFilter.html)
 
 `irish`
-:   Uses Lucene’s [IrishLowerCaseFilter](https://lucene.apache.org/core/10_0_0/analysis/common/org/apache/lucene/analysis/ga/IrishLowerCaseFilter.md)
+:   Uses Lucene’s [IrishLowerCaseFilter](https://lucene.apache.org/core/10_0_0/analysis/common/org/apache/lucene/analysis/ga/IrishLowerCaseFilter.html)
 
 `turkish`
-:   Uses Lucene’s [TurkishLowerCaseFilter](https://lucene.apache.org/core/10_0_0/analysis/common/org/apache/lucene/analysis/tr/TurkishLowerCaseFilter.md)
+:   Uses Lucene’s [TurkishLowerCaseFilter](https://lucene.apache.org/core/10_0_0/analysis/common/org/apache/lucene/analysis/tr/TurkishLowerCaseFilter.html)
 
-If not specified, defaults to Lucene’s [LowerCaseFilter](https://lucene.apache.org/core/10_0_0/analysis/common/org/apache/lucene/analysis/core/LowerCaseFilter.md).
+If not specified, defaults to Lucene’s [LowerCaseFilter](https://lucene.apache.org/core/10_0_0/analysis/common/org/apache/lucene/analysis/core/LowerCaseFilter.html).
 
 
 

muted-tests.yml

@@ -413,15 +413,6 @@ tests:
 - class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
   method: test {p0=search/110_field_collapsing/field collapsing, inner_hits and maxConcurrentGroupRequests}
   issue: https://github.com/elastic/elasticsearch/issues/131348
-- class: org.elasticsearch.xpack.esql.vector.VectorSimilarityFunctionsIT
-  method: testSimilarityBetweenConstantVectors {functionName=v_cosine similarityFunction=COSINE}
-  issue: https://github.com/elastic/elasticsearch/issues/131361
-- class: org.elasticsearch.xpack.esql.vector.VectorSimilarityFunctionsIT
-  method: testDifferentDimensions {functionName=v_cosine similarityFunction=COSINE}
-  issue: https://github.com/elastic/elasticsearch/issues/131362
-- class: org.elasticsearch.xpack.esql.vector.VectorSimilarityFunctionsIT
-  method: testSimilarityBetweenConstantVectorAndField {functionName=v_cosine similarityFunction=COSINE}
-  issue: https://github.com/elastic/elasticsearch/issues/131363
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/delete_expired_data/Test delete expired data with body parameters}
   issue: https://github.com/elastic/elasticsearch/issues/131364
@@ -485,9 +476,6 @@ tests:
 - class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTests
   method: testUpdates
   issue: https://github.com/elastic/elasticsearch/issues/131795
-- class: org.elasticsearch.xpack.esql.vector.VectorSimilarityFunctionsIT
-  method: testDifferentDimensions {functionName=v_dot_product similarityFunction=DOT_PRODUCT}
-  issue: https://github.com/elastic/elasticsearch/issues/131845
 - class: org.elasticsearch.xpack.restart.FullClusterRestartIT
   method: testWatcherWithApiKey {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/131964

server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java

@@ -48,7 +48,6 @@
 import org.elasticsearch.xcontent.DeprecationHandler;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParser.Token;
@@ -98,7 +97,7 @@ public static class Builder extends FieldMapper.Builder {
 
         private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).fieldType().isStored(), false);
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
-        private final Parameter<IndexOptions> indexOptions = new Parameter<>(
+        private final Parameter<SparseVectorIndexOptions> indexOptions = new Parameter<>(
             SPARSE_VECTOR_INDEX_OPTIONS,
             true,
             () -> null,
@@ -128,9 +127,9 @@ protected Parameter<?>[] getParameters() {
 
         @Override
         public SparseVectorFieldMapper build(MapperBuilderContext context) {
-            IndexOptions builderIndexOptions = indexOptions.getValue();
+            SparseVectorIndexOptions builderIndexOptions = indexOptions.getValue();
             if (builderIndexOptions == null) {
-                builderIndexOptions = getDefaultIndexOptions(indexVersionCreated);
+                builderIndexOptions = SparseVectorIndexOptions.getDefaultIndexOptions(indexVersionCreated);
             }
 
             final boolean syntheticVectorFinal = context.isSourceSynthetic() == false && isSyntheticVector;
@@ -149,33 +148,34 @@ public SparseVectorFieldMapper build(MapperBuilderContext context) {
             );
         }
 
-        private IndexOptions getDefaultIndexOptions(IndexVersion indexVersion) {
-            return (indexVersion.onOrAfter(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION)
-                || indexVersion.between(SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_VERSION_8_X, IndexVersions.UPGRADE_TO_LUCENE_10_0_0))
-                    ? IndexOptions.DEFAULT_PRUNING_INDEX_OPTIONS
-                    : null;
+        private boolean indexOptionsSerializerCheck(boolean includeDefaults, boolean isConfigured, SparseVectorIndexOptions value) {
+            return includeDefaults || (SparseVectorIndexOptions.isDefaultOptions(value, indexVersionCreated) == false);
         }
 
-        private boolean indexOptionsSerializerCheck(boolean includeDefaults, boolean isConfigured, IndexOptions value) {
-            return includeDefaults || (IndexOptions.isDefaultOptions(value, indexVersionCreated) == false);
+        public void setIndexOptions(SparseVectorIndexOptions sparseVectorIndexOptions) {
+            indexOptions.setValue(sparseVectorIndexOptions);
         }
     }
 
-    public IndexOptions getIndexOptions() {
+    public SparseVectorIndexOptions getIndexOptions() {
         return fieldType().getIndexOptions();
     }
 
-    private static final ConstructingObjectParser<IndexOptions, Void> INDEX_OPTIONS_PARSER = new ConstructingObjectParser<>(
+    private static final ConstructingObjectParser<SparseVectorIndexOptions, Void> INDEX_OPTIONS_PARSER = new ConstructingObjectParser<>(
         SPARSE_VECTOR_INDEX_OPTIONS,
-        args -> new IndexOptions((Boolean) args[0], (TokenPruningConfig) args[1])
+        args -> new SparseVectorIndexOptions((Boolean) args[0], (TokenPruningConfig) args[1])
     );
 
     static {
-        INDEX_OPTIONS_PARSER.declareBoolean(optionalConstructorArg(), IndexOptions.PRUNE_FIELD_NAME);
-        INDEX_OPTIONS_PARSER.declareObject(optionalConstructorArg(), TokenPruningConfig.PARSER, IndexOptions.PRUNING_CONFIG_FIELD_NAME);
+        INDEX_OPTIONS_PARSER.declareBoolean(optionalConstructorArg(), SparseVectorIndexOptions.PRUNE_FIELD_NAME);
+        INDEX_OPTIONS_PARSER.declareObject(
+            optionalConstructorArg(),
+            TokenPruningConfig.PARSER,
+            SparseVectorIndexOptions.PRUNING_CONFIG_FIELD_NAME
+        );
     }
 
-    private static SparseVectorFieldMapper.IndexOptions parseIndexOptions(MappingParserContext context, Object propNode) {
+    private static SparseVectorIndexOptions parseIndexOptions(MappingParserContext context, Object propNode) {
         if (propNode == null) {
             return null;
         }
@@ -212,7 +212,7 @@ private static SparseVectorFieldMapper.IndexOptions parseIndexOptions(MappingPar
 
     public static final class SparseVectorFieldType extends MappedFieldType {
         private final IndexVersion indexVersionCreated;
-        private final IndexOptions indexOptions;
+        private final SparseVectorIndexOptions indexOptions;
 
         public SparseVectorFieldType(IndexVersion indexVersionCreated, String name, boolean isStored, Map<String, String> meta) {
             this(indexVersionCreated, name, isStored, meta, null);
@@ -223,14 +223,14 @@ public SparseVectorFieldType(
             String name,
             boolean isStored,
             Map<String, String> meta,
-            @Nullable SparseVectorFieldMapper.IndexOptions indexOptions
+            @Nullable SparseVectorIndexOptions indexOptions
         ) {
             super(name, true, isStored, false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             this.indexVersionCreated = indexVersionCreated;
             this.indexOptions = indexOptions;
         }
 
-        public IndexOptions getIndexOptions() {
+        public SparseVectorIndexOptions getIndexOptions() {
             return indexOptions;
         }
 
@@ -560,15 +560,18 @@ public void reset() {
         }
     }
 
-    public static class IndexOptions implements ToXContent {
+    public static class SparseVectorIndexOptions implements IndexOptions {
         public static final ParseField PRUNE_FIELD_NAME = new ParseField("prune");
         public static final ParseField PRUNING_CONFIG_FIELD_NAME = new ParseField("pruning_config");
-        public static final IndexOptions DEFAULT_PRUNING_INDEX_OPTIONS = new IndexOptions(true, new TokenPruningConfig());
+        public static final SparseVectorIndexOptions DEFAULT_PRUNING_INDEX_OPTIONS = new SparseVectorIndexOptions(
+            true,
+            new TokenPruningConfig()
+        );
 
         final Boolean prune;
         final TokenPruningConfig pruningConfig;
 
-        IndexOptions(@Nullable Boolean prune, @Nullable TokenPruningConfig pruningConfig) {
+        public SparseVectorIndexOptions(@Nullable Boolean prune, @Nullable TokenPruningConfig pruningConfig) {
             if (pruningConfig != null && (prune == null || prune == false)) {
                 throw new IllegalArgumentException(
                     "["
@@ -585,14 +588,37 @@ public static class IndexOptions implements ToXContent {
             this.pruningConfig = pruningConfig;
         }
 
-        public static boolean isDefaultOptions(IndexOptions indexOptions, IndexVersion indexVersion) {
-            IndexOptions defaultIndexOptions = indexVersionSupportsDefaultPruningConfig(indexVersion)
+        public static boolean isDefaultOptions(SparseVectorIndexOptions indexOptions, IndexVersion indexVersion) {
+            SparseVectorIndexOptions defaultIndexOptions = indexVersionSupportsDefaultPruningConfig(indexVersion)
                 ? DEFAULT_PRUNING_INDEX_OPTIONS
                 : null;
 
             return Objects.equals(indexOptions, defaultIndexOptions);
         }
 
+        public static SparseVectorIndexOptions getDefaultIndexOptions(IndexVersion indexVersion) {
+            return indexVersionSupportsDefaultPruningConfig(indexVersion) ? DEFAULT_PRUNING_INDEX_OPTIONS : null;
+        }
+
+        public static SparseVectorIndexOptions parseFromMap(Map<String, Object> map) {
+            if (map == null) {
+                return null;
+            }
+
+            try {
+                XContentParser parser = new MapXContentParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.IGNORE_DEPRECATIONS,
+                    map,
+                    XContentType.JSON
+                );
+
+                return INDEX_OPTIONS_PARSER.parse(parser, null);
+            } catch (IOException ioEx) {
+                throw new UncheckedIOException(ioEx);
+            }
+        }
+
         public Boolean getPrune() {
             return prune;
         }
@@ -626,7 +652,7 @@ public final boolean equals(Object other) {
                 return false;
             }
 
-            IndexOptions otherAsIndexOptions = (IndexOptions) other;
+            SparseVectorIndexOptions otherAsIndexOptions = (SparseVectorIndexOptions) other;
             return Objects.equals(prune, otherAsIndexOptions.prune) && Objects.equals(pruningConfig, otherAsIndexOptions.pruningConfig);
         }
 

server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java

@@ -906,4 +906,8 @@ private Map<String, Float> toFloats(Map<String, ?> value) {
         }
         return result;
     }
+
+    public static IndexVersion getIndexOptionsCompatibleIndexVersion() {
+        return IndexVersionUtils.randomVersionBetween(random(), SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, IndexVersion.current());
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java

@@ -40,4 +40,28 @@ public void testIsNotAggregatable() {
         MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType(indexVersion, "field", false, Collections.emptyMap());
         assertFalse(fieldType.isAggregatable());
     }
+
+    public static SparseVectorFieldMapper.SparseVectorIndexOptions randomSparseVectorIndexOptions() {
+        return randomSparseVectorIndexOptions(true);
+    }
+
+    public static SparseVectorFieldMapper.SparseVectorIndexOptions randomSparseVectorIndexOptions(boolean includeNull) {
+        if (includeNull && randomBoolean()) {
+            return null;
+        }
+
+        Boolean prune = randomBoolean() ? null : randomBoolean();
+        if (prune == null) {
+            new SparseVectorFieldMapper.SparseVectorIndexOptions(null, null);
+        }
+
+        if (prune == Boolean.FALSE) {
+            new SparseVectorFieldMapper.SparseVectorIndexOptions(false, null);
+        }
+
+        return new SparseVectorFieldMapper.SparseVectorIndexOptions(
+            true,
+            new TokenPruningConfig(randomFloatBetween(1.0f, 100.0f, true), randomFloatBetween(0.0f, 1.0f, true), randomBoolean())
+        );
+    }
 }

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AbstractTopBytesRefAggregatorFunctionTests.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BlockUtils;
+import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.SequenceBytesRefBlockSourceOperator;
 import org.elasticsearch.compute.operator.SourceOperator;
 
@@ -30,8 +31,8 @@ protected final SourceOperator simpleInput(BlockFactory blockFactory, int size)
     protected abstract BytesRef randomValue();
 
     @Override
-    public final void assertSimpleOutput(List<Block> input, Block result) {
-        Object[] values = input.stream().flatMap(AggregatorFunctionTestCase::allBytesRefs).sorted().limit(LIMIT).toArray(Object[]::new);
+    public final void assertSimpleOutput(List<Page> input, Block result) {
+        Object[] values = input.stream().flatMap(p -> allBytesRefs(p.getBlock(0))).sorted().limit(LIMIT).toArray(Object[]::new);
         assertThat((List<?>) BlockUtils.toJavaObject(result, 0), contains(values));
     }
 }

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java

@@ -58,7 +58,12 @@ protected final int aggregatorIntermediateBlockCount() {
 
     protected abstract String expectedDescriptionOfAggregator();
 
-    protected abstract void assertSimpleOutput(List<Block> input, Block result);
+    /**
+     * Assert that the result is correct given the input.
+     * @param input the input pages build by {@link #simpleInput}
+     * @param result the result of running {@link #aggregatorFunction()}
+     */
+    protected abstract void assertSimpleOutput(List<Page> input, Block result);
 
     @Override
     protected Operator.OperatorFactory simpleWithMode(SimpleOptions options, AggregatorMode mode) {
@@ -99,7 +104,7 @@ protected final void assertSimpleOutput(List<Page> input, List<Page> results) {
         assertThat(results.get(0).getPositionCount(), equalTo(1));
 
         Block result = results.get(0).getBlock(0);
-        assertSimpleOutput(input.stream().map(p -> p.<Block>getBlock(0)).toList(), result);
+        assertSimpleOutput(input, result);
     }
 
     public final void testIgnoresNulls() {

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountAggregatorFunctionTests.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.SourceOperator;
 import org.elasticsearch.compute.test.SequenceLongBlockSourceOperator;
 
@@ -36,8 +37,8 @@ protected String expectedDescriptionOfAggregator() {
     }
 
     @Override
-    protected void assertSimpleOutput(List<Block> input, Block result) {
-        long count = input.stream().flatMapToLong(b -> allLongs(b)).count();
+    protected void assertSimpleOutput(List<Page> input, Block result) {
+        long count = input.stream().flatMapToLong(p -> allLongs(p.getBlock(0))).count();
         assertThat(((LongBlock) result).getLong(0), equalTo(count));
     }