diff --git a/docs/changelog/119967.yaml b/docs/changelog/119967.yaml new file mode 100644 index 0000000000000..be5543be20238 --- /dev/null +++ b/docs/changelog/119967.yaml @@ -0,0 +1,5 @@ +pr: 119967 +summary: Add `index_options` to `semantic_text` field mappings +area: Mapping +type: enhancement +issues: [ ] diff --git a/docs/reference/elasticsearch/mapping-reference/semantic-text.md b/docs/reference/elasticsearch/mapping-reference/semantic-text.md index 2799c91d466ba..0e71155b94ce5 100644 --- a/docs/reference/elasticsearch/mapping-reference/semantic-text.md +++ b/docs/reference/elasticsearch/mapping-reference/semantic-text.md @@ -28,7 +28,7 @@ service. Using `semantic_text`, you won’t need to specify how to generate embeddings for your data, or how to index it. The {{infer}} endpoint automatically determines -the embedding generation, indexing, and query to use. +the embedding generation, indexing, and query to use. Newly created indices with `semantic_text` fields using dense embeddings will be [quantized](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization) to `bbq_hnsw` automatically. @@ -111,6 +111,33 @@ the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/ope to create the endpoint. If not specified, the {{infer}} endpoint defined by `inference_id` will be used at both index and query time. +`index_options` +: (Optional, string) Specifies the index options to override default values +for the field. Currently, `dense_vector` index options are supported. +For text embeddings, `index_options` may match any allowed +[dense_vector index options](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-index-options). + +An example of how to set index_options for a `semantic_text` field: + +```console +PUT my-index-000004 +{ + "mappings": { + "properties": { + "inference_field": { + "type": "semantic_text", + "inference_id": "my-text-embedding-endpoint", + "index_options": { + "dense_vector": { + "type": "int4_flat" + } + } + } + } + } +} +``` + `chunking_settings` : (Optional, object) Settings for chunking text into smaller passages. If specified, these will override the chunking settings set in the {{infer-cap}} @@ -138,8 +165,10 @@ To completely disable chunking, use the `none` chunking strategy. or `1`. Required for `sentence` type chunking settings ::::{warning} -If the input exceeds the maximum token limit of the underlying model, some services (such as OpenAI) may return an -error. In contrast, the `elastic` and `elasticsearch` services will automatically truncate the input to fit within the +If the input exceeds the maximum token limit of the underlying model, some +services (such as OpenAI) may return an +error. In contrast, the `elastic` and `elasticsearch` services will +automatically truncate the input to fit within the model's limit. :::: @@ -173,7 +202,8 @@ For more details on chunking and how to configure chunking settings, see [Configuring chunking](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-inference) in the Inference API documentation. -You can pre-chunk the input by sending it to Elasticsearch as an array of strings. +You can pre-chunk the input by sending it to Elasticsearch as an array of +strings. Example: ```console @@ -203,15 +233,20 @@ PUT test-index/_doc/1 ``` 1. The text is pre-chunked and provided as an array of strings. - Each element in the array represents a single chunk that will be sent directly to the inference service without further chunking. + Each element in the array represents a single chunk that will be sent + directly to the inference service without further chunking. **Important considerations**: -* When providing pre-chunked input, ensure that you set the chunking strategy to `none` to avoid additional processing. -* Each chunk should be sized carefully, staying within the token limit of the inference service and the underlying model. -* If a chunk exceeds the model's token limit, the behavior depends on the service: - * Some services (such as OpenAI) will return an error. - * Others (such as `elastic` and `elasticsearch`) will automatically truncate the input. +* When providing pre-chunked input, ensure that you set the chunking strategy to + `none` to avoid additional processing. +* Each chunk should be sized carefully, staying within the token limit of the + inference service and the underlying model. +* If a chunk exceeds the model's token limit, the behavior depends on the + service: + * Some services (such as OpenAI) will return an error. + * Others (such as `elastic` and `elasticsearch`) will automatically truncate + the input. Refer to [this tutorial](docs-content://solutions/search/semantic-search/semantic-search-semantic-text.md) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index a0e8ceff02248..54c61a105feb4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -85,7 +85,6 @@ import org.elasticsearch.search.vectors.RescoreKnnVectorQuery; import org.elasticsearch.search.vectors.VectorData; import org.elasticsearch.search.vectors.VectorSimilarityQuery; -import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -257,7 +256,7 @@ public static class Builder extends FieldMapper.Builder { }); private final Parameter similarity; - private final Parameter indexOptions; + private final Parameter indexOptions; private final Parameter indexed; private final Parameter> meta = Parameter.metaParam(); @@ -372,7 +371,7 @@ public Builder elementType(ElementType elementType) { return this; } - public Builder indexOptions(IndexOptions indexOptions) { + public Builder indexOptions(DenseVectorIndexOptions indexOptions) { this.indexOptions.setValue(indexOptions); return this; } @@ -1309,10 +1308,10 @@ public final String toString() { public abstract VectorSimilarityFunction vectorSimilarityFunction(IndexVersion indexVersion, ElementType elementType); } - public abstract static class IndexOptions implements ToXContent { + public abstract static class DenseVectorIndexOptions implements IndexOptions { final VectorIndexType type; - IndexOptions(VectorIndexType type) { + DenseVectorIndexOptions(VectorIndexType type) { this.type = type; } @@ -1336,7 +1335,7 @@ final boolean validateElementType(ElementType elementType, boolean throwOnError) return validElementType; } - abstract boolean updatableTo(IndexOptions update); + public abstract boolean updatableTo(DenseVectorIndexOptions update); public boolean validateDimension(int dim) { return validateDimension(dim, true); @@ -1350,10 +1349,14 @@ public boolean validateDimension(int dim, boolean throwOnError) { return supportsDimension; } - abstract boolean doEquals(IndexOptions other); + abstract boolean doEquals(DenseVectorIndexOptions other); abstract int doHashCode(); + public VectorIndexType getType() { + return type; + } + @Override public final boolean equals(Object other) { if (other == this) { @@ -1362,7 +1365,7 @@ public final boolean equals(Object other) { if (other == null || other.getClass() != getClass()) { return false; } - IndexOptions otherOptions = (IndexOptions) other; + DenseVectorIndexOptions otherOptions = (DenseVectorIndexOptions) other; return Objects.equals(type, otherOptions.type) && doEquals(otherOptions); } @@ -1372,7 +1375,7 @@ public final int hashCode() { } } - abstract static class QuantizedIndexOptions extends IndexOptions { + abstract static class QuantizedIndexOptions extends DenseVectorIndexOptions { final RescoreVector rescoreVector; QuantizedIndexOptions(VectorIndexType type, RescoreVector rescoreVector) { @@ -1384,7 +1387,7 @@ abstract static class QuantizedIndexOptions extends IndexOptions { public enum VectorIndexType { HNSW("hnsw", false) { @Override - public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { + public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); if (mNode == null) { @@ -1411,7 +1414,7 @@ public boolean supportsDimension(int dims) { }, INT8_HNSW("int8_hnsw", true) { @Override - public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { + public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval"); @@ -1446,7 +1449,7 @@ public boolean supportsDimension(int dims) { } }, INT4_HNSW("int4_hnsw", true) { - public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { + public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval"); @@ -1482,7 +1485,7 @@ public boolean supportsDimension(int dims) { }, FLAT("flat", false) { @Override - public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { + public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); return new FlatIndexOptions(); } @@ -1499,7 +1502,7 @@ public boolean supportsDimension(int dims) { }, INT8_FLAT("int8_flat", true) { @Override - public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { + public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval"); Float confidenceInterval = null; if (confidenceIntervalNode != null) { @@ -1525,7 +1528,7 @@ public boolean supportsDimension(int dims) { }, INT4_FLAT("int4_flat", true) { @Override - public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { + public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval"); Float confidenceInterval = null; if (confidenceIntervalNode != null) { @@ -1551,7 +1554,7 @@ public boolean supportsDimension(int dims) { }, BBQ_HNSW("bbq_hnsw", true) { @Override - public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { + public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); if (mNode == null) { @@ -1585,7 +1588,7 @@ public boolean supportsDimension(int dims) { }, BBQ_FLAT("bbq_flat", true) { @Override - public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { + public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { RescoreVector rescoreVector = null; if (hasRescoreIndexVersion(indexVersion)) { rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion); @@ -1609,7 +1612,7 @@ public boolean supportsDimension(int dims) { }, BBQ_IVF("bbq_ivf", true) { @Override - public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { + public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object clusterSizeNode = indexOptionsMap.remove("cluster_size"); int clusterSize = IVFVectorsFormat.DEFAULT_VECTORS_PER_CLUSTER; if (clusterSizeNode != null) { @@ -1654,7 +1657,7 @@ public boolean supportsDimension(int dims) { } }; - static Optional fromString(String type) { + public static Optional fromString(String type) { return Stream.of(VectorIndexType.values()) .filter(vectorIndexType -> vectorIndexType != VectorIndexType.BBQ_IVF || IVF_FORMAT.isEnabled()) .filter(vectorIndexType -> vectorIndexType.name.equals(type)) @@ -1669,7 +1672,11 @@ static Optional fromString(String type) { this.quantized = quantized; } - abstract IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion); + public abstract DenseVectorIndexOptions parseIndexOptions( + String fieldName, + Map indexOptionsMap, + IndexVersion indexVersion + ); public abstract boolean supportsElementType(ElementType elementType); @@ -1679,6 +1686,10 @@ public boolean isQuantized() { return quantized; } + public String getName() { + return name; + } + @Override public String toString() { return name; @@ -1714,7 +1725,7 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) { } @Override - boolean doEquals(IndexOptions o) { + boolean doEquals(DenseVectorIndexOptions o) { Int8FlatIndexOptions that = (Int8FlatIndexOptions) o; return Objects.equals(confidenceInterval, that.confidenceInterval) && Objects.equals(rescoreVector, that.rescoreVector); } @@ -1725,7 +1736,7 @@ int doHashCode() { } @Override - boolean updatableTo(IndexOptions update) { + public boolean updatableTo(DenseVectorIndexOptions update) { return update.type.equals(this.type) || update.type.equals(VectorIndexType.HNSW) || update.type.equals(VectorIndexType.INT8_HNSW) @@ -1736,7 +1747,7 @@ boolean updatableTo(IndexOptions update) { } } - static class FlatIndexOptions extends IndexOptions { + static class FlatIndexOptions extends DenseVectorIndexOptions { FlatIndexOptions() { super(VectorIndexType.FLAT); @@ -1759,12 +1770,12 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) { } @Override - boolean updatableTo(IndexOptions update) { + public boolean updatableTo(DenseVectorIndexOptions update) { return true; } @Override - public boolean doEquals(IndexOptions o) { + public boolean doEquals(DenseVectorIndexOptions o) { return o instanceof FlatIndexOptions; } @@ -1774,12 +1785,12 @@ public int doHashCode() { } } - static class Int4HnswIndexOptions extends QuantizedIndexOptions { + public static class Int4HnswIndexOptions extends QuantizedIndexOptions { private final int m; private final int efConstruction; private final float confidenceInterval; - Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) { + public Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) { super(VectorIndexType.INT4_HNSW, rescoreVector); this.m = m; this.efConstruction = efConstruction; @@ -1809,7 +1820,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public boolean doEquals(IndexOptions o) { + public boolean doEquals(DenseVectorIndexOptions o) { Int4HnswIndexOptions that = (Int4HnswIndexOptions) o; return m == that.m && efConstruction == that.efConstruction @@ -1838,7 +1849,7 @@ public String toString() { } @Override - boolean updatableTo(IndexOptions update) { + public boolean updatableTo(DenseVectorIndexOptions update) { boolean updatable = false; if (update.type.equals(VectorIndexType.INT4_HNSW)) { Int4HnswIndexOptions int4HnswIndexOptions = (Int4HnswIndexOptions) update; @@ -1881,7 +1892,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public boolean doEquals(IndexOptions o) { + public boolean doEquals(DenseVectorIndexOptions o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Int4FlatIndexOptions that = (Int4FlatIndexOptions) o; @@ -1899,7 +1910,7 @@ public String toString() { } @Override - boolean updatableTo(IndexOptions update) { + public boolean updatableTo(DenseVectorIndexOptions update) { // TODO: add support for updating from flat, hnsw, and int8_hnsw and updating params return update.type.equals(this.type) || update.type.equals(VectorIndexType.HNSW) @@ -1946,7 +1957,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public boolean doEquals(IndexOptions o) { + public boolean doEquals(DenseVectorIndexOptions o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Int8HnswIndexOptions that = (Int8HnswIndexOptions) o; @@ -1977,7 +1988,7 @@ public String toString() { } @Override - boolean updatableTo(IndexOptions update) { + public boolean updatableTo(DenseVectorIndexOptions update) { boolean updatable; if (update.type.equals(this.type)) { Int8HnswIndexOptions int8HnswIndexOptions = (Int8HnswIndexOptions) update; @@ -1995,7 +2006,7 @@ boolean updatableTo(IndexOptions update) { } } - static class HnswIndexOptions extends IndexOptions { + static class HnswIndexOptions extends DenseVectorIndexOptions { private final int m; private final int efConstruction; @@ -2014,7 +2025,7 @@ public KnnVectorsFormat getVectorsFormat(ElementType elementType) { } @Override - boolean updatableTo(IndexOptions update) { + public boolean updatableTo(DenseVectorIndexOptions update) { boolean updatable = update.type.equals(this.type); if (updatable) { // fewer connections would break assumptions on max number of connections (based on largest previous graph) during merge @@ -2038,7 +2049,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public boolean doEquals(IndexOptions o) { + public boolean doEquals(DenseVectorIndexOptions o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; HnswIndexOptions that = (HnswIndexOptions) o; @@ -2073,12 +2084,12 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) { } @Override - boolean updatableTo(IndexOptions update) { + public boolean updatableTo(DenseVectorIndexOptions update) { return update.type.equals(this.type) && ((BBQHnswIndexOptions) update).m >= this.m; } @Override - boolean doEquals(IndexOptions other) { + boolean doEquals(DenseVectorIndexOptions other) { BBQHnswIndexOptions that = (BBQHnswIndexOptions) other; return m == that.m && efConstruction == that.efConstruction && Objects.equals(rescoreVector, that.rescoreVector); } @@ -2127,12 +2138,12 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) { } @Override - boolean updatableTo(IndexOptions update) { + public boolean updatableTo(DenseVectorIndexOptions update) { return update.type.equals(this.type) || update.type.equals(VectorIndexType.BBQ_HNSW); } @Override - boolean doEquals(IndexOptions other) { + boolean doEquals(DenseVectorIndexOptions other) { return other instanceof BBQFlatIndexOptions; } @@ -2182,12 +2193,12 @@ KnnVectorsFormat getVectorsFormat(ElementType elementType) { } @Override - boolean updatableTo(IndexOptions update) { + public boolean updatableTo(DenseVectorIndexOptions update) { return update.type.equals(this.type); } @Override - boolean doEquals(IndexOptions other) { + boolean doEquals(DenseVectorIndexOptions other) { BBQIVFIndexOptions that = (BBQIVFIndexOptions) other; return clusterSize == that.clusterSize && defaultNProbe == that.defaultNProbe @@ -2259,7 +2270,7 @@ public static final class DenseVectorFieldType extends SimpleMappedFieldType { private final boolean indexed; private final VectorSimilarity similarity; private final IndexVersion indexVersionCreated; - private final IndexOptions indexOptions; + private final DenseVectorIndexOptions indexOptions; private final boolean isSyntheticSource; public DenseVectorFieldType( @@ -2269,7 +2280,7 @@ public DenseVectorFieldType( Integer dims, boolean indexed, VectorSimilarity similarity, - IndexOptions indexOptions, + DenseVectorIndexOptions indexOptions, Map meta, boolean isSyntheticSource ) { @@ -2634,14 +2645,14 @@ public List fetchValues(Source source, int doc, List ignoredValu } } - private final IndexOptions indexOptions; + private final DenseVectorIndexOptions indexOptions; private final IndexVersion indexCreatedVersion; private DenseVectorFieldMapper( String simpleName, MappedFieldType mappedFieldType, BuilderParams params, - IndexOptions indexOptions, + DenseVectorIndexOptions indexOptions, IndexVersion indexCreatedVersion ) { super(simpleName, mappedFieldType, params); @@ -2789,7 +2800,7 @@ public FieldMapper.Builder getMergeBuilder() { return new Builder(leafName(), indexCreatedVersion).init(this); } - private static IndexOptions parseIndexOptions(String fieldName, Object propNode, IndexVersion indexVersion) { + private static DenseVectorIndexOptions parseIndexOptions(String fieldName, Object propNode, IndexVersion indexVersion) { @SuppressWarnings("unchecked") Map indexOptionsMap = (Map) propNode; Object typeNode = indexOptionsMap.remove("type"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/IndexOptions.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/IndexOptions.java new file mode 100644 index 0000000000000..4679b7d3f0982 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/IndexOptions.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.vectors; + +import org.elasticsearch.xcontent.ToXContent; + +/** + * Represents general index options that can be attached to a semantic or vector field. + */ +public interface IndexOptions extends ToXContent {} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 15d36670929b1..6c7964bbf773b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -57,14 +57,14 @@ private static DenseVectorFieldMapper.RescoreVector randomRescoreVector() { return new DenseVectorFieldMapper.RescoreVector(randomBoolean() ? 0 : randomFloatBetween(1.0F, 10.0F, false)); } - private DenseVectorFieldMapper.IndexOptions randomIndexOptionsNonQuantized() { + private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsNonQuantized() { return randomFrom( new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)), new DenseVectorFieldMapper.FlatIndexOptions() ); } - private DenseVectorFieldMapper.IndexOptions randomIndexOptionsAll() { + public static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsAll() { return randomFrom( new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)), new DenseVectorFieldMapper.Int8HnswIndexOptions( @@ -97,11 +97,13 @@ private DenseVectorFieldMapper.IndexOptions randomIndexOptionsAll() { ); } - private DenseVectorFieldMapper.IndexOptions randomIndexOptionsHnswQuantized() { + private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsHnswQuantized() { return randomIndexOptionsHnswQuantized(randomBoolean() ? null : randomRescoreVector()); } - private DenseVectorFieldMapper.IndexOptions randomIndexOptionsHnswQuantized(DenseVectorFieldMapper.RescoreVector rescoreVector) { + private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsHnswQuantized( + DenseVectorFieldMapper.RescoreVector rescoreVector + ) { return randomFrom( new DenseVectorFieldMapper.Int8HnswIndexOptions( randomIntBetween(1, 100), diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index 42e2c1b0e7ba9..ba1694d472181 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -16,6 +16,7 @@ import java.util.Set; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS; +import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_INDEX_OPTIONS; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_SUPPORT_CHUNKING_CONFIG; import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_FILTER_FIX; import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED; @@ -62,7 +63,8 @@ public Set getTestFeatures() { TEST_RULE_RETRIEVER_WITH_INDICES_THAT_DONT_RETURN_RANK_DOCS, SEMANTIC_TEXT_SUPPORT_CHUNKING_CONFIG, SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER, - SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS + SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS, + SEMANTIC_TEXT_INDEX_OPTIONS ); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 92337c8e7fc8d..cf9ac1b666b86 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; @@ -137,12 +138,15 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie public static final NodeFeature SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS = new NodeFeature( "semantic_text.exclude_sub_fields_from_field_caps" ); + public static final NodeFeature SEMANTIC_TEXT_INDEX_OPTIONS = new NodeFeature("semantic_text.index_options"); public static final String CONTENT_TYPE = "semantic_text"; public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID; public static final float DEFAULT_RESCORE_OVERSAMPLE = 3.0f; + static final String INDEX_OPTIONS_FIELD = "index_options"; + public static final TypeParser parser(Supplier modelRegistry) { return new TypeParser( (n, c) -> new Builder(n, c::bitSetProducer, c.getIndexSettings(), modelRegistry.get()), @@ -199,6 +203,16 @@ public static class Builder extends FieldMapper.Builder { Objects::toString ).acceptsNull().setMergeValidator(SemanticTextFieldMapper::canMergeModelSettings); + private final Parameter indexOptions = new Parameter<>( + INDEX_OPTIONS_FIELD, + true, + () -> null, + (n, c, o) -> parseIndexOptionsFromMap(n, o, c.indexVersionCreated()), + mapper -> ((SemanticTextFieldType) mapper.fieldType()).indexOptions, + XContentBuilder::field, + Objects::toString + ).acceptsNull(); + @SuppressWarnings("unchecked") private final Parameter chunkingSettings = new Parameter<>( CHUNKING_SETTINGS_FIELD, @@ -242,6 +256,7 @@ public Builder( indexSettings.getIndexVersionCreated(), useLegacyFormat, resolvedModelSettings, + indexOptions.get(), bitSetProducer, indexSettings ); @@ -270,7 +285,7 @@ public Builder setChunkingSettings(ChunkingSettings value) { @Override protected Parameter[] getParameters() { - return new Parameter[] { inferenceId, searchInferenceId, modelSettings, chunkingSettings, meta }; + return new Parameter[] { inferenceId, searchInferenceId, modelSettings, chunkingSettings, indexOptions, meta }; } @Override @@ -278,12 +293,22 @@ protected void merge(FieldMapper mergeWith, Conflicts conflicts, MapperMergeCont SemanticTextFieldMapper semanticMergeWith = (SemanticTextFieldMapper) mergeWith; semanticMergeWith = copySettings(semanticMergeWith, mapperMergeContext); + // We make sure to merge the inference field first to catch any model conflicts + try { + var context = mapperMergeContext.createChildContext(semanticMergeWith.leafName(), ObjectMapper.Dynamic.FALSE); + var inferenceField = inferenceFieldBuilder.apply(context.getMapperBuilderContext()); + var mergedInferenceField = inferenceField.merge(semanticMergeWith.fieldType().getInferenceField(), context); + inferenceFieldBuilder = c -> mergedInferenceField; + } catch (Exception e) { + // Wrap errors in nicer messages that hide inference field internals + String errorMessage = e.getMessage() != null + ? e.getMessage().replaceAll(SemanticTextField.getEmbeddingsFieldName(""), "") + : ""; + throw new IllegalArgumentException(errorMessage, e); + } + super.merge(semanticMergeWith, conflicts, mapperMergeContext); conflicts.check(); - var context = mapperMergeContext.createChildContext(semanticMergeWith.leafName(), ObjectMapper.Dynamic.FALSE); - var inferenceField = inferenceFieldBuilder.apply(context.getMapperBuilderContext()); - var mergedInferenceField = inferenceField.merge(semanticMergeWith.fieldType().getInferenceField(), context); - inferenceFieldBuilder = c -> mergedInferenceField; } /** @@ -340,6 +365,10 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) { validateServiceSettings(modelSettings.get(), resolvedModelSettings); } + if (context.getMergeReason() != MapperService.MergeReason.MAPPING_RECOVERY && indexOptions.get() != null) { + validateIndexOptions(indexOptions.get(), inferenceId.getValue(), resolvedModelSettings); + } + final String fullName = context.buildFullName(leafName()); if (context.isInNestedContext()) { @@ -356,6 +385,7 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) { searchInferenceId.getValue(), modelSettings.getValue(), chunkingSettings.getValue(), + indexOptions.getValue(), inferenceField, useLegacyFormat, meta.getValue() @@ -392,6 +422,33 @@ private void validateServiceSettings(MinimalServiceSettings settings, MinimalSer } } + private void validateIndexOptions(SemanticTextIndexOptions indexOptions, String inferenceId, MinimalServiceSettings modelSettings) { + if (indexOptions == null) { + return; + } + + if (modelSettings == null) { + throw new IllegalArgumentException( + "Model settings must be set to validate index options for inference ID [" + inferenceId + "]" + ); + } + + if (indexOptions.type() == SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR) { + + if (modelSettings.taskType() != TEXT_EMBEDDING) { + throw new IllegalArgumentException( + "Invalid task type for index options, required [" + TEXT_EMBEDDING + "] but was [" + modelSettings.taskType() + "]" + ); + } + + int dims = modelSettings.dimensions() != null ? modelSettings.dimensions() : 0; + DenseVectorFieldMapper.DenseVectorIndexOptions denseVectorIndexOptions = + (DenseVectorFieldMapper.DenseVectorIndexOptions) indexOptions.indexOptions(); + denseVectorIndexOptions.validate(modelSettings.elementType(), dims, true); + } + + } + /** * As necessary, copy settings from this builder to the passed-in mapper. * Used to preserve {@link MinimalServiceSettings} when updating a semantic text mapping to one where the model settings @@ -666,6 +723,7 @@ public static class SemanticTextFieldType extends SimpleMappedFieldType { private final String searchInferenceId; private final MinimalServiceSettings modelSettings; private final ChunkingSettings chunkingSettings; + private final SemanticTextIndexOptions indexOptions; private final ObjectMapper inferenceField; private final boolean useLegacyFormat; @@ -675,6 +733,7 @@ public SemanticTextFieldType( String searchInferenceId, MinimalServiceSettings modelSettings, ChunkingSettings chunkingSettings, + SemanticTextIndexOptions indexOptions, ObjectMapper inferenceField, boolean useLegacyFormat, Map meta @@ -684,6 +743,7 @@ public SemanticTextFieldType( this.searchInferenceId = searchInferenceId; this.modelSettings = modelSettings; this.chunkingSettings = chunkingSettings; + this.indexOptions = indexOptions; this.inferenceField = inferenceField; this.useLegacyFormat = useLegacyFormat; } @@ -723,6 +783,10 @@ public ChunkingSettings getChunkingSettings() { return chunkingSettings; } + public SemanticTextIndexOptions getIndexOptions() { + return indexOptions; + } + public ObjectMapper getInferenceField() { return inferenceField; } @@ -1037,11 +1101,12 @@ private static ObjectMapper createInferenceField( IndexVersion indexVersionCreated, boolean useLegacyFormat, @Nullable MinimalServiceSettings modelSettings, + @Nullable SemanticTextIndexOptions indexOptions, Function bitSetProducer, IndexSettings indexSettings ) { return new ObjectMapper.Builder(INFERENCE_FIELD, Optional.of(ObjectMapper.Subobjects.ENABLED)).dynamic(ObjectMapper.Dynamic.FALSE) - .add(createChunksField(indexVersionCreated, useLegacyFormat, modelSettings, bitSetProducer, indexSettings)) + .add(createChunksField(indexVersionCreated, useLegacyFormat, modelSettings, indexOptions, bitSetProducer, indexSettings)) .build(context); } @@ -1049,6 +1114,7 @@ private static NestedObjectMapper.Builder createChunksField( IndexVersion indexVersionCreated, boolean useLegacyFormat, @Nullable MinimalServiceSettings modelSettings, + @Nullable SemanticTextIndexOptions indexOptions, Function bitSetProducer, IndexSettings indexSettings ) { @@ -1060,7 +1126,7 @@ private static NestedObjectMapper.Builder createChunksField( ); chunksField.dynamic(ObjectMapper.Dynamic.FALSE); if (modelSettings != null) { - chunksField.add(createEmbeddingsField(indexSettings.getIndexVersionCreated(), modelSettings, useLegacyFormat)); + chunksField.add(createEmbeddingsField(indexSettings.getIndexVersionCreated(), modelSettings, indexOptions, useLegacyFormat)); } if (useLegacyFormat) { var chunkTextField = new KeywordFieldMapper.Builder(TEXT_FIELD, indexVersionCreated).indexed(false).docValues(false); @@ -1074,6 +1140,7 @@ private static NestedObjectMapper.Builder createChunksField( private static Mapper.Builder createEmbeddingsField( IndexVersion indexVersionCreated, MinimalServiceSettings modelSettings, + SemanticTextIndexOptions indexOptions, boolean useLegacyFormat ) { return switch (modelSettings.taskType()) { @@ -1097,15 +1164,28 @@ private static Mapper.Builder createEmbeddingsField( } denseVectorMapperBuilder.dimensions(modelSettings.dimensions()); denseVectorMapperBuilder.elementType(modelSettings.elementType()); - - DenseVectorFieldMapper.IndexOptions defaultIndexOptions = null; - if (indexVersionCreated.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BBQ) - || indexVersionCreated.between(SEMANTIC_TEXT_DEFAULTS_TO_BBQ_BACKPORT_8_X, IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) { - defaultIndexOptions = defaultSemanticDenseIndexOptions(); + if (indexOptions != null) { + DenseVectorFieldMapper.DenseVectorIndexOptions denseVectorIndexOptions = + (DenseVectorFieldMapper.DenseVectorIndexOptions) indexOptions.indexOptions(); + denseVectorMapperBuilder.indexOptions(denseVectorIndexOptions); + denseVectorIndexOptions.validate(modelSettings.elementType(), modelSettings.dimensions(), true); + } else { + DenseVectorFieldMapper.DenseVectorIndexOptions defaultIndexOptions = defaultDenseVectorIndexOptions( + indexVersionCreated, + modelSettings + ); + if (defaultIndexOptions != null) { + denseVectorMapperBuilder.indexOptions(defaultIndexOptions); + } } - if (defaultIndexOptions != null - && defaultIndexOptions.validate(modelSettings.elementType(), modelSettings.dimensions(), false)) { - denseVectorMapperBuilder.indexOptions(defaultIndexOptions); + + boolean hasUserSpecifiedIndexOptions = indexOptions != null; + DenseVectorFieldMapper.DenseVectorIndexOptions denseVectorIndexOptions = hasUserSpecifiedIndexOptions + ? (DenseVectorFieldMapper.DenseVectorIndexOptions) indexOptions.indexOptions() + : defaultDenseVectorIndexOptions(indexVersionCreated, modelSettings); + + if (denseVectorIndexOptions != null) { + denseVectorMapperBuilder.indexOptions(denseVectorIndexOptions); } yield denseVectorMapperBuilder; @@ -1114,15 +1194,47 @@ private static Mapper.Builder createEmbeddingsField( }; } - static DenseVectorFieldMapper.IndexOptions defaultSemanticDenseIndexOptions() { + static DenseVectorFieldMapper.DenseVectorIndexOptions defaultDenseVectorIndexOptions( + IndexVersion indexVersionCreated, + MinimalServiceSettings modelSettings + ) { + + if (modelSettings.dimensions() == null) { + return null; // Cannot determine default index options without dimensions + } + // As embedding models for text perform better with BBQ, we aggressively default semantic_text fields to use optimized index - // options outside of dense_vector defaults + // options + if (indexVersionDefaultsToBbqHnsw(indexVersionCreated)) { + + DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswIndexOptions = defaultBbqHnswDenseVectorIndexOptions(); + return defaultBbqHnswIndexOptions.validate(modelSettings.elementType(), modelSettings.dimensions(), false) + ? defaultBbqHnswIndexOptions + : null; + } + + return null; + } + + static boolean indexVersionDefaultsToBbqHnsw(IndexVersion indexVersion) { + return indexVersion.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BBQ) + || indexVersion.between(SEMANTIC_TEXT_DEFAULTS_TO_BBQ_BACKPORT_8_X, IndexVersions.UPGRADE_TO_LUCENE_10_0_0); + } + + static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDenseVectorIndexOptions() { int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector(DEFAULT_RESCORE_OVERSAMPLE); return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector); } + static SemanticTextIndexOptions defaultBbqHnswSemanticTextIndexOptions() { + return new SemanticTextIndexOptions( + SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + defaultBbqHnswDenseVectorIndexOptions() + ); + } + private static boolean canMergeModelSettings(MinimalServiceSettings previous, MinimalServiceSettings current, Conflicts conflicts) { if (previous != null && current != null && previous.canMergeWith(current)) { return true; @@ -1133,4 +1245,23 @@ private static boolean canMergeModelSettings(MinimalServiceSettings previous, Mi conflicts.addConflict("model_settings", ""); return false; } + + private static SemanticTextIndexOptions parseIndexOptionsFromMap(String fieldName, Object node, IndexVersion indexVersion) { + + if (node == null) { + return null; + } + + Map map = XContentMapValues.nodeMapValue(node, INDEX_OPTIONS_FIELD); + if (map.size() != 1) { + throw new IllegalArgumentException("Too many index options provided, found [" + map.keySet() + "]"); + } + Map.Entry entry = map.entrySet().iterator().next(); + SemanticTextIndexOptions.SupportedIndexOptions indexOptions = SemanticTextIndexOptions.SupportedIndexOptions.fromValue( + entry.getKey() + ); + @SuppressWarnings("unchecked") + Map indexOptionsMap = (Map) entry.getValue(); + return new SemanticTextIndexOptions(indexOptions, indexOptions.parseIndexOptions(fieldName, indexOptionsMap, indexVersion)); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextIndexOptions.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextIndexOptions.java new file mode 100644 index 0000000000000..c062adad2f551 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextIndexOptions.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.mapper; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.IndexOptions; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Locale; +import java.util.Map; + +/** + * Represents index options for a semantic_text field. + * We represent semantic_text index_options as nested within their respective type. For example: + * "index_options": { + * "dense_vector": { + * "type": "bbq_hnsw + * } + * } + */ +public class SemanticTextIndexOptions implements ToXContent { + + private static final String TYPE_FIELD = "type"; + + private final SupportedIndexOptions type; + private final IndexOptions indexOptions; + + public SemanticTextIndexOptions(SupportedIndexOptions type, IndexOptions indexOptions) { + this.type = type; + this.indexOptions = indexOptions; + } + + public SupportedIndexOptions type() { + return type; + } + + public IndexOptions indexOptions() { + return indexOptions; + } + + public enum SupportedIndexOptions { + DENSE_VECTOR("dense_vector") { + @Override + public IndexOptions parseIndexOptions(String fieldName, Map map, IndexVersion indexVersion) { + return parseDenseVectorIndexOptionsFromMap(fieldName, map, indexVersion); + } + }; + + public final String value; + + SupportedIndexOptions(String value) { + this.value = value; + } + + public abstract IndexOptions parseIndexOptions(String fieldName, Map map, IndexVersion indexVersion); + + public static SupportedIndexOptions fromValue(String value) { + return Arrays.stream(SupportedIndexOptions.values()) + .filter(option -> option.value.equals(value)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unknown index options type [" + value + "]")); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(type.value.toLowerCase(Locale.ROOT)); + indexOptions.toXContent(builder, params); + builder.endObject(); + return builder; + } + + @Override + public String toString() { + return Strings.toString(this); + } + + private static DenseVectorFieldMapper.DenseVectorIndexOptions parseDenseVectorIndexOptionsFromMap( + String fieldName, + Map map, + IndexVersion indexVersion + ) { + try { + Object type = map.remove(TYPE_FIELD); + if (type == null) { + throw new IllegalArgumentException("Required " + TYPE_FIELD); + } + DenseVectorFieldMapper.VectorIndexType vectorIndexType = DenseVectorFieldMapper.VectorIndexType.fromString( + XContentMapValues.nodeStringValue(type, null) + ).orElseThrow(() -> new IllegalArgumentException("Unsupported index options " + TYPE_FIELD + " " + type)); + + return vectorIndexType.parseIndexOptions(fieldName, map, indexVersion); + } catch (Exception exc) { + throw new ElasticsearchException(exc); + } + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java index f59ca26a2f2ad..ca19ef0171825 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java @@ -164,5 +164,4 @@ static IndexVersion getRandomCompatibleIndexVersion(boolean useLegacyFormat) { ); } } - } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 747107a764e42..0fab22d45d08c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -53,6 +53,7 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldTypeTests; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.XFeatureField; import org.elasticsearch.index.query.SearchExecutionContext; @@ -87,11 +88,13 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Supplier; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldTypeTests.randomIndexOptionsAll; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKED_EMBEDDINGS_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKS_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_FIELD; @@ -102,6 +105,8 @@ import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_ELSER_2_INFERENCE_ID; +import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_RESCORE_OVERSAMPLE; +import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.INDEX_OPTIONS_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.generateRandomChunkingSettings; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.generateRandomChunkingSettingsOtherThan; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticText; @@ -184,6 +189,17 @@ private static void validateIndexVersion(IndexVersion indexVersion, boolean useL } } + private MapperService createMapperService(String mappings, boolean useLegacyFormat) throws IOException { + var settings = Settings.builder() + .put( + IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), + SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat) + ) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat) + .build(); + return createMapperService(settings, mappings); + } + @Override protected Settings getIndexSettings() { return Settings.builder() @@ -239,7 +255,17 @@ protected IngestScriptSupport ingestScriptSupport() { @Override public MappedFieldType getMappedFieldType() { - return new SemanticTextFieldMapper.SemanticTextFieldType("field", "fake-inference-id", null, null, null, null, false, Map.of()); + return new SemanticTextFieldMapper.SemanticTextFieldType( + "field", + "fake-inference-id", + null, + null, + null, + null, + null, + false, + Map.of() + ); } @Override @@ -257,7 +283,7 @@ public void testDefaults() throws Exception { MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat); DocumentMapper mapper = mapperService.documentMapper(); assertEquals(Strings.toString(expectedMapping), mapper.mappingSource().toString()); - assertSemanticTextField(mapperService, fieldName, false); + assertSemanticTextField(mapperService, fieldName, false, null, null); assertInferenceEndpoints(mapperService, fieldName, DEFAULT_ELSER_2_INFERENCE_ID, DEFAULT_ELSER_2_INFERENCE_ID); ParsedDocument doc1 = mapper.parse(source(this::writeField)); @@ -287,7 +313,7 @@ public void testSetInferenceEndpoints() throws IOException { { final XContentBuilder fieldMapping = fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, inferenceId)); final MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat); - assertSemanticTextField(mapperService, fieldName, false); + assertSemanticTextField(mapperService, fieldName, false, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); assertSerialization.accept(fieldMapping, mapperService); } @@ -301,7 +327,7 @@ public void testSetInferenceEndpoints() throws IOException { .field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId) ); final MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat); - assertSemanticTextField(mapperService, fieldName, false); + assertSemanticTextField(mapperService, fieldName, false, null, null); assertInferenceEndpoints(mapperService, fieldName, DEFAULT_ELSER_2_INFERENCE_ID, searchInferenceId); assertSerialization.accept(expectedMapping, mapperService); } @@ -312,7 +338,7 @@ public void testSetInferenceEndpoints() throws IOException { .field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId) ); MapperService mapperService = createMapperService(fieldMapping, useLegacyFormat); - assertSemanticTextField(mapperService, fieldName, false); + assertSemanticTextField(mapperService, fieldName, false, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId); assertSerialization.accept(fieldMapping, mapperService); } @@ -372,7 +398,7 @@ public void testInvalidTaskTypes() { useLegacyFormat ) ); - assertThat(e.getMessage(), containsString("Failed to parse mapping: Wrong [task_type]")); + assertThat(e.getMessage(), containsString("Wrong [task_type], expected text_embedding or sparse_embedding")); } } @@ -401,7 +427,7 @@ public void testMultiFieldsSupport() throws IOException { b.endObject(); b.endObject(); }), useLegacyFormat); - assertSemanticTextField(mapperService, "field.semantic", true); + assertSemanticTextField(mapperService, "field.semantic", true, null, null); mapperService = createMapperService(fieldMapping(b -> { b.field("type", "semantic_text"); @@ -415,7 +441,7 @@ public void testMultiFieldsSupport() throws IOException { b.endObject(); b.endObject(); }), useLegacyFormat); - assertSemanticTextField(mapperService, "field", true); + assertSemanticTextField(mapperService, "field", true, null, null); mapperService = createMapperService(fieldMapping(b -> { b.field("type", "semantic_text"); @@ -433,8 +459,8 @@ public void testMultiFieldsSupport() throws IOException { b.endObject(); b.endObject(); }), useLegacyFormat); - assertSemanticTextField(mapperService, "field", true); - assertSemanticTextField(mapperService, "field.semantic", true); + assertSemanticTextField(mapperService, "field", true, null, null); + assertSemanticTextField(mapperService, "field.semantic", true, null, null); Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { b.field("type", "semantic_text"); @@ -456,7 +482,7 @@ public void testUpdatesToInferenceIdNotSupported() throws IOException { mapping(b -> b.startObject(fieldName).field("type", "semantic_text").field("inference_id", "test_model").endObject()), useLegacyFormat ); - assertSemanticTextField(mapperService, fieldName, false); + assertSemanticTextField(mapperService, fieldName, false, null, null); Exception e = expectThrows( IllegalArgumentException.class, () -> merge( @@ -478,7 +504,7 @@ public void testDynamicUpdate() throws IOException { inferenceId, new MinimalServiceSettings("service", TaskType.SPARSE_EMBEDDING, null, null, null) ); - assertSemanticTextField(mapperService, fieldName, true); + assertSemanticTextField(mapperService, fieldName, true, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } @@ -489,7 +515,7 @@ public void testDynamicUpdate() throws IOException { searchInferenceId, new MinimalServiceSettings("service", TaskType.SPARSE_EMBEDDING, null, null, null) ); - assertSemanticTextField(mapperService, fieldName, true); + assertSemanticTextField(mapperService, fieldName, true, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId); } } @@ -501,7 +527,7 @@ public void testUpdateModelSettings() throws IOException { mapping(b -> b.startObject(fieldName).field("type", "semantic_text").field("inference_id", "test_model").endObject()), useLegacyFormat ); - assertSemanticTextField(mapperService, fieldName, false); + assertSemanticTextField(mapperService, fieldName, false, null, null); { Exception exc = expectThrows( MapperParsingException.class, @@ -533,14 +559,14 @@ public void testUpdateModelSettings() throws IOException { .endObject() ) ); - assertSemanticTextField(mapperService, fieldName, true); + assertSemanticTextField(mapperService, fieldName, true, null, null); } { merge( mapperService, mapping(b -> b.startObject(fieldName).field("type", "semantic_text").field("inference_id", "test_model").endObject()) ); - assertSemanticTextField(mapperService, fieldName, true); + assertSemanticTextField(mapperService, fieldName, true, null, null); } { Exception exc = expectThrows( @@ -561,18 +587,33 @@ public void testUpdateModelSettings() throws IOException { ) ) ); - assertThat( - exc.getMessage(), - containsString( - "Cannot update parameter [model_settings] " - + "from [service=null, task_type=sparse_embedding] " - + "to [service=null, task_type=text_embedding, dimensions=10, similarity=cosine, element_type=float]" - ) - ); + assertThat(exc.getMessage(), containsString("cannot be changed from type [sparse_vector] to [dense_vector]")); } } } + public void testDenseVectorIndexOptionValidation() throws IOException { + for (int depth = 1; depth < 5; depth++) { + String inferenceId = "test_model"; + String fieldName = randomFieldName(depth); + + DenseVectorFieldMapper.DenseVectorIndexOptions indexOptions = DenseVectorFieldTypeTests.randomIndexOptionsAll(); + Exception exc = expectThrows(MapperParsingException.class, () -> createMapperService(mapping(b -> { + b.startObject(fieldName); + b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); + b.field(INFERENCE_ID_FIELD, inferenceId); + b.startObject(INDEX_OPTIONS_FIELD); + b.startObject("dense_vector"); + b.field("type", indexOptions.getType().name().toLowerCase(Locale.ROOT)); + b.field("unsupported_param", "any_value"); + b.endObject(); + b.endObject(); + b.endObject(); + }), useLegacyFormat)); + assertTrue(exc.getMessage().contains("unsupported parameters")); + } + } + public void testUpdateSearchInferenceId() throws IOException { final String inferenceId = "test_inference_id"; final String searchInferenceId1 = "test_search_inference_id_1"; @@ -589,19 +630,19 @@ public void testUpdateSearchInferenceId() throws IOException { for (int depth = 1; depth < 5; depth++) { String fieldName = randomFieldName(depth); MapperService mapperService = createMapperService(buildMapping.apply(fieldName, null), useLegacyFormat); - assertSemanticTextField(mapperService, fieldName, false); + assertSemanticTextField(mapperService, fieldName, false, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId1)); - assertSemanticTextField(mapperService, fieldName, false); + assertSemanticTextField(mapperService, fieldName, false, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId2)); - assertSemanticTextField(mapperService, fieldName, false); + assertSemanticTextField(mapperService, fieldName, false, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); merge(mapperService, buildMapping.apply(fieldName, null)); - assertSemanticTextField(mapperService, fieldName, false); + assertSemanticTextField(mapperService, fieldName, false, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); mapperService = mapperServiceForFieldWithModelSettings( @@ -609,19 +650,19 @@ public void testUpdateSearchInferenceId() throws IOException { inferenceId, new MinimalServiceSettings("my-service", TaskType.SPARSE_EMBEDDING, null, null, null) ); - assertSemanticTextField(mapperService, fieldName, true); + assertSemanticTextField(mapperService, fieldName, true, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId1)); - assertSemanticTextField(mapperService, fieldName, true); + assertSemanticTextField(mapperService, fieldName, true, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId2)); - assertSemanticTextField(mapperService, fieldName, true); + assertSemanticTextField(mapperService, fieldName, true, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); merge(mapperService, buildMapping.apply(fieldName, null)); - assertSemanticTextField(mapperService, fieldName, true); + assertSemanticTextField(mapperService, fieldName, true, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } } @@ -635,7 +676,7 @@ private static void assertSemanticTextField( String fieldName, boolean expectedModelSettings, ChunkingSettings expectedChunkingSettings, - DenseVectorFieldMapper.IndexOptions expectedIndexOptions + SemanticTextIndexOptions expectedIndexOptions ) { Mapper mapper = mapperService.mappingLookup().getMapper(fieldName); assertNotNull(mapper); @@ -646,7 +687,7 @@ private static void assertSemanticTextField( assertNotNull(fieldType); assertThat(fieldType, instanceOf(SemanticTextFieldMapper.SemanticTextFieldType.class)); SemanticTextFieldMapper.SemanticTextFieldType semanticTextFieldType = (SemanticTextFieldMapper.SemanticTextFieldType) fieldType; - assertTrue(semanticFieldMapper.fieldType() == semanticTextFieldType); + assertSame(semanticFieldMapper.fieldType(), semanticTextFieldType); NestedObjectMapper chunksMapper = mapperService.mappingLookup() .nestedLookup() @@ -674,7 +715,7 @@ private static void assertSemanticTextField( assertNotNull(embeddingsMapper); assertThat(embeddingsMapper, instanceOf(FieldMapper.class)); FieldMapper embeddingsFieldMapper = (FieldMapper) embeddingsMapper; - assertTrue(embeddingsFieldMapper.fieldType() == mapperService.mappingLookup().getFieldType(getEmbeddingsFieldName(fieldName))); + assertSame(embeddingsFieldMapper.fieldType(), mapperService.mappingLookup().getFieldType(getEmbeddingsFieldName(fieldName))); assertThat(embeddingsMapper.fullPath(), equalTo(getEmbeddingsFieldName(fieldName))); switch (semanticFieldMapper.fieldType().getModelSettings().taskType()) { case SPARSE_EMBEDDING -> { @@ -687,7 +728,7 @@ private static void assertSemanticTextField( assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class)); DenseVectorFieldMapper denseVectorFieldMapper = (DenseVectorFieldMapper) embeddingsMapper; if (expectedIndexOptions != null) { - assertEquals(expectedIndexOptions, denseVectorFieldMapper.fieldType().getIndexOptions()); + assertEquals(expectedIndexOptions.indexOptions(), denseVectorFieldMapper.fieldType().getIndexOptions()); } else { assertNull(denseVectorFieldMapper.fieldType().getIndexOptions()); } @@ -727,35 +768,39 @@ public void testSuccessfulParse() throws IOException { final String searchInferenceId = randomAlphaOfLength(8); final boolean setSearchInferenceId = randomBoolean(); - Model model1 = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); - Model model2 = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); + TaskType taskType = TaskType.SPARSE_EMBEDDING; + Model model1 = TestModel.createRandomInstance(taskType); + Model model2 = TestModel.createRandomInstance(taskType); ChunkingSettings chunkingSettings = null; // Some chunking settings configs can produce different Lucene docs counts + SemanticTextIndexOptions indexOptions = randomSemanticTextIndexOptions(taskType); XContentBuilder mapping = mapping(b -> { addSemanticTextMapping( b, fieldName1, model1.getInferenceEntityId(), setSearchInferenceId ? searchInferenceId : null, - chunkingSettings + chunkingSettings, + indexOptions ); addSemanticTextMapping( b, fieldName2, model2.getInferenceEntityId(), setSearchInferenceId ? searchInferenceId : null, - chunkingSettings + chunkingSettings, + indexOptions ); }); MapperService mapperService = createMapperService(mapping, useLegacyFormat); - assertSemanticTextField(mapperService, fieldName1, false); + assertSemanticTextField(mapperService, fieldName1, false, null, null); assertInferenceEndpoints( mapperService, fieldName1, model1.getInferenceEntityId(), setSearchInferenceId ? searchInferenceId : model1.getInferenceEntityId() ); - assertSemanticTextField(mapperService, fieldName2, false); + assertSemanticTextField(mapperService, fieldName2, false, null, null); assertInferenceEndpoints( mapperService, fieldName2, @@ -859,7 +904,7 @@ public void testSuccessfulParse() throws IOException { public void testMissingInferenceId() throws IOException { final MapperService mapperService = createMapperService( - mapping(b -> addSemanticTextMapping(b, "field", "my_id", null, null)), + mapping(b -> addSemanticTextMapping(b, "field", "my_id", null, null, null)), useLegacyFormat ); @@ -887,7 +932,7 @@ public void testMissingInferenceId() throws IOException { public void testMissingModelSettingsAndChunks() throws IOException { MapperService mapperService = createMapperService( - mapping(b -> addSemanticTextMapping(b, "field", "my_id", null, null)), + mapping(b -> addSemanticTextMapping(b, "field", "my_id", null, null, null)), useLegacyFormat ); IllegalArgumentException ex = expectThrows( @@ -907,7 +952,7 @@ public void testMissingModelSettingsAndChunks() throws IOException { public void testMissingTaskType() throws IOException { MapperService mapperService = createMapperService( - mapping(b -> addSemanticTextMapping(b, "field", "my_id", null, null)), + mapping(b -> addSemanticTextMapping(b, "field", "my_id", null, null, null)), useLegacyFormat ); IllegalArgumentException ex = expectThrows( @@ -971,6 +1016,7 @@ public void testDenseVectorElementType() throws IOException { public void testSettingAndUpdatingChunkingSettings() throws IOException { Model model = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); final ChunkingSettings chunkingSettings = generateRandomChunkingSettings(false); + final SemanticTextIndexOptions indexOptions = null; String fieldName = "field"; SemanticTextField randomSemanticText = randomSemanticText( @@ -983,20 +1029,25 @@ public void testSettingAndUpdatingChunkingSettings() throws IOException { ); MapperService mapperService = createMapperService( - mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, chunkingSettings)), + mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, chunkingSettings, indexOptions)), useLegacyFormat ); assertSemanticTextField(mapperService, fieldName, false, chunkingSettings, null); ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings); - merge(mapperService, mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, newChunkingSettings))); - assertSemanticTextField(mapperService, fieldName, false, newChunkingSettings, null); + merge( + mapperService, + mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, newChunkingSettings, indexOptions)) + ); + assertSemanticTextField(mapperService, fieldName, false, newChunkingSettings, indexOptions); } public void testModelSettingsRequiredWithChunks() throws IOException { // Create inference results where model settings are set to null and chunks are provided - Model model = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); + TaskType taskType = TaskType.SPARSE_EMBEDDING; + Model model = TestModel.createRandomInstance(taskType); ChunkingSettings chunkingSettings = generateRandomChunkingSettings(false); + SemanticTextIndexOptions indexOptions = randomSemanticTextIndexOptions(taskType); SemanticTextField randomSemanticText = randomSemanticText( useLegacyFormat, "field", @@ -1019,7 +1070,7 @@ public void testModelSettingsRequiredWithChunks() throws IOException { ); MapperService mapperService = createMapperService( - mapping(b -> addSemanticTextMapping(b, "field", model.getInferenceEntityId(), null, chunkingSettings)), + mapping(b -> addSemanticTextMapping(b, "field", model.getInferenceEntityId(), null, chunkingSettings, indexOptions)), useLegacyFormat ); SourceToParse source = source(b -> addSemanticTextInferenceResults(useLegacyFormat, b, List.of(inferenceResults))); @@ -1120,13 +1171,31 @@ public void testExistsQueryDenseVector() throws IOException { assertThat(existsQuery, instanceOf(ESToParentBlockJoinQuery.class)); } - private static DenseVectorFieldMapper.IndexOptions defaultDenseVectorIndexOptions() { + private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultDenseVectorIndexOptions() { // These are the default index options for dense_vector fields, and used for semantic_text fields incompatible with BBQ. int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; return new DenseVectorFieldMapper.Int8HnswIndexOptions(m, efConstruction, null, null); } + private static SemanticTextIndexOptions defaultDenseVectorSemanticIndexOptions() { + return new SemanticTextIndexOptions(SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, defaultDenseVectorIndexOptions()); + } + + private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDenseVectorIndexOptions() { + int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; + int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector(DEFAULT_RESCORE_OVERSAMPLE); + return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector); + } + + private static SemanticTextIndexOptions defaultBbqHnswSemanticTextIndexOptions() { + return new SemanticTextIndexOptions( + SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + defaultBbqHnswDenseVectorIndexOptions() + ); + } + public void testDefaultIndexOptions() throws IOException { // We default to BBQ for eligible dense vectors @@ -1140,7 +1209,7 @@ public void testDefaultIndexOptions() throws IOException { b.field("element_type", "float"); b.endObject(); }), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ); - assertSemanticTextField(mapperService, "field", true, null, SemanticTextFieldMapper.defaultSemanticDenseIndexOptions()); + assertSemanticTextField(mapperService, "field", true, null, defaultBbqHnswSemanticTextIndexOptions()); // Element types that are incompatible with BBQ will continue to use dense_vector defaults mapperService = createMapperService(fieldMapping(b -> { @@ -1166,7 +1235,42 @@ public void testDefaultIndexOptions() throws IOException { b.field("element_type", "float"); b.endObject(); }), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ); - assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorIndexOptions()); + assertSemanticTextField( + mapperService, + "field", + true, + null, + new SemanticTextIndexOptions(SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, defaultDenseVectorIndexOptions()) + ); + + // If we explicitly set index options, we respect those over the defaults + mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "semantic_text"); + b.field("inference_id", "another_inference_id"); + b.startObject("model_settings"); + b.field("task_type", "text_embedding"); + b.field("dimensions", 100); + b.field("similarity", "cosine"); + b.field("element_type", "float"); + b.endObject(); + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "int4_hnsw"); + b.field("m", 25); + b.field("ef_construction", 100); + b.endObject(); + b.endObject(); + }), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ); + assertSemanticTextField( + mapperService, + "field", + true, + null, + new SemanticTextIndexOptions( + SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new DenseVectorFieldMapper.Int4HnswIndexOptions(25, 100, null, null) + ) + ); // Previous index versions do not set BBQ index options mapperService = createMapperService(fieldMapping(b -> { @@ -1183,7 +1287,7 @@ public void testDefaultIndexOptions() throws IOException { IndexVersions.INFERENCE_METADATA_FIELDS, IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ) ); - assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorIndexOptions()); + assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorSemanticIndexOptions()); // 8.x index versions that use backported default BBQ set default BBQ index options as expected mapperService = createMapperService(fieldMapping(b -> { @@ -1196,7 +1300,7 @@ public void testDefaultIndexOptions() throws IOException { b.field("element_type", "float"); b.endObject(); }), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ_BACKPORT_8_X, IndexVersions.UPGRADE_TO_LUCENE_10_0_0); - assertSemanticTextField(mapperService, "field", true, null, SemanticTextFieldMapper.defaultSemanticDenseIndexOptions()); + assertSemanticTextField(mapperService, "field", true, null, defaultBbqHnswSemanticTextIndexOptions()); // Previous 8.x index versions do not set BBQ index options mapperService = createMapperService(fieldMapping(b -> { @@ -1213,7 +1317,134 @@ public void testDefaultIndexOptions() throws IOException { IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ_BACKPORT_8_X) ); - assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorIndexOptions()); + assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorSemanticIndexOptions()); + } + + public void testSpecifiedDenseVectorIndexOptions() throws IOException { + + // Specifying index options will override default index option settings + var mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "semantic_text"); + b.field("inference_id", "another_inference_id"); + b.startObject("model_settings"); + b.field("task_type", "text_embedding"); + b.field("dimensions", 100); + b.field("similarity", "cosine"); + b.field("element_type", "float"); + b.endObject(); + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "int4_hnsw"); + b.field("m", 20); + b.field("ef_construction", 90); + b.field("confidence_interval", 0.4); + b.endObject(); + b.endObject(); + }), useLegacyFormat, IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); + assertSemanticTextField( + mapperService, + "field", + true, + null, + new SemanticTextIndexOptions( + SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, 0.4f, null) + ) + ); + + // Specifying partial index options will in the remainder index options with defaults + mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "semantic_text"); + b.field("inference_id", "another_inference_id"); + b.startObject("model_settings"); + b.field("task_type", "text_embedding"); + b.field("dimensions", 100); + b.field("similarity", "cosine"); + b.field("element_type", "float"); + b.endObject(); + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "int4_hnsw"); + b.endObject(); + b.endObject(); + }), useLegacyFormat, IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); + assertSemanticTextField( + mapperService, + "field", + true, + null, + new SemanticTextIndexOptions( + SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, 0f, null) + ) + ); + + // Incompatible index options will fail + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { + b.field("type", "semantic_text"); + b.field("inference_id", "another_inference_id"); + b.startObject("model_settings"); + b.field("task_type", "sparse_embedding"); + b.endObject(); + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "int8_hnsw"); + b.endObject(); + b.endObject(); + }), useLegacyFormat, IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + assertThat(e.getMessage(), containsString("Invalid task type")); + + e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { + b.field("type", "semantic_text"); + b.field("inference_id", "another_inference_id"); + b.startObject("model_settings"); + b.field("task_type", "text_embedding"); + b.field("dimensions", 100); + b.field("similarity", "cosine"); + b.field("element_type", "float"); + b.endObject(); + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "bbq_flat"); + b.field("ef_construction", 100); + b.endObject(); + b.endObject(); + }), useLegacyFormat, IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + assertThat(e.getMessage(), containsString("unsupported parameters: [ef_construction : 100]")); + + e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { + b.field("type", "semantic_text"); + b.field("inference_id", "another_inference_id"); + b.startObject("model_settings"); + b.field("task_type", "text_embedding"); + b.field("dimensions", 100); + b.field("similarity", "cosine"); + b.field("element_type", "float"); + b.endObject(); + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "invalid"); + b.endObject(); + b.endObject(); + }), useLegacyFormat, IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + assertThat(e.getMessage(), containsString("Unsupported index options type invalid")); + + } + + public static SemanticTextIndexOptions randomSemanticTextIndexOptions() { + TaskType taskType = randomFrom(TaskType.SPARSE_EMBEDDING, TaskType.TEXT_EMBEDDING); + return randomSemanticTextIndexOptions(taskType); + } + + public static SemanticTextIndexOptions randomSemanticTextIndexOptions(TaskType taskType) { + + if (taskType == TaskType.TEXT_EMBEDDING) { + return randomBoolean() + ? null + : new SemanticTextIndexOptions(SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, randomIndexOptionsAll()); + } + + return null; } @Override @@ -1227,7 +1458,8 @@ private static void addSemanticTextMapping( String fieldName, String inferenceId, String searchInferenceId, - ChunkingSettings chunkingSettings + ChunkingSettings chunkingSettings, + SemanticTextIndexOptions indexOptions ) throws IOException { mappingBuilder.startObject(fieldName); mappingBuilder.field("type", SemanticTextFieldMapper.CONTENT_TYPE); @@ -1240,6 +1472,10 @@ private static void addSemanticTextMapping( mappingBuilder.mapContents(chunkingSettings.asMap()); mappingBuilder.endObject(); } + if (indexOptions != null) { + mappingBuilder.field(INDEX_OPTIONS_FIELD); + indexOptions.toXContent(mappingBuilder, null); + } mappingBuilder.endObject(); } diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml index a1c2663b22cc9..5cc0d83685169 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml @@ -3,6 +3,56 @@ setup: cluster_features: "gte_v8.15.0" reason: semantic_text introduced in 8.15.0 + - do: + inference.put: + task_type: sparse_embedding + inference_id: sparse-inference-id + body: > + { + "service": "test_service", + "service_settings": { + "model": "my_model", + "api_key": "abc64" + }, + "task_settings": { + } + } + + - do: + inference.put: + task_type: text_embedding + inference_id: dense-inference-id + body: > + { + "service": "text_embedding_test_service", + "service_settings": { + "model": "my_model", + "dimensions": 4, + "similarity": "cosine", + "api_key": "abc64" + }, + "task_settings": { + } + } + + - do: + inference.put: + task_type: text_embedding + inference_id: dense-inference-id-compatible-with-bbq + body: > + { + "service": "text_embedding_test_service", + "service_settings": { + "model": "my_model", + "dimensions": 64, + "similarity": "cosine", + "api_key": "abc64" + }, + "task_settings": { + } + } + + - do: indices.create: index: test-index @@ -157,7 +207,7 @@ setup: - match: { "test-index.mappings.properties.dense_field.type": semantic_text } - match: { "test-index.mappings.properties.dense_field.inference_id": dense-inference-id } - - length: { "test-index.mappings.properties.dense_field": 2 } + - not_exists: test-index.mappings.properties.dense_field.model_settings - do: index: @@ -177,10 +227,10 @@ setup: dense_field: - start_offset: 0 end_offset: 44 - embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416] + embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ] - start_offset: 44 end_offset: 67 - embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896] + embeddings: [ 0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896 ] # Checks mapping is updated when first doc arrives - do: @@ -190,7 +240,72 @@ setup: - match: { "test-index.mappings.properties.dense_field.type": semantic_text } - match: { "test-index.mappings.properties.dense_field.inference_id": dense-inference-id } - match: { "test-index.mappings.properties.dense_field.model_settings.task_type": text_embedding } - - length: { "test-index.mappings.properties.dense_field": 3 } + - exists: test-index.mappings.properties.dense_field.model_settings + +--- +"Indexes dense vector document with bbq compatible model": + - requires: + cluster_features: "semantic_text.index_options" + reason: index_options introduced in 8.19.0 + + - do: + indices.create: + index: test-index-options-with-bbq + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: false + mappings: + properties: + dense_field: + type: semantic_text + inference_id: dense-inference-id-compatible-with-bbq + + # Checks vector mapping is not updated until first doc arrives + - do: + indices.get_mapping: + index: test-index-options-with-bbq + + - match: { "test-index-options-with-bbq.mappings.properties.dense_field.type": semantic_text } + - match: { "test-index-options-with-bbq.mappings.properties.dense_field.inference_id": dense-inference-id-compatible-with-bbq } + - not_exists: test-index-options-with-bbq.mappings.properties.dense_field.index_options + - not_exists: test-index-options-with-bbq.mappings.properties.dense_field.model_settings + + - do: + index: + index: test-index-options-with-bbq + id: doc_2 + body: + dense_field: "these are not the droids you're looking for. He's free to go around" + _inference_fields.dense_field: + inference: + inference_id: dense-inference-id-compatible-with-bbq + model_settings: + task_type: text_embedding + dimensions: 64 + similarity: cosine + element_type: float + chunks: + dense_field: + - start_offset: 0 + end_offset: 44 + embeddings: [ 0.05, -0.03, -0.03, 0.06, 0.01, -0.02, 0.07, 0.02, -0.04, 0.03, 0.00, 0.05, -0.06, 0.04, -0.01, 0.02, -0.05, 0.01, 0.03, -0.02, 0.06, -0.04, 0.00, 0.05, -0.03, 0.02, 0.01, -0.01, 0.04, -0.06, 0.03, 0.02, -0.02, 0.06, -0.01, 0.00, 0.04, -0.05, 0.01, 0.03, -0.04, 0.02, -0.03, 0.05, -0.02, 0.01, 0.03, -0.06, 0.04, 0.00, -0.01, 0.06, -0.03, 0.02, 0.01, -0.04, 0.05, -0.01, 0.00, 0.04, -0.05, 0.02, 0.03, -0.02 ] + - start_offset: 44 + end_offset: 67 + embeddings: [ 0.05, -0.03, -0.03, 0.06, 0.01, -0.02, 0.07, 0.02, -0.04, 0.03, 0.00, 0.05, -0.06, 0.04, -0.01, 0.02, -0.05, 0.01, 0.03, -0.02, 0.06, -0.04, 0.00, 0.05, -0.03, 0.02, 0.01, -0.01, 0.04, -0.06, 0.03, 0.02, -0.02, 0.06, -0.01, 0.00, 0.04, -0.05, 0.01, 0.03, -0.04, 0.02, -0.03, 0.05, -0.02, 0.01, 0.03, -0.06, 0.04, 0.00, -0.01, 0.06, -0.03, 0.02, 0.01, -0.04, 0.05, -0.01, 0.00, 0.04, -0.05, 0.02, 0.03, -0.02 ] + + + # Checks mapping is updated when first doc arrives + - do: + indices.get_mapping: + index: test-index-options-with-bbq + + - match: { "test-index-options-with-bbq.mappings.properties.dense_field.type": semantic_text } + - match: { "test-index-options-with-bbq.mappings.properties.dense_field.inference_id": dense-inference-id-compatible-with-bbq } + - match: { "test-index-options-with-bbq.mappings.properties.dense_field.model_settings.task_type": text_embedding } + - not_exists: test-index-options-with-bbq.mappings.properties.dense_field.index_options --- "Field caps with text embedding": @@ -236,10 +351,10 @@ setup: dense_field: - start_offset: 0 end_offset: 44 - embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416] + embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ] - start_offset: 44 end_offset: 67 - embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896] + embeddings: [ 0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896 ] refresh: true - do: @@ -268,43 +383,43 @@ setup: --- "Cannot be used directly as a nested field": - - do: - catch: /semantic_text field \[nested.semantic\] cannot be nested/ - indices.create: - index: test-nested-index - body: - mappings: - properties: - nested: - type: nested - properties: - semantic: - type: semantic_text - inference_id: sparse-inference-id - another_field: - type: keyword + - do: + catch: /semantic_text field \[nested.semantic\] cannot be nested/ + indices.create: + index: test-nested-index + body: + mappings: + properties: + nested: + type: nested + properties: + semantic: + type: semantic_text + inference_id: sparse-inference-id + another_field: + type: keyword --- "Cannot be used as a nested field on nested objects": - - do: - catch: /semantic_text field \[nested.nested_object.semantic\] cannot be nested/ - indices.create: - index: test-nested-index - body: - mappings: - properties: - nested: - type: nested - properties: - nested_object: - type: object - properties: - semantic: - type: semantic_text - inference_id: sparse-inference-id - another_field: - type: keyword + - do: + catch: /semantic_text field \[nested.nested_object.semantic\] cannot be nested/ + indices.create: + index: test-nested-index + body: + mappings: + properties: + nested: + type: nested + properties: + nested_object: + type: object + properties: + semantic: + type: semantic_text + inference_id: sparse-inference-id + another_field: + type: keyword --- "Cannot be in an object field with subobjects disabled": @@ -339,11 +454,11 @@ setup: - requires: cluster_features: "semantic_text.always_emit_inference_id_fix" reason: always emit inference ID fix added in 8.17.0 - test_runner_features: [capabilities] + test_runner_features: [ capabilities ] capabilities: - method: GET path: /_inference - capabilities: [default_elser_2] + capabilities: [ default_elser_2 ] - do: indices.create: @@ -432,3 +547,289 @@ setup: - not_exists: fields.dense_field.inference.chunks.offset - not_exists: fields.dense_field.inference.chunks - not_exists: fields.dense_field.inference + +--- +"Users can set dense vector index options and index documents using those options": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + indices.create: + index: test-index-options + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: false + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: int8_hnsw + m: 20 + ef_construction: 100 + confidence_interval: 1.0 + + - do: + indices.get_mapping: + index: test-index-options + + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.type": "int8_hnsw" } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.m": 20 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.ef_construction": 100 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.confidence_interval": 1.0 } + + - do: + index: + index: test-index-options + id: doc_1 + body: + semantic_field: "these are not the droids you're looking for. He's free to go around" + _inference_fields.semantic_field: + inference: + inference_id: dense-inference-id + model_settings: + task_type: text_embedding + dimensions: 4 + similarity: cosine + element_type: float + chunks: + semantic_field: + - start_offset: 0 + end_offset: 44 + embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ] + - start_offset: 44 + end_offset: 67 + embeddings: [ 0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896 ] + + - do: + indices.get_mapping: + index: test-index-options + + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.type": int8_hnsw } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.m": 20 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.ef_construction": 100 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.confidence_interval": 1.0 } + +--- +"Specifying incompatible dense vector index options will fail": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + catch: /unsupported parameters/ + indices.create: + index: test-incompatible-index-options + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: false + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: bbq_flat + ef_construction: 100 + +--- +"Specifying unsupported index option types will fail": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + catch: /Unsupported index options type/ + indices.create: + index: test-invalid-index-options-dense + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: false + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: foo + - do: + catch: bad_request + indices.create: + index: test-invalid-index-options-sparse + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: false + mappings: + properties: + semantic_field: + type: semantic_text + index_options: + sparse_vector: + type: int8_hnsw + +--- +"Index option type is required": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + catch: /Required type/ + indices.create: + index: test-invalid-index-options-dense + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: false + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + foo: bar + +--- +"Specifying index options requires model information": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + catch: /Model settings must be set to validate index options/ + indices.create: + index: my-custom-semantic-index + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: false + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: nonexistent-inference-id + index_options: + dense_vector: + type: int8_hnsw + + - match: { status: 400 } + + - do: + indices.create: + index: my-custom-semantic-index + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: false + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: nonexistent-inference-id + + - do: + indices.get_mapping: + index: my-custom-semantic-index + + - match: { "my-custom-semantic-index.mappings.properties.semantic_field.type": semantic_text } + - match: { "my-custom-semantic-index.mappings.properties.semantic_field.inference_id": nonexistent-inference-id } + - not_exists: my-custom-semantic-index.mappings.properties.semantic_field.index_options + +--- +"Updating index options": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + indices.create: + index: test-index-options + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: false + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: int8_hnsw + m: 16 + ef_construction: 100 + confidence_interval: 1.0 + + - do: + indices.get_mapping: + index: test-index-options + + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.type": "int8_hnsw" } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.m": 16 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.ef_construction": 100 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.confidence_interval": 1.0 } + + - do: + indices.put_mapping: + index: test-index-options + body: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: int8_hnsw + m: 20 + ef_construction: 90 + confidence_interval: 1.0 + + - do: + indices.get_mapping: + index: test-index-options + + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.type": "int8_hnsw" } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.m": 20 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.ef_construction": 90 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.confidence_interval": 1.0 } + + - do: + catch: /Cannot update parameter \[index_options\]/ + indices.put_mapping: + index: test-index-options + body: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: int8_flat + + - match: { status: 400 } diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml index fa935ac450f88..b089d8c439330 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml @@ -3,6 +3,55 @@ setup: cluster_features: "gte_v8.15.0" reason: semantic_text introduced in 8.15.0 + - do: + inference.put: + task_type: sparse_embedding + inference_id: sparse-inference-id + body: > + { + "service": "test_service", + "service_settings": { + "model": "my_model", + "api_key": "abc64" + }, + "task_settings": { + } + } + + - do: + inference.put: + task_type: text_embedding + inference_id: dense-inference-id + body: > + { + "service": "text_embedding_test_service", + "service_settings": { + "model": "my_model", + "dimensions": 4, + "similarity": "cosine", + "api_key": "abc64" + }, + "task_settings": { + } + } + + - do: + inference.put: + task_type: text_embedding + inference_id: dense-inference-id-compatible-with-bbq + body: > + { + "service": "text_embedding_test_service", + "service_settings": { + "model": "my_model", + "dimensions": 64, + "similarity": "cosine", + "api_key": "abc64" + }, + "task_settings": { + } + } + - do: indices.create: index: test-index @@ -148,7 +197,7 @@ setup: - match: { "test-index.mappings.properties.dense_field.type": semantic_text } - match: { "test-index.mappings.properties.dense_field.inference_id": dense-inference-id } - - length: { "test-index.mappings.properties.dense_field": 2 } + - not_exists: test-index.mappings.properties.dense_field.model_settings - do: index: @@ -164,11 +213,17 @@ setup: dimensions: 4 similarity: cosine element_type: float + index_options: + dense_vector: + type: int8_hnsw + m: 16 + ef_construction: 100 chunks: - text: "these are not the droids you're looking for" - embeddings: [0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416] + embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ] - text: "He's free to go around" - embeddings: [0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896] + embeddings: [ 0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896 ] + refresh: true # Checks mapping is updated when first doc arrives - do: @@ -178,7 +233,69 @@ setup: - match: { "test-index.mappings.properties.dense_field.type": semantic_text } - match: { "test-index.mappings.properties.dense_field.inference_id": dense-inference-id } - match: { "test-index.mappings.properties.dense_field.model_settings.task_type": text_embedding } - - length: { "test-index.mappings.properties.dense_field": 3 } + - exists: test-index.mappings.properties.dense_field.model_settings + +--- +"Indexes dense vector document with bbq compatible model": + - requires: + cluster_features: "semantic_text.index_options" + reason: index_options introduced in 8.19.0 + + - do: + indices.create: + index: test-index-options-with-bbq + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: true + mappings: + properties: + dense_field: + type: semantic_text + inference_id: dense-inference-id-compatible-with-bbq + + # Checks vector mapping is not updated until first doc arrives + - do: + indices.get_mapping: + index: test-index-options-with-bbq + + - match: { "test-index-options-with-bbq.mappings.properties.dense_field.type": semantic_text } + - match: { "test-index-options-with-bbq.mappings.properties.dense_field.inference_id": dense-inference-id-compatible-with-bbq } + - not_exists: test-index-options-with-bbq.mappings.properties.dense_field.index_options + - not_exists: test-index-options-with-bbq.mappings.properties.dense_field.model_settings + + - do: + index: + index: test-index-options-with-bbq + id: doc_2 + body: + dense_field: + text: "these are not the droids you're looking for. He's free to go around" + inference: + inference_id: dense-inference-id-compatible-with-bbq + model_settings: + task_type: text_embedding + dimensions: 64 + similarity: cosine + element_type: float + chunks: + - text: "these are not the droids you're looking for" + embeddings: [ 0.05, -0.03, -0.03, 0.06, 0.01, -0.02, 0.07, 0.02, -0.04, 0.03, 0.00, 0.05, -0.06, 0.04, -0.01, 0.02, -0.05, 0.01, 0.03, -0.02, 0.06, -0.04, 0.00, 0.05, -0.03, 0.02, 0.01, -0.01, 0.04, -0.06, 0.03, 0.02, -0.02, 0.06, -0.01, 0.00, 0.04, -0.05, 0.01, 0.03, -0.04, 0.02, -0.03, 0.05, -0.02, 0.01, 0.03, -0.06, 0.04, 0.00, -0.01, 0.06, -0.03, 0.02, 0.01, -0.04, 0.05, -0.01, 0.00, 0.04, -0.05, 0.02, 0.03, -0.02 ] + - text: "He's free to go around" + embeddings: [ 0.05, -0.03, -0.03, 0.06, 0.01, -0.02, 0.07, 0.02, -0.04, 0.03, 0.00, 0.05, -0.06, 0.04, -0.01, 0.02, -0.05, 0.01, 0.03, -0.02, 0.06, -0.04, 0.00, 0.05, -0.03, 0.02, 0.01, -0.01, 0.04, -0.06, 0.03, 0.02, -0.02, 0.06, -0.01, 0.00, 0.04, -0.05, 0.01, 0.03, -0.04, 0.02, -0.03, 0.05, -0.02, 0.01, 0.03, -0.06, 0.04, 0.00, -0.01, 0.06, -0.03, 0.02, 0.01, -0.04, 0.05, -0.01, 0.00, 0.04, -0.05, 0.02, 0.03, -0.02 ] + refresh: true + + # Checks mapping is updated when first doc arrives + - do: + indices.get_mapping: + index: test-index-options-with-bbq + + - match: { "test-index-options-with-bbq.mappings.properties.dense_field.type": semantic_text } + - match: { "test-index-options-with-bbq.mappings.properties.dense_field.inference_id": dense-inference-id-compatible-with-bbq } + - match: { "test-index-options-with-bbq.mappings.properties.dense_field.model_settings.task_type": text_embedding } + - not_exists: test-index-options-with-bbq.mappings.properties.dense_field.index_options --- "Field caps with text embedding": @@ -330,3 +447,292 @@ setup: - not_exists: fields.dense_field.inference.chunks.text - not_exists: fields.dense_field.inference.chunks - not_exists: fields.dense_field.inference +--- +"Users can set dense vector index options and index documents using those options": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + indices.create: + index: test-index-options + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: true + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: int8_hnsw + m: 20 + ef_construction: 100 + confidence_interval: 1.0 + + - do: + indices.get_mapping: + index: test-index-options + + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.type": "int8_hnsw" } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.m": 20 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.ef_construction": 100 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.confidence_interval": 1.0 } + + - do: + index: + index: test-index_options + id: doc_1 + body: + dense_field: + text: "these are not the droids you're looking for. He's free to go around" + inference: + inference_id: dense-inference-id + model_settings: + task_type: text_embedding + dimensions: 4 + similarity: cosine + element_type: float + index_options: + dense_vector: + type: int8_hnsw + m: 20 + ef_construction: 100 + confidence_interval: 1.0 + chunks: + - text: "these are not the droids you're looking for" + embeddings: [ 0.04673296958208084, -0.03237321600317955, -0.02543032355606556, 0.056035321205854416 ] + - text: "He's free to go around" + embeddings: [ 0.00641461368650198, -0.0016253676731139421, -0.05126338079571724, 0.053438711911439896 ] + refresh: true + + - do: + indices.get_mapping: + index: test-index-options + + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.type": "int8_hnsw" } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.m": 20 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.ef_construction": 100 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.confidence_interval": 1.0 } + +--- +"Specifying incompatible dense vector index options will fail": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + catch: /unsupported parameters/ + indices.create: + index: test-incompatible-index-options + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: true + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: bbq_flat + ef_construction: 100 + +--- +"Specifying unsupported index option types will fail": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + catch: /Unsupported index options type/ + indices.create: + index: test-invalid-index-options-dense + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: true + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: foo + - do: + catch: bad_request + indices.create: + index: test-invalid-index-options-sparse + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: true + mappings: + properties: + semantic_field: + type: semantic_text + index_options: + sparse_vector: + type: int8_hnsw + +--- +"Index option type is required": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + catch: /Required type/ + indices.create: + index: test-invalid-index-options-dense + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: true + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + foo: bar + +--- +"Specifying index options requires model information": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + catch: /Model settings must be set to validate index options/ + indices.create: + index: my-custom-semantic-index + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: true + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: nonexistent-inference-id + index_options: + dense_vector: + type: int8_hnsw + + - match: { status: 400 } + + - do: + indices.create: + index: my-custom-semantic-index + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: true + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: nonexistent-inference-id + + - do: + indices.get_mapping: + index: my-custom-semantic-index + + - match: { "my-custom-semantic-index.mappings.properties.semantic_field.type": semantic_text } + - match: { "my-custom-semantic-index.mappings.properties.semantic_field.inference_id": nonexistent-inference-id } + - not_exists: my-custom-semantic-index.mappings.properties.semantic_field.index_options + +--- +"Updating index options": + - requires: + cluster_features: "semantic_text.index_options" + reason: Index options introduced in 8.19.0 + + - do: + indices.create: + index: test-index-options + body: + settings: + index: + mapping: + semantic_text: + use_legacy_format: true + mappings: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: int8_hnsw + m: 16 + ef_construction: 100 + confidence_interval: 1.0 + + - do: + indices.get_mapping: + index: test-index-options + + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.type": "int8_hnsw" } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.m": 16 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.ef_construction": 100 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.confidence_interval": 1.0 } + + - do: + indices.put_mapping: + index: test-index-options + body: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: int8_hnsw + m: 20 + ef_construction: 90 + confidence_interval: 1.0 + + - do: + indices.get_mapping: + index: test-index-options + + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.type": "int8_hnsw" } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.m": 20 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.ef_construction": 90 } + - match: { "test-index-options.mappings.properties.semantic_field.index_options.dense_vector.confidence_interval": 1.0 } + + - do: + catch: /Cannot update parameter \[index_options\]/ + indices.put_mapping: + index: test-index-options + body: + properties: + semantic_field: + type: semantic_text + inference_id: dense-inference-id + index_options: + dense_vector: + type: int8_flat + + - match: { status: 400 }