Adding support to exclude semantic_text subfields from mapper builders

Author: Samiul-TheSoccerFan

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java

@@ -14,11 +14,8 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.RuntimeField;
-import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
-import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -152,18 +149,6 @@ private FieldCapabilitiesIndexResponse doFetch(
         return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), indexMappingHash, responseMap, true, indexMode);
     }
 
-    /**
-     * Returns true if the field should be excluded from the field capabilities response.
-     * This is used to exclude fields that are not useful for the user, such as
-     * offset_source and inference chunk embeddings.
-     */
-    private static boolean shouldExcludeField(MappedFieldType ft) {
-        return ft.typeName().equals("offset_source")
-            || ((ft instanceof SparseVectorFieldMapper.SparseVectorFieldType
-                || ft instanceof DenseVectorFieldMapper.DenseVectorFieldType
-                || ft instanceof KeywordFieldMapper.KeywordFieldType) && ft.name().contains(".inference.chunks"));
-    }
-
     static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
         SearchExecutionContext context,
         Predicate<String> fieldNameFilter,
@@ -189,7 +174,7 @@ static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
             if ((includeEmptyFields || ft.fieldHasValue(fieldInfos))
                 && (fieldPredicate.test(ft.name()) || context.isMetadataField(ft.name()))
                 && (filter == null || filter.test(ft))
-                && shouldExcludeField(ft) == false) {
+                && ft.familyTypeName() != null) {
                 IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(
                     field,
                     ft.familyTypeName(),

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -205,6 +205,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
         private final IndexVersion indexCreatedVersion;
         private final boolean useDocValuesSkipper;
         private final SourceKeepMode indexSourceKeepMode;
+        private boolean excludeFromFieldCaps = false;
 
         public Builder(final String name, final MappingParserContext mappingParserContext) {
             this(
@@ -300,6 +301,11 @@ public Builder ignoreAbove(int ignoreAbove) {
             return this;
         }
 
+        public Builder excludeFromFieldCaps(boolean excludeFromFieldCaps) {
+            this.excludeFromFieldCaps = excludeFromFieldCaps;
+            return this;
+        }
+
         Builder normalizer(String normalizerName) {
             this.normalizer.setValue(normalizerName);
             return this;
@@ -448,10 +454,13 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
                 indexCreatedVersion,
                 IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD
             );
+
+            KeywordFieldType keywordFieldType = buildFieldType(context, fieldtype);
+            keywordFieldType.setExcludeFromFieldCaps(excludeFromFieldCaps);
             return new KeywordFieldMapper(
                 leafName(),
                 fieldtype,
-                buildFieldType(context, fieldtype),
+                keywordFieldType,
                 builderParams(this, context),
                 context.isSourceSynthetic(),
                 useDocValuesSkipper,

server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java

@@ -68,6 +68,7 @@ public abstract class MappedFieldType {
     private final boolean isStored;
     private final TextSearchInfo textSearchInfo;
     private final Map<String, String> meta;
+    private boolean excludeFromFieldCaps;
 
     public MappedFieldType(
         String name,
@@ -122,7 +123,11 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
 
     /** Returns the field family type, as used in field capabilities */
     public String familyTypeName() {
-        return typeName();
+        return excludeFromFieldCaps ? null : typeName();
+    }
+
+    public void setExcludeFromFieldCaps(boolean exclude) {
+        this.excludeFromFieldCaps = exclude;
     }
 
     public String name() {

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -186,6 +186,12 @@ public static class Builder extends FieldMapper.Builder {
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
         final IndexVersion indexVersionCreated;
+        private boolean excludeFromFieldCaps = false;
+
+        public Builder(String name, IndexVersion indexVersionCreated, boolean excludeFromFieldCaps) {
+            this(name, indexVersionCreated);
+            this.excludeFromFieldCaps = excludeFromFieldCaps;
+        }
 
         public Builder(String name, IndexVersion indexVersionCreated) {
             super(name);
@@ -302,21 +308,23 @@ public Builder indexOptions(IndexOptions indexOptions) {
 
         @Override
         public DenseVectorFieldMapper build(MapperBuilderContext context) {
+            DenseVectorFieldType denseVectorFieldType = new DenseVectorFieldType(
+                context.buildFullName(leafName()),
+                indexVersionCreated,
+                elementType.getValue(),
+                dims.getValue(),
+                indexed.getValue(),
+                similarity.getValue(),
+                indexOptions.getValue(),
+                meta.getValue()
+            );
+            denseVectorFieldType.setExcludeFromFieldCaps(this.excludeFromFieldCaps);
             // Validate again here because the dimensions or element type could have been set programmatically,
             // which affects index option validity
             validate();
             return new DenseVectorFieldMapper(
                 leafName(),
-                new DenseVectorFieldType(
-                    context.buildFullName(leafName()),
-                    indexVersionCreated,
-                    elementType.getValue(),
-                    dims.getValue(),
-                    indexed.getValue(),
-                    similarity.getValue(),
-                    indexOptions.getValue(),
-                    meta.getValue()
-                ),
+                denseVectorFieldType,
                 builderParams(this, context),
                 indexOptions.getValue(),
                 indexVersionCreated

server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java

@@ -73,11 +73,17 @@ private static SparseVectorFieldMapper toType(FieldMapper in) {
     public static class Builder extends FieldMapper.Builder {
         private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).fieldType().isStored(), false);
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
+        private boolean excludeFromFieldCaps = false;
 
         public Builder(String name) {
             super(name);
         }
 
+        public Builder(String name, boolean excludeFromFieldCaps) {
+            super(name);
+            this.excludeFromFieldCaps = excludeFromFieldCaps;
+        }
+
         public Builder setStored(boolean value) {
             stored.setValue(value);
             return this;
@@ -90,11 +96,13 @@ protected Parameter<?>[] getParameters() {
 
         @Override
         public SparseVectorFieldMapper build(MapperBuilderContext context) {
-            return new SparseVectorFieldMapper(
-                leafName(),
-                new SparseVectorFieldType(context.buildFullName(leafName()), stored.getValue(), meta.getValue()),
-                builderParams(this, context)
+            SparseVectorFieldType sparseVectorFieldType = new SparseVectorFieldType(
+                context.buildFullName(leafName()),
+                stored.getValue(),
+                meta.getValue()
             );
+            sparseVectorFieldType.setExcludeFromFieldCaps(excludeFromFieldCaps);
+            return new SparseVectorFieldMapper(leafName(), sparseVectorFieldType, builderParams(this, context));
         }
     }
 

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapper.java

@@ -119,11 +119,11 @@ protected Parameter<?>[] getParameters() {
 
         @Override
         public OffsetSourceFieldMapper build(MapperBuilderContext context) {
-            return new OffsetSourceFieldMapper(
-                leafName(),
-                new OffsetSourceFieldType(context.buildFullName(leafName()), charset.get(), meta.getValue()),
-                builderParams(this, context)
-            );
+            OffsetSourceFieldType fieldType = new OffsetSourceFieldType(context.buildFullName(leafName()), charset.get(), meta.getValue());
+            // explicitly setting the field type to be not available for field caps
+            fieldType.setExcludeFromFieldCaps(true);
+
+            return new OffsetSourceFieldMapper(leafName(), fieldType, builderParams(this, context));
         }
     }
 

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java

@@ -1041,7 +1041,9 @@ private static NestedObjectMapper.Builder createChunksField(
             chunksField.add(createEmbeddingsField(indexSettings.getIndexVersionCreated(), modelSettings, useLegacyFormat));
         }
         if (useLegacyFormat) {
-            var chunkTextField = new KeywordFieldMapper.Builder(TEXT_FIELD, indexVersionCreated).indexed(false).docValues(false);
+            var chunkTextField = new KeywordFieldMapper.Builder(TEXT_FIELD, indexVersionCreated).indexed(false)
+                .docValues(false)
+                .excludeFromFieldCaps(true);
             chunksField.add(chunkTextField);
         } else {
             chunksField.add(new OffsetSourceFieldMapper.Builder(CHUNKED_OFFSET_FIELD));
@@ -1055,11 +1057,14 @@ private static Mapper.Builder createEmbeddingsField(
         boolean useLegacyFormat
     ) {
         return switch (modelSettings.taskType()) {
-            case SPARSE_EMBEDDING -> new SparseVectorFieldMapper.Builder(CHUNKED_EMBEDDINGS_FIELD).setStored(useLegacyFormat == false);
+            case SPARSE_EMBEDDING -> new SparseVectorFieldMapper.Builder(CHUNKED_EMBEDDINGS_FIELD, true).setStored(
+                useLegacyFormat == false
+            );
             case TEXT_EMBEDDING -> {
                 DenseVectorFieldMapper.Builder denseVectorMapperBuilder = new DenseVectorFieldMapper.Builder(
                     CHUNKED_EMBEDDINGS_FIELD,
-                    indexVersionCreated
+                    indexVersionCreated,
+                    true
                 );
 
                 SimilarityMeasure similarity = modelSettings.similarity();