[8.19] Adding support to exclude semantic_text subfields (#127664)

docs/changelog/127664.yaml

@@ -0,0 +1,5 @@
+pr: 127664
+summary: Exclude `semantic_text` subfields from field capabilities API
+area: "Mapping"
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
+import org.elasticsearch.cluster.metadata.InferenceFieldMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
@@ -30,6 +31,7 @@
 import org.elasticsearch.tasks.CancellableTask;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -256,6 +258,14 @@ private static Predicate<MappedFieldType> buildFilter(String[] filters, String[]
             Set<String> acceptedTypes = Set.of(fieldTypes);
             fcf = ft -> acceptedTypes.contains(ft.familyTypeName());
         }
+
+        // Exclude internal ".inference" subfields of semantic_text fields from the field capabilities response
+        Collection<InferenceFieldMetadata> inferenceFields = context.getMappingLookup().inferenceFields().values();
+        for (InferenceFieldMetadata inferenceField : inferenceFields) {
+            Predicate<MappedFieldType> next = ft -> ft.name().startsWith(inferenceField.getName() + ".inference") == false;
+            fcf = fcf == null ? next : fcf.and(next);
+        }
+
         for (String filter : filters) {
             if ("parent".equals(filter) || "-parent".equals(filter)) {
                 continue;

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java

@@ -16,6 +16,7 @@
 
 import java.util.Set;
 
+import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.SEMANTIC_TEXT_SUPPORT_CHUNKING_CONFIG;
 import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_FILTER_FIX;
 import static org.elasticsearch.xpack.inference.queries.SemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED;
@@ -67,7 +68,8 @@ public Set<NodeFeature> getTestFeatures() {
             SemanticTextFieldMapper.SEMANTIC_TEXT_BIT_VECTOR_SUPPORT,
             SemanticTextFieldMapper.SEMANTIC_TEXT_HANDLE_EMPTY_INPUT,
             SEMANTIC_TEXT_SUPPORT_CHUNKING_CONFIG,
-            SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER
+            SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER,
+            SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS
         );
     }
 }

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java

@@ -134,6 +134,9 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
     public static final NodeFeature SEMANTIC_TEXT_SKIP_INFERENCE_FIELDS = new NodeFeature("semantic_text.skip_inference_fields");
     public static final NodeFeature SEMANTIC_TEXT_BIT_VECTOR_SUPPORT = new NodeFeature("semantic_text.bit_vector_support");
     public static final NodeFeature SEMANTIC_TEXT_SUPPORT_CHUNKING_CONFIG = new NodeFeature("semantic_text.support_chunking_config");
+    public static final NodeFeature SEMANTIC_TEXT_EXCLUDE_SUB_FIELDS_FROM_FIELD_CAPS = new NodeFeature(
+        "semantic_text.exclude_sub_fields_from_field_caps"
+    );
 
     public static final String CONTENT_TYPE = "semantic_text";
     public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID;

x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping.yml

@@ -359,3 +359,76 @@ setup:
         index: test-always-include-inference-id-index
 
   - exists: test-always-include-inference-id-index.mappings.properties.semantic_field.inference_id
+
+---
+"Field caps exclude chunks and embedding fields":
+  - requires:
+      cluster_features: "semantic_text.exclude_sub_fields_from_field_caps"
+      reason: field caps api exclude semantic_text subfields from 9.1.0 & 8.19.0
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+  - not_exists: fields.sparse_field.inference.chunks.embeddings
+  - not_exists: fields.sparse_field.inference.chunks.offset
+  - not_exists: fields.sparse_field.inference.chunks
+  - not_exists: fields.sparse_field.inference
+  - not_exists: fields.dense_field.inference.chunks.embeddings
+  - not_exists: fields.dense_field.inference.chunks.offset
+  - not_exists: fields.dense_field.inference.chunks
+  - not_exists: fields.dense_field.inference
+
+---
+"Field caps does not exclude multi-fields under semantic_text":
+  - requires:
+      cluster_features: "semantic_text.exclude_sub_fields_from_field_caps"
+      reason: field caps api exclude semantic_text subfields from 9.1.0 & 8.19.0
+  - do:
+      indices.create:
+        index: test-multi-field-index
+        body:
+          settings:
+            index:
+              mapping:
+                semantic_text:
+                  use_legacy_format: false
+          mappings:
+            properties:
+              sparse_field:
+                type: semantic_text
+                inference_id: sparse-inference-id
+                fields:
+                  sparse_keyword_field:
+                    type: keyword
+              dense_field:
+                type: semantic_text
+                inference_id: dense-inference-id
+                fields:
+                  dense_keyword_field:
+                    type: keyword
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-multi-field-index
+        fields: "*"
+
+  - match: { indices: [ "test-multi-field-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+  - exists: fields.sparse_field\.sparse_keyword_field
+  - exists: fields.dense_field\.dense_keyword_field
+  - not_exists: fields.sparse_field.inference.chunks.embeddings
+  - not_exists: fields.sparse_field.inference.chunks.offset
+  - not_exists: fields.sparse_field.inference.chunks
+  - not_exists: fields.sparse_field.inference
+  - not_exists: fields.dense_field.inference.chunks.embeddings
+  - not_exists: fields.dense_field.inference.chunks.offset
+  - not_exists: fields.dense_field.inference.chunks
+  - not_exists: fields.dense_field.inference

x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_field_mapping_bwc.yml

@@ -307,3 +307,26 @@ setup:
               another_field:
                 type: keyword
 
+---
+"Field caps exclude chunks embedding and text fields":
+  - requires:
+      cluster_features: "semantic_text.exclude_sub_fields_from_field_caps"
+      reason: field caps api exclude semantic_text subfields from 9.1.0 & 8.19.0
+
+  - do:
+      field_caps:
+        include_empty_fields: true
+        index: test-index
+        fields: "*"
+
+  - match: { indices: [ "test-index" ] }
+  - exists: fields.sparse_field
+  - exists: fields.dense_field
+  - not_exists: fields.sparse_field.inference.chunks.embeddings
+  - not_exists: fields.sparse_field.inference.chunks.text
+  - not_exists: fields.sparse_field.inference.chunks
+  - not_exists: fields.sparse_field.inference
+  - not_exists: fields.dense_field.inference.chunks.embeddings
+  - not_exists: fields.dense_field.inference.chunks.text
+  - not_exists: fields.dense_field.inference.chunks
+  - not_exists: fields.dense_field.inference