Fix - Requesting _inference_fields when using legacy format causes shard failure (#121720) (#122177)

* Adding condition to verify if the field belongs to an index

* Update docs/changelog/121720.yaml

* Remove unnecessary comma from yaml file

* remove duplicate inference endpoint creation

* updating isMetadata to return true if mapper has the correct type

* remove unnecessary index creation in yaml tests

* Adding check if the document has returned in the yaml test

* Updating test to skip time series  check if index mode is standard

* Refactor tests to consider verifying every metafields with all index modes

* refactoring test to verify for all cases

* Adding assetFalse if not time_series and fields are from time_series

* updating test texts to have better description

Author: Samiul-TheSoccerFan

docs/changelog/121720.yaml

@@ -0,0 +1,5 @@
+pr: 121720
+summary: Skip fetching _inference_fields field in legacy semantic_text format
+area: Search
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -801,7 +801,8 @@ public static boolean isMetadataFieldStatic(String fieldName) {
      * this method considers all mapper plugins
      */
     public boolean isMetadataField(String field) {
-        return mapperRegistry.getMetadataMapperParsers(indexVersionCreated).containsKey(field);
+        var mapper = mappingLookup().getMapper(field);
+        return mapper instanceof MetadataFieldMapper;
     }
 
     /**

server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java

@@ -15,6 +15,9 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
@@ -26,6 +29,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
@@ -304,16 +308,56 @@ public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable {
 
     public void testIsMetadataField() throws IOException {
         IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random());
-        Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
 
-        MapperService mapperService = createMapperService(settings, mapping(b -> {}));
-        assertFalse(mapperService.isMetadataField(randomAlphaOfLengthBetween(10, 15)));
+        CheckedFunction<IndexMode, MapperService, IOException> initMapperService = (indexMode) -> {
+            Settings.Builder settingsBuilder = Settings.builder()
+                .put(IndexMetadata.SETTING_VERSION_CREATED, version)
+                .put(IndexSettings.MODE.getKey(), indexMode);
 
-        for (String builtIn : IndicesModule.getBuiltInMetadataFields()) {
-            if (NestedPathFieldMapper.NAME.equals(builtIn) && version.before(IndexVersions.V_8_0_0)) {
-                continue;   // Nested field does not exist in the 7x line
+            if (indexMode == IndexMode.TIME_SERIES) {
+                settingsBuilder.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo");
             }
-            assertTrue("Expected " + builtIn + " to be a metadata field for version " + version, mapperService.isMetadataField(builtIn));
+
+            return createMapperService(settingsBuilder.build(), mapping(b -> {}));
+        };
+
+        Consumer<MapperService> assertMapperService = (mapperService) -> {
+            assertFalse(mapperService.isMetadataField(randomAlphaOfLengthBetween(10, 15)));
+
+            for (String builtIn : IndicesModule.getBuiltInMetadataFields()) {
+                if (NestedPathFieldMapper.NAME.equals(builtIn) && version.before(IndexVersions.V_8_0_0)) {
+                    continue;  // Nested field does not exist in the 7x line
+                }
+                boolean isTimeSeriesField = builtIn.equals("_tsid") || builtIn.equals("_ts_routing_hash");
+                boolean isTimeSeriesMode = mapperService.getIndexSettings().getMode().equals(IndexMode.TIME_SERIES);
+
+                if (isTimeSeriesField && isTimeSeriesMode == false) {
+                    assertFalse(
+                        "Expected "
+                            + builtIn
+                            + " to not be a metadata field for version "
+                            + version
+                            + " and index mode "
+                            + mapperService.getIndexSettings().getMode(),
+                        mapperService.isMetadataField(builtIn)
+                    );
+                } else {
+                    assertTrue(
+                        "Expected "
+                            + builtIn
+                            + " to be a metadata field for version "
+                            + version
+                            + " and index mode "
+                            + mapperService.getIndexSettings().getMode(),
+                        mapperService.isMetadataField(builtIn)
+                    );
+                }
+            }
+        };
+
+        for (IndexMode indexMode : IndexMode.values()) {
+            MapperService mapperService = initMapperService.apply(indexMode);
+            assertMapperService.accept(mapperService);
         }
     }
 

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java

@@ -36,6 +36,7 @@ public Set<NodeFeature> getTestFeatures() {
             SemanticTextFieldMapper.SEMANTIC_TEXT_DELETE_FIX,
             SemanticTextFieldMapper.SEMANTIC_TEXT_ZERO_SIZE_FIX,
             SemanticTextFieldMapper.SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX,
+            SemanticTextFieldMapper.SEMANTIC_TEXT_SKIP_INFERENCE_FIELDS,
             SEMANTIC_TEXT_HIGHLIGHTER,
             SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED,
             SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED,

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java

@@ -117,6 +117,7 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
     public static final NodeFeature SEMANTIC_TEXT_ALWAYS_EMIT_INFERENCE_ID_FIX = new NodeFeature(
         "semantic_text.always_emit_inference_id_fix"
     );
+    public static final NodeFeature SEMANTIC_TEXT_SKIP_INFERENCE_FIELDS = new NodeFeature("semantic_text.skip_inference_fields");
 
     public static final String CONTENT_TYPE = "semantic_text";
     public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID;

x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference_bwc.yml

@@ -650,3 +650,28 @@ setup:
   - match: { hits.total.value: 1 }
   - match: { hits.total.relation: eq }
   - match: { hits.hits.0._source.dense_field.text: "updated text" }
+
+---
+"Skip fetching _inference_fields":
+  - requires:
+      cluster_features: semantic_text.skip_inference_fields
+      reason: Skip _inference_fields when search is performed on legacy semantic_text format.
+
+  - do:
+      index:
+        index: test-index
+        id: doc_1
+        body:
+          sparse_field: "test value"
+        refresh: true
+
+  - do:
+      search:
+        index: test-index
+        body:
+          fields: [ _inference_fields ]
+          query:
+            match_all: { }
+
+  - match: { hits.total.value: 1 }
+  - not_exists: hits.hits.0._source._inference_fields