Fix inference fields handling on old indices

docs/changelog/136312.yaml

@@ -0,0 +1,7 @@
+pr: 136312
+summary: Fix _inference_fields handling on old indices
+area: Vector Search
+type: bug
+issues: [
+  136130
+]

server/src/main/java/org/elasticsearch/index/get/ShardGetService.java

@@ -311,8 +311,7 @@ private GetResult innerGetFetch(
             fetchSourceContext = res.v1();
         }
 
-        if (mappingLookup.inferenceFields().isEmpty() == false
-            && shouldExcludeInferenceFieldsFromSource(indexSettings, fetchSourceContext) == false) {
+        if (mappingLookup.inferenceFields().isEmpty() == false && shouldExcludeInferenceFieldsFromSource(fetchSourceContext) == false) {
             storedFieldSet.add(InferenceMetadataFieldsMapper.NAME);
         }
 
@@ -424,17 +423,30 @@ private static Boolean shouldExcludeVectorsFromSourceExplicit(FetchSourceContext
         return fetchSourceContext != null ? fetchSourceContext.excludeVectors() : null;
     }
 
-    public static boolean shouldExcludeInferenceFieldsFromSource(IndexSettings indexSettings, FetchSourceContext fetchSourceContext) {
-        var explicit = shouldExcludeInferenceFieldsFromSourceExplicit(fetchSourceContext);
-        var filter = fetchSourceContext != null ? fetchSourceContext.filter() : null;
-        if (filter != null) {
-            if (filter.isPathFiltered(InferenceMetadataFieldsMapper.NAME, true)) {
+    public static boolean shouldExcludeInferenceFieldsFromSource(FetchSourceContext fetchSourceContext) {
+        if (fetchSourceContext != null) {
+            if (fetchSourceContext.fetchSource() == false) {
+                // Source is disabled
                 return true;
-            } else if (filter.isExplicitlyIncluded(InferenceMetadataFieldsMapper.NAME)) {
-                return false;
+            }
+
+            var filter = fetchSourceContext.filter();
+            if (filter != null) {
+                if (filter.isPathFiltered(InferenceMetadataFieldsMapper.NAME, true)) {
+                    return true;
+                } else if (filter.isExplicitlyIncluded(InferenceMetadataFieldsMapper.NAME)) {
+                    return false;
+                }
+            }
+
+            Boolean excludeInferenceFieldsExplicit = shouldExcludeInferenceFieldsFromSourceExplicit(fetchSourceContext);
+            if (excludeInferenceFieldsExplicit != null) {
+                return excludeInferenceFieldsExplicit;
             }
         }
-        return explicit != null ? explicit : INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.get(indexSettings.getSettings());
+
+        // We always default to excluding the inference metadata field, unless the fetch source context says otherwise
+        return true;
     }
 
     private static Boolean shouldExcludeInferenceFieldsFromSourceExplicit(FetchSourceContext fetchSourceContext) {

server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java

@@ -134,8 +134,7 @@ private SearchHits buildSearchHits(SearchContext context, int[] docIdsToLoad, Pr
             context.fetchSourceContext(res.v1());
         }
 
-        if (lookup.inferenceFields().isEmpty() == false
-            && shouldExcludeInferenceFieldsFromSource(context.indexShard().indexSettings(), context.fetchSourceContext()) == false) {
+        if (lookup.inferenceFields().isEmpty() == false && shouldExcludeInferenceFieldsFromSource(context.fetchSourceContext()) == false) {
             // Rehydrate the inference fields into the {@code _source} because they were explicitly requested.
             var oldFetchFieldsContext = context.fetchFieldsContext();
             var newFetchFieldsContext = new FetchFieldsContext(new ArrayList<>());

server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java

@@ -24,9 +24,12 @@
 import org.elasticsearch.index.engine.TranslogOperationAsserter;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.get.ShardGetService;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
@@ -411,6 +414,16 @@ public void testGetFromTranslog() throws IOException {
         closeShards(primary);
     }
 
+    public void testShouldExcludeInferenceFieldsFromSource() {
+        for (int i = 0; i < 100; i++) {
+            ExcludeInferenceFieldsTestScenario scenario = new ExcludeInferenceFieldsTestScenario();
+            assertThat(
+                ShardGetService.shouldExcludeInferenceFieldsFromSource(scenario.fetchSourceContext),
+                equalTo(scenario.shouldExcludeInferenceFields())
+            );
+        }
+    }
+
     Translog.Index toIndexOp(String source) throws IOException {
         XContentParser parser = createParser(XContentType.JSON.xContent(), source);
         XContentBuilder builder = XContentFactory.jsonBuilder();
@@ -425,4 +438,74 @@ Translog.Index toIndexOp(String source) throws IOException {
             IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP
         );
     }
+
+    private static class ExcludeInferenceFieldsTestScenario {
+        private final FetchSourceContext fetchSourceContext;
+
+        private ExcludeInferenceFieldsTestScenario() {
+            this.fetchSourceContext = generateRandomFetchSourceContext();
+        }
+
+        private boolean shouldExcludeInferenceFields() {
+            if (fetchSourceContext != null) {
+                if (fetchSourceContext.fetchSource() == false) {
+                    return true;
+                }
+
+                SourceFilter filter = fetchSourceContext.filter();
+                if (filter != null) {
+                    if (Arrays.asList(filter.getExcludes()).contains(InferenceMetadataFieldsMapper.NAME)) {
+                        return true;
+                    } else if (filter.getIncludes().length > 0) {
+                        return Arrays.asList(filter.getIncludes()).contains(InferenceMetadataFieldsMapper.NAME) == false;
+                    }
+                }
+
+                Boolean excludeInferenceFieldsExplicit = fetchSourceContext.excludeInferenceFields();
+                if (excludeInferenceFieldsExplicit != null) {
+                    return excludeInferenceFieldsExplicit;
+                }
+            }
+
+            return true;
+        }
+
+        private static FetchSourceContext generateRandomFetchSourceContext() {
+            FetchSourceContext fetchSourceContext = switch (randomIntBetween(0, 4)) {
+                case 0 -> FetchSourceContext.FETCH_SOURCE;
+                case 1 -> FetchSourceContext.FETCH_ALL_SOURCE;
+                case 2 -> FetchSourceContext.FETCH_ALL_SOURCE_EXCLUDE_INFERENCE_FIELDS;
+                case 3 -> FetchSourceContext.DO_NOT_FETCH_SOURCE;
+                case 4 -> null;
+                default -> throw new IllegalStateException("Unhandled randomized case");
+            };
+
+            if (fetchSourceContext != null && fetchSourceContext.fetchSource()) {
+                String[] includes = null;
+                String[] excludes = null;
+                if (randomBoolean()) {
+                    // Randomly include a non-existent field to test explicit inclusion handling
+                    String field = randomBoolean() ? InferenceMetadataFieldsMapper.NAME : randomIdentifier();
+                    includes = new String[] { field };
+                }
+                if (randomBoolean()) {
+                    // Randomly exclude a non-existent field to test implicit inclusion handling
+                    String field = randomBoolean() ? InferenceMetadataFieldsMapper.NAME : randomIdentifier();
+                    excludes = new String[] { field };
+                }
+
+                if (includes != null || excludes != null) {
+                    fetchSourceContext = FetchSourceContext.of(
+                        fetchSourceContext.fetchSource(),
+                        fetchSourceContext.excludeVectors(),
+                        fetchSourceContext.excludeInferenceFields(),
+                        includes,
+                        excludes
+                    );
+                }
+            }
+
+            return fetchSourceContext;
+        }
+    }
 }