Fix inference fields handling on old indices

docs/changelog/136312.yaml

@@ -0,0 +1,5 @@
+pr: 136312
+summary: Fix _inference_fields handling on old indices
+area: Vector Search
+type: bug
+issues: [136130]

server/src/main/java/org/elasticsearch/index/get/ShardGetService.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexSettings;
@@ -425,16 +426,32 @@ private static Boolean shouldExcludeVectorsFromSourceExplicit(FetchSourceContext
     }
 
     public static boolean shouldExcludeInferenceFieldsFromSource(IndexSettings indexSettings, FetchSourceContext fetchSourceContext) {
-        var explicit = shouldExcludeInferenceFieldsFromSourceExplicit(fetchSourceContext);
-        var filter = fetchSourceContext != null ? fetchSourceContext.filter() : null;
-        if (filter != null) {
-            if (filter.isPathFiltered(InferenceMetadataFieldsMapper.NAME, true)) {
+        if (fetchSourceContext != null) {
+            if (fetchSourceContext.fetchSource() == false) {
+                // Source is disabled
                 return true;
-            } else if (filter.isExplicitlyIncluded(InferenceMetadataFieldsMapper.NAME)) {
-                return false;
+            }
+
+            var filter = fetchSourceContext.filter();
+            if (filter != null) {
+                if (filter.isPathFiltered(InferenceMetadataFieldsMapper.NAME, true)) {
+                    return true;
+                } else if (filter.isExplicitlyIncluded(InferenceMetadataFieldsMapper.NAME)) {
+                    return false;
+                }
+            }
+
+            Boolean excludeInferenceFieldsExplicit = shouldExcludeInferenceFieldsFromSourceExplicit(fetchSourceContext);
+            if (excludeInferenceFieldsExplicit != null) {
+                return excludeInferenceFieldsExplicit;
             }
         }
-        return explicit != null ? explicit : INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.get(indexSettings.getSettings());
+
+        // We always default to excluding the inference metadata field. We only use the index setting when it is explicitly set.
+        Settings settings = indexSettings.getSettings();
+        return INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.exists(settings)
+            ? INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.get(settings)
+            : true;
     }
 
     private static Boolean shouldExcludeInferenceFieldsFromSourceExplicit(FetchSourceContext fetchSourceContext) {

server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineTestCase;
@@ -24,9 +25,13 @@
 import org.elasticsearch.index.engine.TranslogOperationAsserter;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.index.get.ShardGetService;
+import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
+import org.elasticsearch.search.lookup.SourceFilter;
+import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
@@ -411,6 +416,29 @@ public void testGetFromTranslog() throws IOException {
         closeShards(primary);
     }
 
+    public void testShouldExcludeInferenceFieldsFromSource() {
+        for (int i = 0; i < 100; i++) {
+            ExcludeInferenceFieldsTestScenario scenario = new ExcludeInferenceFieldsTestScenario(IndexVersion.current());
+            assertThat(
+                ShardGetService.shouldExcludeInferenceFieldsFromSource(scenario.indexSettings, scenario.fetchSourceContext),
+                equalTo(scenario.shouldExcludeInferenceFields())
+            );
+        }
+
+        for (int i = 0; i < 200; i++) {
+            IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(
+                random(),
+                IndexVersions.MINIMUM_COMPATIBLE,
+                IndexVersionUtils.getPreviousVersion(IndexVersion.current())
+            );
+            ExcludeInferenceFieldsTestScenario scenario = new ExcludeInferenceFieldsTestScenario(indexVersion);
+            assertThat(
+                ShardGetService.shouldExcludeInferenceFieldsFromSource(scenario.indexSettings, scenario.fetchSourceContext),
+                equalTo(scenario.shouldExcludeInferenceFields())
+            );
+        }
+    }
+
     Translog.Index toIndexOp(String source) throws IOException {
         XContentParser parser = createParser(XContentType.JSON.xContent(), source);
         XContentBuilder builder = XContentFactory.jsonBuilder();
@@ -425,4 +453,92 @@ Translog.Index toIndexOp(String source) throws IOException {
             IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP
         );
     }
+
+    private static class ExcludeInferenceFieldsTestScenario {
+        private final IndexSettings indexSettings;
+        private final FetchSourceContext fetchSourceContext;
+
+        private ExcludeInferenceFieldsTestScenario(IndexVersion indexVersion) {
+            this.indexSettings = generateRandomIndexSettings(indexVersion);
+            this.fetchSourceContext = generateRandomFetchSourceContext();
+        }
+
+        private boolean shouldExcludeInferenceFields() {
+            if (fetchSourceContext != null) {
+                if (fetchSourceContext.fetchSource() == false) {
+                    return true;
+                }
+
+                SourceFilter filter = fetchSourceContext.filter();
+                if (filter != null) {
+                    if (Arrays.asList(filter.getExcludes()).contains(InferenceMetadataFieldsMapper.NAME)) {
+                        return true;
+                    } else if (filter.getIncludes().length > 0) {
+                        return Arrays.asList(filter.getIncludes()).contains(InferenceMetadataFieldsMapper.NAME) == false;
+                    }
+                }
+
+                Boolean excludeInferenceFieldsExplicit = fetchSourceContext.excludeInferenceFields();
+                if (excludeInferenceFieldsExplicit != null) {
+                    return excludeInferenceFieldsExplicit;
+                }
+            }
+
+            Settings settings = indexSettings.getSettings();
+            return IndexSettings.INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.exists(settings)
+                ? IndexSettings.INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.get(settings)
+                : true;
+        }
+
+        private static IndexSettings generateRandomIndexSettings(IndexVersion indexVersion) {
+            Settings.Builder settings = Settings.builder()
+                .put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion)
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+            if (randomBoolean()) {
+                settings.put(IndexSettings.INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.getKey(), randomBoolean());
+            }
+
+            return new IndexSettings(IndexMetadata.builder(randomIdentifier()).settings(settings).build(), settings.build());
+        }
+
+        private static FetchSourceContext generateRandomFetchSourceContext() {
+            FetchSourceContext fetchSourceContext = switch (randomIntBetween(0, 4)) {
+                case 0 -> FetchSourceContext.FETCH_SOURCE;
+                case 1 -> FetchSourceContext.FETCH_ALL_SOURCE;
+                case 2 -> FetchSourceContext.FETCH_ALL_SOURCE_EXCLUDE_INFERENCE_FIELDS;
+                case 3 -> FetchSourceContext.DO_NOT_FETCH_SOURCE;
+                case 4 -> null;
+                default -> throw new IllegalStateException("Unhandled randomized case");
+            };
+
+            if (fetchSourceContext != null && fetchSourceContext.fetchSource()) {
+                String[] includes = null;
+                String[] excludes = null;
+                if (randomBoolean()) {
+                    // Randomly include a non-existent field to test explicit inclusion handling
+                    String field = randomBoolean() ? InferenceMetadataFieldsMapper.NAME : randomIdentifier();
+                    includes = new String[] { field };
+                }
+                if (randomBoolean()) {
+                    // Randomly exclude a non-existent field to test implicit inclusion handling
+                    String field = randomBoolean() ? InferenceMetadataFieldsMapper.NAME : randomIdentifier();
+                    excludes = new String[] { field };
+                }
+
+                if (includes != null || excludes != null) {
+                    fetchSourceContext = FetchSourceContext.of(
+                        fetchSourceContext.fetchSource(),
+                        fetchSourceContext.excludeVectors(),
+                        fetchSourceContext.excludeInferenceFields(),
+                        includes,
+                        excludes
+                    );
+                }
+            }
+
+            return fetchSourceContext;
+        }
+    }
 }