Adaptive Hybrid Synthetic Source for Pruned _source Fields

modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java

@@ -235,4 +235,61 @@ public void testReindexIncludeVectors() throws Exception {
             searchResponse.decRef();
         }
     }
+
+    public void testReindexAutoIncludeVectors() throws Exception {
+        var resp1 = prepareCreate("test").setSettings(
+            Settings.builder().put(IndexSettings.INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.getKey(), false).build()
+        ).setMapping(
+            Map.of(
+                "_source",
+                Map.of("enabled", true, "excludes", List.of("foo", "bar")),
+                "properties",
+                Map.of(
+                    "foo", Map.of("type", "dense_vector", "similarity", "l2_norm"),
+                    "bar", Map.of("type", "sparse_vector", "store", true)
+                )
+            )
+        ).get();
+        assertAcked(resp1);
+
+        var resp2 = prepareCreate("test_reindex").setSettings(
+            Settings.builder().put(IndexSettings.INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.getKey(), false).build()
+        ).setMapping(
+            Map.of(
+                "_source",
+                Map.of("enabled", true, "excludes", List.of("foo", "bar")),
+                "properties",
+                Map.of(
+                    "foo", Map.of("type", "dense_vector", "similarity", "l2_norm"),
+                    "bar", Map.of("type", "sparse_vector", "store", true)
+                )
+            )
+        ).get();
+        assertAcked(resp2);
+
+        indexRandom(
+            true,
+            prepareIndex("test").setId("1").setSource("foo", List.of(3f, 2f, 1.5f), "bar", Map.of("token_1", 4f, "token_2", 7f))
+        );
+
+        // Copy all the docs
+        ReindexRequestBuilder copy = reindex().source("test").destination("test_reindex").refresh(true);
+        var reindexResponse = copy.get();
+        assertThat(reindexResponse, matcher().created(1));
+
+        var searchResponse = prepareSearch("test_reindex").get();
+        try {
+            assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(1L));
+            assertThat(searchResponse.getHits().getHits().length, equalTo(1));
+            var sourceMap = searchResponse.getHits().getAt(0).getSourceAsMap();
+            assertThat(sourceMap.get("foo"), anyOf(equalTo(List.of(3f, 2f, 1.5f)), equalTo(List.of(3d, 2d, 1.5d))));
+            assertThat(
+                sourceMap.get("bar"),
+                anyOf(equalTo(Map.of("token_1", 4f, "token_2", 7f)), equalTo(Map.of("token_1", 4d, "token_2", 7d)))
+            );
+        } finally {
+            searchResponse.decRef();
+        }
+    }
+
 }

server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java

@@ -83,7 +83,9 @@ public LuceneChangesSnapshot(
         this.lastSeenSeqNo = fromSeqNo - 1;
         final TopDocs topDocs = nextTopDocs();
         this.maxDocIndex = topDocs.scoreDocs.length;
-        this.syntheticVectorPatchLoader = mapperService.mappingLookup().getMapping().syntheticVectorsLoader(null);
+        this.syntheticVectorPatchLoader = mapperService.mappingLookup()
+            .getMapping()
+            .syntheticVectorsLoader(null, mapperService.mappingLookup().isFieldMapperAutoHybrid());
         fillParallelArray(topDocs.scoreDocs, parallelArray);
     }
 

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -464,6 +464,13 @@ public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() {
         return null;
     }
 
+    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) {
+        if (checker.check(this)) {
+            return syntheticVectorsLoader();
+        }
+        return null;
+    }
+
     /**
      * <p>
      * Specifies the mode of synthetic source support by the mapper.

server/src/main/java/org/elasticsearch/index/mapper/Mapping.java

@@ -138,8 +138,11 @@ Mapping mappingUpdate(RootObjectMapper rootObjectMapper) {
      * @return a {@link SourceLoader.SyntheticVectorsLoader} for extracting synthetic vectors,
      *         potentially using the provided filter
      */
-    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(@Nullable SourceFilter filter) {
-        return root.syntheticVectorsLoader(filter);
+    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(
+        @Nullable SourceFilter filter,
+        SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker
+    ) {
+        return root.syntheticVectorsLoader(filter, checker);
     }
 
     private boolean isSourceSynthetic() {

server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

@@ -194,7 +194,7 @@ private MappingLookup(
             if (mapper instanceof InferenceFieldMapper inferenceFieldMapper) {
                 inferenceFields.put(mapper.fullPath(), inferenceFieldMapper.getMetadata(fieldTypeLookup.sourcePaths(mapper.fullPath())));
             }
-            if (mapper.syntheticVectorsLoader() != null) {
+            if (mapper.syntheticVectorsLoader(this.isFieldMapperAutoHybrid()) != null) {
                 syntheticVectorFields.add(mapper.fullPath());
             }
         }
@@ -489,14 +489,31 @@ public boolean isSourceSynthetic() {
         return sfm != null && sfm.isSynthetic();
     }
 
+    /**
+     * Auto use partial synthetic source combine with stored source
+     */
+    public SourceLoader.SyntheticVectorsLoader.AutoHybridChecker isFieldMapperAutoHybrid() {
+        SourceFieldMapper sfm = mapping.getMetadataMapperByClass(SourceFieldMapper.class);
+        return fieldMapper -> {
+            if (sfm != null && sfm.isStored()) {
+                for (String exclude : sfm.getExcludes()) {
+                    if (exclude.equals(fieldMapper.fullPath())) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        };
+    }
+
     /**
      * Build something to load source {@code _source}.
      */
     public SourceLoader newSourceLoader(@Nullable SourceFilter filter, SourceFieldMetrics metrics) {
         if (isSourceSynthetic()) {
             return new SourceLoader.Synthetic(filter, () -> mapping.syntheticFieldLoader(filter), metrics, mapping.ignoredSourceFormat());
         }
-        var syntheticVectorsLoader = mapping.syntheticVectorsLoader(filter);
+        var syntheticVectorsLoader = mapping.syntheticVectorsLoader(filter, this.isFieldMapperAutoHybrid());
         if (syntheticVectorsLoader != null) {
             return new SourceLoader.SyntheticVectors(removeExcludedSyntheticVectorFields(filter), syntheticVectorsLoader);
         }

server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java

@@ -410,8 +410,11 @@ protected MapperMergeContext createChildContext(MapperMergeContext mapperMergeCo
     }
 
     @Override
-    protected SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceFilter sourceFilter) {
-        var patchLoader = super.syntheticVectorsLoader(sourceFilter);
+    protected SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(
+        SourceFilter sourceFilter,
+        SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker
+    ) {
+        var patchLoader = super.syntheticVectorsLoader(sourceFilter, checker);
         if (patchLoader == null) {
             return null;
         }

server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java

@@ -912,23 +912,30 @@ public ObjectMapper findParentMapper(String leafFieldPath) {
         return null;
     }
 
-    private static SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(Mapper mapper, SourceFilter sourceFilter) {
+    private static SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(
+        Mapper mapper,
+        SourceFilter sourceFilter,
+        SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker
+    ) {
         if (sourceFilter != null && sourceFilter.isPathFiltered(mapper.fullPath(), false)) {
             return null;
         }
         if (mapper instanceof ObjectMapper objMapper) {
-            return objMapper.syntheticVectorsLoader(sourceFilter);
+            return objMapper.syntheticVectorsLoader(sourceFilter, checker);
         } else if (mapper instanceof FieldMapper fieldMapper) {
-            return fieldMapper.syntheticVectorsLoader();
+            return fieldMapper.syntheticVectorsLoader(checker);
         } else {
             return null;
         }
     }
 
-    SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceFilter sourceFilter) {
+    SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(
+        SourceFilter sourceFilter,
+        SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker
+    ) {
         var loaders = mappers.values()
             .stream()
-            .map(m -> syntheticVectorsLoader(m, sourceFilter))
+            .map(m -> syntheticVectorsLoader(m, sourceFilter, checker))
             .filter(l -> l != null)
             .collect(Collectors.toList());
         if (loaders.isEmpty()) {

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -615,4 +615,8 @@ private static void removeSyntheticVectorFields(
                 destination.copyCurrentEvent(parser);
         }
     }
+
+    public String[] getExcludes() {
+        return sourceFilter == null ? Strings.EMPTY_ARRAY : sourceFilter.getExcludes();
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java

@@ -548,5 +548,9 @@ interface Leaf {
              */
             void load(int doc, List<SyntheticVectorPatch> acc) throws IOException;
         }
+
+        interface AutoHybridChecker {
+            boolean check(FieldMapper mapper);
+        }
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -3041,9 +3041,14 @@ public String toString() {
 
     @Override
     public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() {
-        if (isExcludeSourceVectors) {
-            var syntheticField = new IndexedSyntheticFieldLoader(indexCreatedVersion, fieldType().similarity);
-            return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyVectorAsList);
+        var syntheticField = new IndexedSyntheticFieldLoader(indexCreatedVersion, fieldType().similarity);
+        return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyVectorAsList);
+    }
+
+    @Override
+    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) {
+        if (isExcludeSourceVectors || checker.check(this)) {
+            return syntheticVectorsLoader();
         }
         return null;
     }

server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java

@@ -327,9 +327,14 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
 
     @Override
     public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() {
-        if (isExcludeSourceVectors) {
-            var syntheticField = new SparseVectorSyntheticFieldLoader(fullPath(), leafName());
-            return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyAsMap);
+        var syntheticField = new SparseVectorSyntheticFieldLoader(fullPath(), leafName());
+        return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyAsMap);
+    }
+
+    @Override
+    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) {
+        if (isExcludeSourceVectors || checker.check(this)) {
+            return syntheticVectorsLoader();
         }
         return null;
     }

x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java

@@ -406,9 +406,14 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
 
     @Override
     public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() {
-        if (isExcludeSourceVectors) {
-            var syntheticField = new DocValuesSyntheticFieldLoader();
-            return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyVectorsAsList);
+        var syntheticField = new DocValuesSyntheticFieldLoader();
+        return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyVectorsAsList);
+    }
+
+    @Override
+    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) {
+        if (isExcludeSourceVectors || checker.check(this)) {
+            return syntheticVectorsLoader();
         }
         return null;
     }