POC - Adaptive hybrid synthetic source reconstruction for pruned vector fields

FAST IMPLEMENTATION focusing only on vector fields, implements the first stage of adaptive hybrid source:
Automatically reconstructs VECTOR fields from doc_values during:
   - Reindex operations
   - Update/update_by_query
   - Get/search requests

Resolves failing test: testReindexAutoIncludeVectors

Author: Rassyan

server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java

@@ -83,7 +83,9 @@ public LuceneChangesSnapshot(
         this.lastSeenSeqNo = fromSeqNo - 1;
         final TopDocs topDocs = nextTopDocs();
         this.maxDocIndex = topDocs.scoreDocs.length;
-        this.syntheticVectorPatchLoader = mapperService.mappingLookup().getMapping().syntheticVectorsLoader(null);
+        this.syntheticVectorPatchLoader = mapperService.mappingLookup()
+            .getMapping()
+            .syntheticVectorsLoader(null, mapperService.mappingLookup().isFieldMapperAutoHybrid());
         fillParallelArray(topDocs.scoreDocs, parallelArray);
     }
 

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -464,6 +464,13 @@ public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() {
         return null;
     }
 
+    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) {
+        if (checker.check(this)) {
+            return syntheticVectorsLoader();
+        }
+        return null;
+    }
+
     /**
      * <p>
      * Specifies the mode of synthetic source support by the mapper.

server/src/main/java/org/elasticsearch/index/mapper/Mapping.java

@@ -138,8 +138,11 @@ Mapping mappingUpdate(RootObjectMapper rootObjectMapper) {
      * @return a {@link SourceLoader.SyntheticVectorsLoader} for extracting synthetic vectors,
      *         potentially using the provided filter
      */
-    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(@Nullable SourceFilter filter) {
-        return root.syntheticVectorsLoader(filter);
+    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(
+        @Nullable SourceFilter filter,
+        SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker
+    ) {
+        return root.syntheticVectorsLoader(filter, checker);
     }
 
     private boolean isSourceSynthetic() {

server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

@@ -194,7 +194,7 @@ private MappingLookup(
             if (mapper instanceof InferenceFieldMapper inferenceFieldMapper) {
                 inferenceFields.put(mapper.fullPath(), inferenceFieldMapper.getMetadata(fieldTypeLookup.sourcePaths(mapper.fullPath())));
             }
-            if (mapper.syntheticVectorsLoader() != null) {
+            if (mapper.syntheticVectorsLoader(this.isFieldMapperAutoHybrid()) != null) {
                 syntheticVectorFields.add(mapper.fullPath());
             }
         }
@@ -489,14 +489,31 @@ public boolean isSourceSynthetic() {
         return sfm != null && sfm.isSynthetic();
     }
 
+    /**
+     * Auto use partial synthetic source combine with stored source
+     */
+    public SourceLoader.SyntheticVectorsLoader.AutoHybridChecker isFieldMapperAutoHybrid() {
+        SourceFieldMapper sfm = mapping.getMetadataMapperByClass(SourceFieldMapper.class);
+        return fieldMapper -> {
+            if (sfm != null && sfm.isStored()) {
+                for (String exclude : sfm.getExcludes()) {
+                    if (exclude.equals(fieldMapper.fullPath())) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        };
+    }
+
     /**
      * Build something to load source {@code _source}.
      */
     public SourceLoader newSourceLoader(@Nullable SourceFilter filter, SourceFieldMetrics metrics) {
         if (isSourceSynthetic()) {
             return new SourceLoader.Synthetic(filter, () -> mapping.syntheticFieldLoader(filter), metrics, mapping.ignoredSourceFormat());
         }
-        var syntheticVectorsLoader = mapping.syntheticVectorsLoader(filter);
+        var syntheticVectorsLoader = mapping.syntheticVectorsLoader(filter, this.isFieldMapperAutoHybrid());
         if (syntheticVectorsLoader != null) {
             return new SourceLoader.SyntheticVectors(removeExcludedSyntheticVectorFields(filter), syntheticVectorsLoader);
         }

server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java

@@ -410,8 +410,11 @@ protected MapperMergeContext createChildContext(MapperMergeContext mapperMergeCo
     }
 
     @Override
-    protected SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceFilter sourceFilter) {
-        var patchLoader = super.syntheticVectorsLoader(sourceFilter);
+    protected SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(
+        SourceFilter sourceFilter,
+        SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker
+    ) {
+        var patchLoader = super.syntheticVectorsLoader(sourceFilter, checker);
         if (patchLoader == null) {
             return null;
         }

server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java

@@ -912,23 +912,30 @@ public ObjectMapper findParentMapper(String leafFieldPath) {
         return null;
     }
 
-    private static SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(Mapper mapper, SourceFilter sourceFilter) {
+    private static SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(
+        Mapper mapper,
+        SourceFilter sourceFilter,
+        SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker
+    ) {
         if (sourceFilter != null && sourceFilter.isPathFiltered(mapper.fullPath(), false)) {
             return null;
         }
         if (mapper instanceof ObjectMapper objMapper) {
-            return objMapper.syntheticVectorsLoader(sourceFilter);
+            return objMapper.syntheticVectorsLoader(sourceFilter, checker);
         } else if (mapper instanceof FieldMapper fieldMapper) {
-            return fieldMapper.syntheticVectorsLoader();
+            return fieldMapper.syntheticVectorsLoader(checker);
         } else {
             return null;
         }
     }
 
-    SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceFilter sourceFilter) {
+    SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(
+        SourceFilter sourceFilter,
+        SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker
+    ) {
         var loaders = mappers.values()
             .stream()
-            .map(m -> syntheticVectorsLoader(m, sourceFilter))
+            .map(m -> syntheticVectorsLoader(m, sourceFilter, checker))
             .filter(l -> l != null)
             .collect(Collectors.toList());
         if (loaders.isEmpty()) {

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -615,4 +615,8 @@ private static void removeSyntheticVectorFields(
                 destination.copyCurrentEvent(parser);
         }
     }
+
+    public String[] getExcludes() {
+        return sourceFilter == null ? Strings.EMPTY_ARRAY : sourceFilter.getExcludes();
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java

@@ -548,5 +548,9 @@ interface Leaf {
              */
             void load(int doc, List<SyntheticVectorPatch> acc) throws IOException;
         }
+
+        interface AutoHybridChecker {
+            boolean check(FieldMapper mapper);
+        }
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -3041,9 +3041,14 @@ public String toString() {
 
     @Override
     public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() {
-        if (isExcludeSourceVectors) {
-            var syntheticField = new IndexedSyntheticFieldLoader(indexCreatedVersion, fieldType().similarity);
-            return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyVectorAsList);
+        var syntheticField = new IndexedSyntheticFieldLoader(indexCreatedVersion, fieldType().similarity);
+        return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyVectorAsList);
+    }
+
+    @Override
+    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) {
+        if (isExcludeSourceVectors || checker.check(this)) {
+            return syntheticVectorsLoader();
         }
         return null;
     }

server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java

@@ -327,9 +327,14 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
 
     @Override
     public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() {
-        if (isExcludeSourceVectors) {
-            var syntheticField = new SparseVectorSyntheticFieldLoader(fullPath(), leafName());
-            return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyAsMap);
+        var syntheticField = new SparseVectorSyntheticFieldLoader(fullPath(), leafName());
+        return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyAsMap);
+    }
+
+    @Override
+    public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) {
+        if (isExcludeSourceVectors || checker.check(this)) {
+            return syntheticVectorsLoader();
         }
         return null;
     }