diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java index 92aa897bf6287..8fa293de93554 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java @@ -235,4 +235,61 @@ public void testReindexIncludeVectors() throws Exception { searchResponse.decRef(); } } + + public void testReindexAutoIncludeVectors() throws Exception { + var resp1 = prepareCreate("test").setSettings( + Settings.builder().put(IndexSettings.INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.getKey(), false).build() + ).setMapping( + Map.of( + "_source", + Map.of("enabled", true, "excludes", List.of("foo", "bar")), + "properties", + Map.of( + "foo", Map.of("type", "dense_vector", "similarity", "l2_norm"), + "bar", Map.of("type", "sparse_vector", "store", true) + ) + ) + ).get(); + assertAcked(resp1); + + var resp2 = prepareCreate("test_reindex").setSettings( + Settings.builder().put(IndexSettings.INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.getKey(), false).build() + ).setMapping( + Map.of( + "_source", + Map.of("enabled", true, "excludes", List.of("foo", "bar")), + "properties", + Map.of( + "foo", Map.of("type", "dense_vector", "similarity", "l2_norm"), + "bar", Map.of("type", "sparse_vector", "store", true) + ) + ) + ).get(); + assertAcked(resp2); + + indexRandom( + true, + prepareIndex("test").setId("1").setSource("foo", List.of(3f, 2f, 1.5f), "bar", Map.of("token_1", 4f, "token_2", 7f)) + ); + + // Copy all the docs + ReindexRequestBuilder copy = reindex().source("test").destination("test_reindex").refresh(true); + var reindexResponse = copy.get(); + assertThat(reindexResponse, matcher().created(1)); + + var searchResponse = prepareSearch("test_reindex").get(); + try { + assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(1L)); + assertThat(searchResponse.getHits().getHits().length, equalTo(1)); + var sourceMap = searchResponse.getHits().getAt(0).getSourceAsMap(); + assertThat(sourceMap.get("foo"), anyOf(equalTo(List.of(3f, 2f, 1.5f)), equalTo(List.of(3d, 2d, 1.5d)))); + assertThat( + sourceMap.get("bar"), + anyOf(equalTo(Map.of("token_1", 4f, "token_2", 7f)), equalTo(Map.of("token_1", 4d, "token_2", 7d))) + ); + } finally { + searchResponse.decRef(); + } + } + } diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java index 54010cab0f3f4..5a46b4a4b9f7b 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java @@ -83,7 +83,9 @@ public LuceneChangesSnapshot( this.lastSeenSeqNo = fromSeqNo - 1; final TopDocs topDocs = nextTopDocs(); this.maxDocIndex = topDocs.scoreDocs.length; - this.syntheticVectorPatchLoader = mapperService.mappingLookup().getMapping().syntheticVectorsLoader(null); + this.syntheticVectorPatchLoader = mapperService.mappingLookup() + .getMapping() + .syntheticVectorsLoader(null, mapperService.mappingLookup().isFieldMapperAutoHybrid()); fillParallelArray(topDocs.scoreDocs, parallelArray); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index a43575b8f990c..e573a8e99c4e8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -464,6 +464,13 @@ public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() { return null; } + public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) { + if (checker.check(this)) { + return syntheticVectorsLoader(); + } + return null; + } + /** *

* Specifies the mode of synthetic source support by the mapper. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java index 24de538bab81a..13ec533af2637 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -138,8 +138,11 @@ Mapping mappingUpdate(RootObjectMapper rootObjectMapper) { * @return a {@link SourceLoader.SyntheticVectorsLoader} for extracting synthetic vectors, * potentially using the provided filter */ - public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(@Nullable SourceFilter filter) { - return root.syntheticVectorsLoader(filter); + public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader( + @Nullable SourceFilter filter, + SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker + ) { + return root.syntheticVectorsLoader(filter, checker); } private boolean isSourceSynthetic() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 64461d0fd2fd5..fc2f3ae49e4d4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -194,7 +194,7 @@ private MappingLookup( if (mapper instanceof InferenceFieldMapper inferenceFieldMapper) { inferenceFields.put(mapper.fullPath(), inferenceFieldMapper.getMetadata(fieldTypeLookup.sourcePaths(mapper.fullPath()))); } - if (mapper.syntheticVectorsLoader() != null) { + if (mapper.syntheticVectorsLoader(this.isFieldMapperAutoHybrid()) != null) { syntheticVectorFields.add(mapper.fullPath()); } } @@ -489,6 +489,23 @@ public boolean isSourceSynthetic() { return sfm != null && sfm.isSynthetic(); } + /** + * Auto use partial synthetic source combine with stored source + */ + public SourceLoader.SyntheticVectorsLoader.AutoHybridChecker isFieldMapperAutoHybrid() { + SourceFieldMapper sfm = mapping.getMetadataMapperByClass(SourceFieldMapper.class); + return fieldMapper -> { + if (sfm != null && sfm.isStored()) { + for (String exclude : sfm.getExcludes()) { + if (exclude.equals(fieldMapper.fullPath())) { + return true; + } + } + } + return false; + }; + } + /** * Build something to load source {@code _source}. */ @@ -496,7 +513,7 @@ public SourceLoader newSourceLoader(@Nullable SourceFilter filter, SourceFieldMe if (isSourceSynthetic()) { return new SourceLoader.Synthetic(filter, () -> mapping.syntheticFieldLoader(filter), metrics, mapping.ignoredSourceFormat()); } - var syntheticVectorsLoader = mapping.syntheticVectorsLoader(filter); + var syntheticVectorsLoader = mapping.syntheticVectorsLoader(filter, this.isFieldMapperAutoHybrid()); if (syntheticVectorsLoader != null) { return new SourceLoader.SyntheticVectors(removeExcludedSyntheticVectorFields(filter), syntheticVectorsLoader); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index 7cfa7ef1b7988..d382844f35439 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -410,8 +410,11 @@ protected MapperMergeContext createChildContext(MapperMergeContext mapperMergeCo } @Override - protected SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceFilter sourceFilter) { - var patchLoader = super.syntheticVectorsLoader(sourceFilter); + protected SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader( + SourceFilter sourceFilter, + SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker + ) { + var patchLoader = super.syntheticVectorsLoader(sourceFilter, checker); if (patchLoader == null) { return null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 33ed032730561..f2de0c7a65bcc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -912,23 +912,30 @@ public ObjectMapper findParentMapper(String leafFieldPath) { return null; } - private static SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(Mapper mapper, SourceFilter sourceFilter) { + private static SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader( + Mapper mapper, + SourceFilter sourceFilter, + SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker + ) { if (sourceFilter != null && sourceFilter.isPathFiltered(mapper.fullPath(), false)) { return null; } if (mapper instanceof ObjectMapper objMapper) { - return objMapper.syntheticVectorsLoader(sourceFilter); + return objMapper.syntheticVectorsLoader(sourceFilter, checker); } else if (mapper instanceof FieldMapper fieldMapper) { - return fieldMapper.syntheticVectorsLoader(); + return fieldMapper.syntheticVectorsLoader(checker); } else { return null; } } - SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceFilter sourceFilter) { + SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader( + SourceFilter sourceFilter, + SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker + ) { var loaders = mappers.values() .stream() - .map(m -> syntheticVectorsLoader(m, sourceFilter)) + .map(m -> syntheticVectorsLoader(m, sourceFilter, checker)) .filter(l -> l != null) .collect(Collectors.toList()); if (loaders.isEmpty()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index a4c8b0a5b50b1..74a29115cad5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -615,4 +615,8 @@ private static void removeSyntheticVectorFields( destination.copyCurrentEvent(parser); } } + + public String[] getExcludes() { + return sourceFilter == null ? Strings.EMPTY_ARRAY : sourceFilter.getExcludes(); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 8efe6219b059a..2b48d393606ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -548,5 +548,9 @@ interface Leaf { */ void load(int doc, List acc) throws IOException; } + + interface AutoHybridChecker { + boolean check(FieldMapper mapper); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 9c5d28a4942d3..785e153d0f684 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -3041,9 +3041,14 @@ public String toString() { @Override public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() { - if (isExcludeSourceVectors) { - var syntheticField = new IndexedSyntheticFieldLoader(indexCreatedVersion, fieldType().similarity); - return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyVectorAsList); + var syntheticField = new IndexedSyntheticFieldLoader(indexCreatedVersion, fieldType().similarity); + return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyVectorAsList); + } + + @Override + public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) { + if (isExcludeSourceVectors || checker.check(this)) { + return syntheticVectorsLoader(); } return null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java index ef76540525898..92f3ef5880e85 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java @@ -327,9 +327,14 @@ protected SyntheticSourceSupport syntheticSourceSupport() { @Override public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() { - if (isExcludeSourceVectors) { - var syntheticField = new SparseVectorSyntheticFieldLoader(fullPath(), leafName()); - return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyAsMap); + var syntheticField = new SparseVectorSyntheticFieldLoader(fullPath(), leafName()); + return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyAsMap); + } + + @Override + public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) { + if (isExcludeSourceVectors || checker.check(this)) { + return syntheticVectorsLoader(); } return null; } diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index e2f314abc553f..a54e15e55a292 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -406,9 +406,14 @@ protected SyntheticSourceSupport syntheticSourceSupport() { @Override public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader() { - if (isExcludeSourceVectors) { - var syntheticField = new DocValuesSyntheticFieldLoader(); - return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyVectorsAsList); + var syntheticField = new DocValuesSyntheticFieldLoader(); + return new SyntheticVectorsPatchFieldLoader(syntheticField, syntheticField::copyVectorsAsList); + } + + @Override + public SourceLoader.SyntheticVectorsLoader syntheticVectorsLoader(SourceLoader.SyntheticVectorsLoader.AutoHybridChecker checker) { + if (isExcludeSourceVectors || checker.check(this)) { + return syntheticVectorsLoader(); } return null; }