From 6652523c9c3989348c6232277cab76ea9c513653 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 10 Oct 2025 23:39:25 +0200 Subject: [PATCH 01/14] BlockSourceReader should apply source filtering --- .../esql/ValuesSourceReaderBenchmark.java | 2 +- .../extras/MatchOnlyTextFieldMapper.java | 10 ++- .../mapper/extras/ScaledFloatFieldMapper.java | 9 ++- .../mapper/AbstractGeometryFieldMapper.java | 8 ++- .../index/mapper/BlockSourceReader.java | 55 +++++++++----- .../index/mapper/BooleanFieldMapper.java | 9 ++- .../index/mapper/DateFieldMapper.java | 9 ++- .../FallbackSyntheticSourceBlockLoader.java | 2 +- .../index/mapper/IpFieldMapper.java | 10 ++- .../index/mapper/KeywordFieldMapper.java | 8 ++- .../index/mapper/NumberFieldMapper.java | 71 ++++++++++++++----- .../index/mapper/TextFieldMapper.java | 9 ++- .../vectors/DenseVectorFieldMapper.java | 16 ++++- .../search/fetch/StoredFieldsSpec.java | 28 +++++++- .../IgnoredSourceFieldLoaderTests.java | 12 ++-- .../fieldvisitor/StoredFieldLoaderTests.java | 2 +- .../index/mapper/BlockSourceReaderTests.java | 11 ++- .../compute/lucene/ShardContext.java | 3 +- .../lucene/read/ValuesFromManyReader.java | 2 +- .../lucene/read/ValuesFromSingleReader.java | 2 +- .../read/ValuesSourceReaderOperator.java | 9 ++- .../elasticsearch/compute/OperatorTests.java | 2 +- .../lucene/LuceneQueryEvaluatorTests.java | 2 +- .../lucene/LuceneSourceOperatorTests.java | 3 +- .../ValueSourceReaderTypeConversionTests.java | 12 ++-- .../read/ValuesSourceReaderOperatorTests.java | 20 +++--- .../xpack/esql/action/LookupFromIndexIT.java | 2 +- .../planner/EsPhysicalOperationProviders.java | 6 +- .../mapper/SemanticTextFieldMapper.java | 10 ++- .../unsignedlong/UnsignedLongFieldMapper.java | 8 ++- 30 files changed, 264 insertions(+), 88 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java index bbd58d3309ab6..2cf7cafa3dd74 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java @@ -368,7 +368,7 @@ public void benchmark() { blockFactory, ByteSizeValue.ofMb(1).getBytes(), fields(name), - List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException("can't load _source here"); }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))), 0 diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 70c28ee18184f..d0cddf01dff4f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; @@ -604,11 +605,18 @@ protected String delegatingTo() { } } + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (isSyntheticSourceEnabled()) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } + // fallback to _source (synthetic or not) SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); // MatchOnlyText never has norms, so we have to use the field names field BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup, name(), format); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index d2bae840b905d..544165758b463 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -397,13 +397,18 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (isSyntheticSource) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name())); BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup); + return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup, name(), format); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 78a8f85f5cf66..6ae4e859eedb3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -187,7 +187,13 @@ protected Object parseSourceValue(Object value) { protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB); // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) - return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (blContext.indexSettings().getIndexMappingSourceMode() == SourceFieldMapper.Mode.SYNTHETIC) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } + return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), name(), format); } protected abstract Object nullValueAsSource(T nullValue); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 3b467e383f20d..a1d6a81c8abb5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -18,6 +18,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -25,6 +26,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Set; /** * Loads values from {@code _source}. This whole process is very slow and cast-tastic, @@ -94,10 +96,19 @@ public interface LeafIteratorLookup { private abstract static class SourceBlockLoader implements BlockLoader { protected final ValueFetcher fetcher; private final LeafIteratorLookup lookup; - - private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + private final String fieldName; + private final IgnoredSourceFormat ignoredSourceFormat; + + private SourceBlockLoader( + ValueFetcher fetcher, + LeafIteratorLookup lookup, + String fieldName, + IgnoredSourceFormat ignoredSourceFormat + ) { this.fetcher = fetcher; this.lookup = lookup; + this.fieldName = fieldName; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -107,7 +118,7 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) @Override public final StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, Set.of(fieldName)); } @Override @@ -143,8 +154,8 @@ public final String toString() { * Load {@code boolean}s from {@code _source}. */ public static class BooleansBlockLoader extends SourceBlockLoader { - public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { + super(fetcher, lookup, name, format); } @Override @@ -183,8 +194,8 @@ public String toString() { * Load {@link BytesRef}s from {@code _source}. */ public static class BytesRefsBlockLoader extends SourceBlockLoader { - public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { + super(fetcher, lookup, name, format); } @Override @@ -204,8 +215,8 @@ protected String name() { } public static class GeometriesBlockLoader extends SourceBlockLoader { - public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { + super(fetcher, lookup, name, format); } @Override @@ -267,8 +278,8 @@ public String toString() { * Load {@code double}s from {@code _source}. */ public static class DoublesBlockLoader extends SourceBlockLoader { - public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { + super(fetcher, lookup, name, format); } @Override @@ -309,8 +320,14 @@ public String toString() { public static class DenseVectorBlockLoader extends SourceBlockLoader { private final int dimensions; - public DenseVectorBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, int dimensions) { - super(fetcher, lookup); + public DenseVectorBlockLoader( + ValueFetcher fetcher, + LeafIteratorLookup lookup, + int dimensions, + String name, + IgnoredSourceFormat format + ) { + super(fetcher, lookup, name, format); this.dimensions = dimensions; } @@ -350,8 +367,8 @@ public String toString() { * Load {@code int}s from {@code _source}. */ public static class IntsBlockLoader extends SourceBlockLoader { - public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { + super(fetcher, lookup, name, format); } @Override @@ -390,8 +407,8 @@ public String toString() { * Load {@code long}s from {@code _source}. */ public static class LongsBlockLoader extends SourceBlockLoader { - public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { + super(fetcher, lookup, name, format); } @Override @@ -430,8 +447,8 @@ public String toString() { * Load {@code ip}s from {@code _source}. */ public static class IpsBlockLoader extends SourceBlockLoader { - public IpsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public IpsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { + super(fetcher, lookup, name, format); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 56ff91eef69f0..5d5130349db04 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -364,11 +364,18 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (isSyntheticSource) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } + ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); BlockSourceReader.LeafIteratorLookup lookup = indexType.hasTerms() || isStored() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup); + return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup, name(), format); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index cc4ec0c2f8296..3537745d8ab01 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -995,11 +995,16 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (isSyntheticSource) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } BlockSourceReader.LeafIteratorLookup lookup = isStored() || indexType.hasPoints() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, name(), format); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index b66730d6704e0..59d36e58fd3af 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -65,7 +65,7 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep @Override public StoredFieldsSpec rowStrideStoredFieldSpec() { - return new StoredFieldsSpec(false, false, Set.of(), new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat)); + return new StoredFieldsSpec(false, false, Set.of(), new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat), null); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 6db87007747ad..36d5f4b4cb14a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.IpFieldScript; import org.elasticsearch.script.Script; @@ -481,12 +482,17 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (isSyntheticSource && blContext.parentField(name()) == null) { return blockLoaderFromFallbackSyntheticSource(blContext); } - + IgnoredSourceFormat format; + if (isSyntheticSource) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFormat.NO_IGNORED_SOURCE; + } // see #indexValue BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && hasPoints ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, name(), format); } private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 89a158cf3bdb4..0557cc80e0de6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -820,8 +820,14 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (isSyntheticSourceEnabled()) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext)); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext), name(), format); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 6ca488e137d60..243e90a3a1066 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -45,6 +45,7 @@ import org.elasticsearch.index.fielddata.SourceValueFetcherSortedNumericIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedDoublesIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.mapper.TimeSeriesParams.MetricType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.DoubleFieldScript; @@ -494,8 +495,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + IgnoredSourceFormat format + ) { + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, name(), format); } @Override @@ -688,8 +693,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + IgnoredSourceFormat format + ) { + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, name(), format); } @Override @@ -848,8 +857,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + IgnoredSourceFormat format + ) { + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, name(), format); } @Override @@ -976,8 +989,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + IgnoredSourceFormat format + ) { + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, name(), format); } @Override @@ -1104,8 +1121,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + IgnoredSourceFormat format + ) { + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, name(), format); } @Override @@ -1306,8 +1327,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + IgnoredSourceFormat format + ) { + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, name(), format); } @Override @@ -1468,8 +1493,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + IgnoredSourceFormat format + ) { + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup, name(), format); } @Override @@ -1779,7 +1808,11 @@ public void writeValue(XContentBuilder b, long value) throws IOException { abstract BlockLoader blockLoaderFromDocValues(String fieldName); - abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup); + abstract BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + IgnoredSourceFormat format + ); abstract BlockLoader blockLoaderFromFallbackSyntheticSource( String fieldName, @@ -2055,7 +2088,13 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { // We only write the field names field if there aren't doc values or norms ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + IgnoredSourceFormat format; + if (isSyntheticSource) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFormat.NO_IGNORED_SOURCE; + } + return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup, format); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 5acb088040485..cfb7c76061a54 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1116,10 +1116,15 @@ protected String delegatingTo() { if (isSyntheticSourceEnabled() && syntheticSourceDelegate.isEmpty() && parentField == null) { return fallbackSyntheticSourceBlockLoader(blContext); } - + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (isAggregatable()) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } // otherwise, load values from _source (synthetic or not) SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext)); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), name(), format); } FallbackSyntheticSourceBlockLoader fallbackSyntheticSourceBlockLoader(BlockLoaderContext blContext) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 33d99c5628732..d54c9ba4b76ab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -63,6 +63,7 @@ import org.elasticsearch.index.mapper.BlockSourceReader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -2691,9 +2692,20 @@ public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) { return new BlockDocValuesReader.DenseVectorFromBinaryBlockLoader(name(), dims, indexVersionCreated, element.elementType()); } - + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (isSyntheticSource) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.DenseVectorBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, dims); + return new BlockSourceReader.DenseVectorBlockLoader( + sourceValueFetcher(blContext.sourcePaths(name())), + lookup, + dims, + name(), + format + ); } private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java index ce52fabb20049..384c98390e612 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java @@ -10,6 +10,7 @@ package org.elasticsearch.search.fetch; import org.elasticsearch.index.mapper.IgnoredFieldsSpec; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import java.util.Collection; import java.util.HashSet; @@ -25,10 +26,11 @@ public record StoredFieldsSpec( boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields, - IgnoredFieldsSpec ignoredFieldsSpec + IgnoredFieldsSpec ignoredFieldsSpec, + Set sourcePaths ) { public StoredFieldsSpec(boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields) { - this(requiresSource, requiresMetadata, requiredStoredFields, IgnoredFieldsSpec.NONE); + this(requiresSource, requiresMetadata, requiredStoredFields, IgnoredFieldsSpec.NONE, null); } public boolean noRequirements() { @@ -52,6 +54,14 @@ public boolean onlyRequiresIgnoredFields() { */ public static final StoredFieldsSpec NEEDS_SOURCE = new StoredFieldsSpec(true, false, Set.of()); + public static StoredFieldsSpec withSourcePaths(IgnoredSourceFormat ignoredSourceFormat, Set sourcePaths) { + // The fields in source paths might also be in ignored source, so include source paths there as well. + IgnoredFieldsSpec ignoredFieldsSpec = ignoredSourceFormat == IgnoredSourceFormat.NO_IGNORED_SOURCE + ? new IgnoredFieldsSpec(sourcePaths, ignoredSourceFormat) + : IgnoredFieldsSpec.NONE; + return new StoredFieldsSpec(true, false, Set.of(), ignoredFieldsSpec, sourcePaths); + } + /** * Combine these stored field requirements with those from another StoredFieldsSpec */ @@ -70,11 +80,23 @@ public StoredFieldsSpec merge(StoredFieldsSpec other) { mergedFields = new HashSet<>(this.requiredStoredFields); mergedFields.addAll(other.requiredStoredFields); } + Set mergedSourcePaths; + if (this.sourcePaths != null && other.sourcePaths != null) { + mergedSourcePaths = new HashSet<>(this.sourcePaths); + mergedSourcePaths.addAll(other.sourcePaths); + } else if (this.sourcePaths != null) { + mergedSourcePaths = this.sourcePaths; + } else if (other.sourcePaths != null) { + mergedSourcePaths = other.sourcePaths; + } else { + mergedSourcePaths = null; + } return new StoredFieldsSpec( this.requiresSource || other.requiresSource, this.requiresMetadata || other.requiresMetadata, mergedFields, - ignoredFieldsSpec.merge(other.ignoredFieldsSpec) + ignoredFieldsSpec.merge(other.ignoredFieldsSpec), + mergedSourcePaths ); } diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java index e460b5a460194..46d12f2a65e8d 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java @@ -39,7 +39,8 @@ public void testSupports() { false, false, Set.of(), - new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ) ) ); @@ -50,7 +51,8 @@ public void testSupports() { false, false, Set.of(), - new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ) ) ); @@ -61,7 +63,8 @@ public void testSupports() { true, false, Set.of(), - new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ) ) ); @@ -129,7 +132,8 @@ private void testLoader( false, false, Set.of(), - new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ); assertTrue(IgnoredSourceFieldLoader.supports(spec)); iw.addDocument(doc); diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java index 0255506816799..bff84ec345522 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java @@ -48,7 +48,7 @@ private StoredFieldsSpec fieldsSpec( Set ignoredFields, IgnoredSourceFieldMapper.IgnoredSourceFormat format ) { - return new StoredFieldsSpec(false, false, storedFields, new IgnoredFieldsSpec(ignoredFields, format)); + return new StoredFieldsSpec(false, false, storedFields, new IgnoredFieldsSpec(ignoredFields, format), null); } public void testEmpty() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index 1fa9c85a5c738..b07cd35eb4356 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -49,12 +49,19 @@ public void testEmptyArray() throws IOException { } private void loadBlock(LeafReaderContext ctx, Consumer test) throws IOException { + boolean synteticSource = randomBoolean(); + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (synteticSource) { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE; + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field")); BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field"); - BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup); + BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup, "field", format); assertThat(loader.columnAtATimeReader(ctx), nullValue()); BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx); - assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.NEEDS_SOURCE)); + assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.withSourcePaths(format, Set.of("field")))); BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null), loader.rowStrideStoredFieldSpec().requiresSource() ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) : null diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java index d20a002407be6..965d1a5037205 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.Set; /** * Context of each shard we're operating against. @@ -48,7 +49,7 @@ public interface ShardContext extends RefCounted { /** * Build something to load source {@code _source}. */ - SourceLoader newSourceLoader(); + SourceLoader newSourceLoader(Set sourcePaths); /** * Returns something to load values from this field into a {@link Block}. diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java index 6f00e97a1f9f2..8acc4d2a4cc93 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java @@ -181,7 +181,7 @@ private void fieldsMoved(LeafReaderContext ctx, int shard) throws IOException { } SourceLoader sourceLoader = null; if (storedFieldsSpec.requiresSource()) { - sourceLoader = operator.shardContexts.get(shard).newSourceLoader().get(); + sourceLoader = operator.shardContexts.get(shard).newSourceLoader().apply(storedFieldsSpec.sourcePaths()); storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } storedFields = new BlockLoaderStoredFieldsFromLeafLoader( diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java index 59e54103eb310..6203842dc9e0d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java @@ -139,7 +139,7 @@ private void loadFromRowStrideReaders( SourceLoader sourceLoader = null; ValuesSourceReaderOperator.ShardContext shardContext = operator.shardContexts.get(shard); if (storedFieldsSpec.requiresSource()) { - sourceLoader = shardContext.newSourceLoader().get(); + sourceLoader = shardContext.newSourceLoader().apply(storedFieldsSpec.sourcePaths()); storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } if (storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java index 3f3d73eec88ae..0a230f1bbe609 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java @@ -28,9 +28,10 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.function.Function; import java.util.function.IntFunction; -import java.util.function.Supplier; /** * Operator that extracts doc_values from a Lucene index out of pages that have been produced by {@link LuceneSourceOperator} @@ -91,7 +92,11 @@ public String describe() { */ public record FieldInfo(String name, ElementType type, boolean nullsFiltered, IntFunction blockLoader) {} - public record ShardContext(IndexReader reader, Supplier newSourceLoader, double storedFieldsSequentialProportion) {} + public record ShardContext( + IndexReader reader, + Function, SourceLoader> newSourceLoader, + double storedFieldsSequentialProportion + ) {} final BlockFactory blockFactory; /** diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java index 56acaa86c299d..758a9f55d5655 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java @@ -190,7 +190,7 @@ public void testPushRoundToToQuery() throws IOException { f -> new BlockDocValuesReader.LongsBlockLoader("v") ) ), - List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException(); }, 0.8)), 0 diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java index 5b25fb6fb64af..505cd7c212bb9 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java @@ -210,7 +210,7 @@ private List runQuery(Set values, Query query, boolean shuffleDocs unused -> new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(FIELD) ) ), - List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException(); }, 0.2)), 0 diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java index 3249031824210..609cc5875d3ff 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @@ -456,7 +457,7 @@ public IndexSearcher searcher() { } @Override - public SourceLoader newSourceLoader() { + public SourceLoader newSourceLoader(Set sourcePaths) { return SourceLoader.FROM_STORED_SOURCE; } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java index 4e562217a00e3..15c43ce16acfa 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java @@ -208,7 +208,7 @@ private MapperService mapperService(String indexKey) { private List initShardContexts() { return INDICES.keySet() .stream() - .map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), () -> SourceLoader.FROM_STORED_SOURCE, 0.2)) + .map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2)) .toList(); } @@ -1325,7 +1325,11 @@ public void testWithNulls() throws IOException { LuceneOperator.NO_LIMIT, false // no scoring ); - var vsShardContext = new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE, 0.2); + var vsShardContext = new ValuesSourceReaderOperator.ShardContext( + reader(indexKey), + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, + 0.2 + ); try ( Driver driver = TestDriverFactory.create( driverContext, @@ -1455,7 +1459,7 @@ public void testDescriptionOfMany() throws IOException { ValuesSourceReaderOperator.Factory factory = new ValuesSourceReaderOperator.Factory( ByteSizeValue.ofGb(1), cases.stream().map(c -> c.info).toList(), - List.of(new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE, 0.2)), + List.of(new ValuesSourceReaderOperator.ShardContext(reader(indexKey), (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2)), 0 ); assertThat(factory.describe(), equalTo("ValuesSourceReaderOperator[fields = [" + cases.size() + " fields]]")); @@ -1484,7 +1488,7 @@ public void testManyShards() throws IOException { for (int s = 0; s < shardCount; s++) { contexts.add(new LuceneSourceOperatorTests.MockShardContext(readers[s], s)); readerShardContexts.add( - new ValuesSourceReaderOperator.ShardContext(readers[s], () -> SourceLoader.FROM_STORED_SOURCE, 0.2) + new ValuesSourceReaderOperator.ShardContext(readers[s], (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2) ); } var luceneFactory = new LuceneSourceOperator.Factory( diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java index 3a83d365f6f57..801a4e7fc9056 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java @@ -164,7 +164,7 @@ static Operator.OperatorFactory factory(IndexReader reader, String name, Element List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -500,7 +500,7 @@ public void testManySingleDocPages() { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -620,7 +620,7 @@ private void loadSimpleAndAssert( List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -639,7 +639,7 @@ private void loadSimpleAndAssert( List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -738,7 +738,7 @@ private void testLoadAllStatus(boolean allInOnePage) { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -972,7 +972,7 @@ private void testLoadLong(boolean shuffle, boolean manySegments) throws IOExcept List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1641,7 +1641,7 @@ public void testNullsShared() { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1693,7 +1693,7 @@ private void testSequentialStoredFields(boolean sequential, int docCount) throws List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1728,7 +1728,7 @@ public void testDescriptionOfMany() throws IOException { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1761,7 +1761,7 @@ public void testManyShards() throws IOException { readerShardContexts.add( new ValuesSourceReaderOperator.ShardContext( readers[s], - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java index 7da6db1b5235e..9a551093bb7a0 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java @@ -360,7 +360,7 @@ private void runLookup(List keyTypes, PopulateIndices populateIndices, ValuesSourceReaderOperator.Factory reader = new ValuesSourceReaderOperator.Factory( PlannerSettings.VALUES_LOADING_JUMBO_SIZE.getDefault(Settings.EMPTY), fieldInfos, - List.of(new ValuesSourceReaderOperator.ShardContext(searchContext.getSearchExecutionContext().getIndexReader(), () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(searchContext.getSearchExecutionContext().getIndexReader(), (paths) -> { throw new IllegalStateException("can't load source here"); }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))), 0 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index cd5cfab6eb102..0ce74047affc4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -59,6 +59,7 @@ import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.xpack.esql.core.expression.Attribute; @@ -453,8 +454,9 @@ public String shardIdentifier() { } @Override - public SourceLoader newSourceLoader() { - return ctx.newSourceLoader(null, false); + public SourceLoader newSourceLoader(Set sourcePaths) { + var filter = sourcePaths != null ? new SourceFilter(sourcePaths.toArray(new String[0]), null) : null; + return ctx.newSourceLoader(filter, false); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 1a8b162eb1b46..0d667ffedcd58 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -42,6 +42,7 @@ import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.InferenceFieldMapper; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; @@ -56,6 +57,7 @@ import org.elasticsearch.index.mapper.NestedObjectMapper; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.SimpleMappedFieldType; +import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -1054,7 +1056,13 @@ private String generateInvalidQueryInferenceResultsMessage(StringBuilder baseMes public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { String name = useLegacyFormat ? name().concat(".text") : name(); SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name)); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (blContext.indexSettings().getIndexMappingSourceMode() == SourceFieldMapper.Mode.SYNTHETIC) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), name(), format); } private class SemanticTextFieldValueFetcher implements ValueFetcher { diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 74459072b8979..adec7dbdb65c1 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -411,11 +411,17 @@ protected Object parseSourceValue(Object value) { return unsignedToSortableSignedLong(parseUnsignedLong(value)); } }; + IgnoredSourceFieldMapper.IgnoredSourceFormat format; + if (isSyntheticSource) { + format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); + } else { + format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup); + return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup, name(), format); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { From 2a849b9f78188e366981d9ab3818be209919da99 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Sun, 12 Oct 2025 13:25:12 +0200 Subject: [PATCH 02/14] iter --- .../index/mapper/extras/MatchOnlyTextFieldMapper.java | 8 +------- .../index/mapper/extras/ScaledFloatFieldMapper.java | 7 +------ .../index/mapper/AbstractGeometryFieldMapper.java | 7 +------ .../elasticsearch/index/mapper/BooleanFieldMapper.java | 8 +------- .../org/elasticsearch/index/mapper/DateFieldMapper.java | 7 +------ .../org/elasticsearch/index/mapper/IpFieldMapper.java | 7 +------ .../elasticsearch/index/mapper/KeywordFieldMapper.java | 7 +------ .../org/elasticsearch/index/mapper/MappedFieldType.java | 9 +++++++++ .../elasticsearch/index/mapper/NumberFieldMapper.java | 7 +------ .../org/elasticsearch/index/mapper/TextFieldMapper.java | 7 +------ .../index/mapper/vectors/DenseVectorFieldMapper.java | 7 +------ .../xpack/inference/mapper/SemanticTextFieldMapper.java | 7 +------ .../xpack/unsignedlong/UnsignedLongFieldMapper.java | 7 +------ 13 files changed, 21 insertions(+), 74 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index d0cddf01dff4f..1cbb42a4fd110 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -605,13 +605,7 @@ protected String delegatingTo() { } } - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (isSyntheticSourceEnabled()) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } - + IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); // fallback to _source (synthetic or not) SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); // MatchOnlyText never has norms, so we have to use the field names field diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 544165758b463..a28bb18c82a07 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -397,12 +397,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (isSyntheticSource) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } + IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name())); BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 6ae4e859eedb3..02cf8224ce96b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -187,12 +187,7 @@ protected Object parseSourceValue(Object value) { protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB); // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (blContext.indexSettings().getIndexMappingSourceMode() == SourceFieldMapper.Mode.SYNTHETIC) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } + IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), name(), format); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 5d5130349db04..5fe438d7ca3d9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -364,13 +364,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (isSyntheticSource) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } - + IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); BlockSourceReader.LeafIteratorLookup lookup = indexType.hasTerms() || isStored() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 3537745d8ab01..38c398a57dc9b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -995,12 +995,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (isSyntheticSource) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } + IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); BlockSourceReader.LeafIteratorLookup lookup = isStored() || indexType.hasPoints() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 36d5f4b4cb14a..862f9ed6e2f45 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -482,12 +482,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (isSyntheticSource && blContext.parentField(name()) == null) { return blockLoaderFromFallbackSyntheticSource(blContext); } - IgnoredSourceFormat format; - if (isSyntheticSource) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFormat.NO_IGNORED_SOURCE; - } + IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); // see #indexValue BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && hasPoints ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 0557cc80e0de6..e2a2baebd8910 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -820,12 +820,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (isSyntheticSourceEnabled()) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } + IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext), name(), format); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 771a54afa267a..bcc9bc78fad3a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -703,6 +703,15 @@ public interface BlockLoaderContext { * The {@code _field_names} field mapper, mostly used to check if it is enabled. */ FieldNamesFieldMapper.FieldNamesFieldType fieldNames(); + + default IgnoredSourceFieldMapper.IgnoredSourceFormat getIgnoredSourceFormat() { + IndexSettings indexSettings = indexSettings(); + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) { + return IgnoredSourceFieldMapper.ignoredSourceFormat(indexSettings.getIndexVersionCreated()); + } else { + return IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 243e90a3a1066..170c72bbdc9b0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -2088,12 +2088,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { // We only write the field names field if there aren't doc values or norms ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - IgnoredSourceFormat format; - if (isSyntheticSource) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFormat.NO_IGNORED_SOURCE; - } + IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup, format); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index cfb7c76061a54..567487c224f09 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1116,12 +1116,7 @@ protected String delegatingTo() { if (isSyntheticSourceEnabled() && syntheticSourceDelegate.isEmpty() && parentField == null) { return fallbackSyntheticSourceBlockLoader(blContext); } - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (isAggregatable()) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } + IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); // otherwise, load values from _source (synthetic or not) SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), name(), format); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index d54c9ba4b76ab..623865b42038c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2692,12 +2692,7 @@ public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) { return new BlockDocValuesReader.DenseVectorFromBinaryBlockLoader(name(), dims, indexVersionCreated, element.elementType()); } - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (isSyntheticSource) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } + IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupMatchingAll(); return new BlockSourceReader.DenseVectorBlockLoader( sourceValueFetcher(blContext.sourcePaths(name())), diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 0d667ffedcd58..088b4cc214a7f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -1056,12 +1056,7 @@ private String generateInvalidQueryInferenceResultsMessage(StringBuilder baseMes public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { String name = useLegacyFormat ? name().concat(".text") : name(); SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name)); - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (blContext.indexSettings().getIndexMappingSourceMode() == SourceFieldMapper.Mode.SYNTHETIC) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } + IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), name(), format); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index adec7dbdb65c1..6810ade52044d 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -411,12 +411,7 @@ protected Object parseSourceValue(Object value) { return unsignedToSortableSignedLong(parseUnsignedLong(value)); } }; - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (isSyntheticSource) { - format = IgnoredSourceFieldMapper.ignoredSourceFormat(blContext.indexSettings().getIndexVersionCreated()); - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } + IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) From 0dbadd4e535934e502af49ec686a5534a3337179 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sun, 12 Oct 2025 11:31:42 +0000 Subject: [PATCH 03/14] [CI] Auto commit changes from spotless --- .../xpack/inference/mapper/SemanticTextFieldMapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 088b4cc214a7f..adc3898ec5dfb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -57,7 +57,6 @@ import org.elasticsearch.index.mapper.NestedObjectMapper; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.SimpleMappedFieldType; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextFieldMapper; From b65a3971761f15707ad353b080196498ff8b63fe Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Sun, 12 Oct 2025 14:11:34 +0200 Subject: [PATCH 04/14] iter2 --- .../extras/MatchOnlyTextFieldMapper.java | 3 +- .../mapper/extras/ScaledFloatFieldMapper.java | 3 +- .../mapper/AbstractGeometryFieldMapper.java | 8 +-- .../index/mapper/ArraySourceValueFetcher.java | 4 ++ .../index/mapper/BlockSourceReader.java | 55 +++++++++---------- .../index/mapper/BooleanFieldMapper.java | 3 +- .../index/mapper/DateFieldMapper.java | 4 +- .../index/mapper/KeywordFieldMapper.java | 4 +- .../index/mapper/NumberFieldMapper.java | 14 ++--- .../index/mapper/SourceValueFetcher.java | 4 ++ .../index/mapper/TextFieldMapper.java | 4 +- .../vectors/DenseVectorFieldMapper.java | 8 +-- .../index/mapper/BlockSourceReaderTests.java | 2 +- .../mapper/SemanticTextFieldMapper.java | 5 +- .../unsignedlong/UnsignedLongFieldMapper.java | 3 +- 15 files changed, 58 insertions(+), 66 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 1cbb42a4fd110..75b9cf309d8d7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -605,12 +605,11 @@ protected String delegatingTo() { } } - IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); // fallback to _source (synthetic or not) SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); // MatchOnlyText never has norms, so we have to use the field names field BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup, name(), format); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup, blContext.getIgnoredSourceFormat()); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index a28bb18c82a07..0e2dc35a73c22 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -397,13 +397,12 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name())); BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup, name(), format); + return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup, blContext.getIgnoredSourceFormat()); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 02cf8224ce96b..4c88cbf09ec20 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -172,7 +172,7 @@ protected Object parseSourceValue(Object value) { }; } - public ValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format) { + public ArraySourceValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format) { Function, List> formatter = getFormatter(format != null ? format : GeometryFormatterFactory.GEOJSON); return new ArraySourceValueFetcher(sourcePaths, nullValueAsSource(nullValue)) { @Override @@ -185,10 +185,10 @@ protected Object parseSourceValue(Object value) { } protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { - ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB); + var fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB); // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) - IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); - return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), name(), format); + var format = blContext.getIgnoredSourceFormat(); + return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), format); } protected abstract Object nullValueAsSource(T nullValue); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index fa835e4356667..00e4fc3905ea0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -79,6 +79,10 @@ public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NEEDS_SOURCE; } + public Set getSourcePaths() { + return sourcePaths; + } + /** * Given a value that has been extracted from a document's source, parse it into a standard * format. This parsing logic should closely mirror the value parsing in diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index a1d6a81c8abb5..32bf3962cb3fe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -96,18 +96,19 @@ public interface LeafIteratorLookup { private abstract static class SourceBlockLoader implements BlockLoader { protected final ValueFetcher fetcher; private final LeafIteratorLookup lookup; - private final String fieldName; + private final Set sourcePaths; private final IgnoredSourceFormat ignoredSourceFormat; - private SourceBlockLoader( - ValueFetcher fetcher, - LeafIteratorLookup lookup, - String fieldName, - IgnoredSourceFormat ignoredSourceFormat - ) { + private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat ignoredSourceFormat) { this.fetcher = fetcher; this.lookup = lookup; - this.fieldName = fieldName; + if (fetcher instanceof SourceValueFetcher sourceValueFetcher) { + this.sourcePaths = sourceValueFetcher.getSourcePaths(); + } else if (fetcher instanceof ArraySourceValueFetcher arraySourceValueFetcher) { + this.sourcePaths = arraySourceValueFetcher.getSourcePaths(); + } else { + throw new IllegalStateException("unexpected fetcher type " + fetcher); + } this.ignoredSourceFormat = ignoredSourceFormat; } @@ -118,7 +119,7 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) @Override public final StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, Set.of(fieldName)); + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, sourcePaths); } @Override @@ -154,8 +155,8 @@ public final String toString() { * Load {@code boolean}s from {@code _source}. */ public static class BooleansBlockLoader extends SourceBlockLoader { - public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { - super(fetcher, lookup, name, format); + public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { + super(fetcher, lookup, format); } @Override @@ -194,8 +195,8 @@ public String toString() { * Load {@link BytesRef}s from {@code _source}. */ public static class BytesRefsBlockLoader extends SourceBlockLoader { - public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { - super(fetcher, lookup, name, format); + public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { + super(fetcher, lookup, format); } @Override @@ -215,8 +216,8 @@ protected String name() { } public static class GeometriesBlockLoader extends SourceBlockLoader { - public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { - super(fetcher, lookup, name, format); + public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { + super(fetcher, lookup, format); } @Override @@ -278,8 +279,8 @@ public String toString() { * Load {@code double}s from {@code _source}. */ public static class DoublesBlockLoader extends SourceBlockLoader { - public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { - super(fetcher, lookup, name, format); + public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { + super(fetcher, lookup, format); } @Override @@ -320,14 +321,8 @@ public String toString() { public static class DenseVectorBlockLoader extends SourceBlockLoader { private final int dimensions; - public DenseVectorBlockLoader( - ValueFetcher fetcher, - LeafIteratorLookup lookup, - int dimensions, - String name, - IgnoredSourceFormat format - ) { - super(fetcher, lookup, name, format); + public DenseVectorBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, int dimensions, IgnoredSourceFormat format) { + super(fetcher, lookup, format); this.dimensions = dimensions; } @@ -367,8 +362,8 @@ public String toString() { * Load {@code int}s from {@code _source}. */ public static class IntsBlockLoader extends SourceBlockLoader { - public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { - super(fetcher, lookup, name, format); + public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { + super(fetcher, lookup, format); } @Override @@ -407,8 +402,8 @@ public String toString() { * Load {@code long}s from {@code _source}. */ public static class LongsBlockLoader extends SourceBlockLoader { - public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { - super(fetcher, lookup, name, format); + public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { + super(fetcher, lookup, format); } @Override @@ -448,7 +443,7 @@ public String toString() { */ public static class IpsBlockLoader extends SourceBlockLoader { public IpsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { - super(fetcher, lookup, name, format); + super(fetcher, lookup, format); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 5fe438d7ca3d9..d9b7455febe0c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -364,12 +364,11 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); BlockSourceReader.LeafIteratorLookup lookup = indexType.hasTerms() || isStored() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup, name(), format); + return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup, blContext.getIgnoredSourceFormat()); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 38c398a57dc9b..196647a821b42 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -995,11 +995,11 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); BlockSourceReader.LeafIteratorLookup lookup = isStored() || indexType.hasPoints() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, name(), format); + var format = blContext.getIgnoredSourceFormat(); + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, format); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e2a2baebd8910..0aa193168310b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -820,9 +820,9 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext), name(), format); + var format = blContext.getIgnoredSourceFormat(); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext), format); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 170c72bbdc9b0..8d5cb4bfbb743 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -500,7 +500,7 @@ BlockLoader blockLoaderFromSource( BlockSourceReader.LeafIteratorLookup lookup, IgnoredSourceFormat format ) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, name(), format); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, format); } @Override @@ -698,7 +698,7 @@ BlockLoader blockLoaderFromSource( BlockSourceReader.LeafIteratorLookup lookup, IgnoredSourceFormat format ) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, name(), format); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, format); } @Override @@ -862,7 +862,7 @@ BlockLoader blockLoaderFromSource( BlockSourceReader.LeafIteratorLookup lookup, IgnoredSourceFormat format ) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, name(), format); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, format); } @Override @@ -994,7 +994,7 @@ BlockLoader blockLoaderFromSource( BlockSourceReader.LeafIteratorLookup lookup, IgnoredSourceFormat format ) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, name(), format); + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, format); } @Override @@ -1126,7 +1126,7 @@ BlockLoader blockLoaderFromSource( BlockSourceReader.LeafIteratorLookup lookup, IgnoredSourceFormat format ) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, name(), format); + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, format); } @Override @@ -1332,7 +1332,7 @@ BlockLoader blockLoaderFromSource( BlockSourceReader.LeafIteratorLookup lookup, IgnoredSourceFormat format ) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, name(), format); + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, format); } @Override @@ -1498,7 +1498,7 @@ BlockLoader blockLoaderFromSource( BlockSourceReader.LeafIteratorLookup lookup, IgnoredSourceFormat format ) { - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup, name(), format); + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup, format); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index 7d1e722200ba3..6a93d44b3168b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -101,6 +101,10 @@ public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NEEDS_SOURCE; } + public Set getSourcePaths() { + return sourcePaths; + } + /** * Given a value that has been extracted from a document's source, parse it into a standard * format. This parsing logic should closely mirror the value parsing in diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 567487c224f09..f67cf6d46d921 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1116,10 +1116,10 @@ protected String delegatingTo() { if (isSyntheticSourceEnabled() && syntheticSourceDelegate.isEmpty() && parentField == null) { return fallbackSyntheticSourceBlockLoader(blContext); } - IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); // otherwise, load values from _source (synthetic or not) SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), name(), format); + var format = blContext.getIgnoredSourceFormat(); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), format); } FallbackSyntheticSourceBlockLoader fallbackSyntheticSourceBlockLoader(BlockLoaderContext blContext) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 623865b42038c..a503c0f1cee71 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2694,13 +2694,7 @@ public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { } IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.DenseVectorBlockLoader( - sourceValueFetcher(blContext.sourcePaths(name())), - lookup, - dims, - name(), - format - ); + return new BlockSourceReader.DenseVectorBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, dims, format); } private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index b07cd35eb4356..f87c732208a9f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -58,7 +58,7 @@ private void loadBlock(LeafReaderContext ctx, Consumer test) throws I } ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field")); BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field"); - BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup, "field", format); + BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup, format); assertThat(loader.columnAtATimeReader(ctx), nullValue()); BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx); assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.withSourcePaths(format, Set.of("field")))); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index adc3898ec5dfb..052b163e1f2f6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -42,7 +42,6 @@ import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.InferenceFieldMapper; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; @@ -1055,8 +1054,8 @@ private String generateInvalidQueryInferenceResultsMessage(StringBuilder baseMes public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { String name = useLegacyFormat ? name().concat(".text") : name(); SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name)); - IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), name(), format); + var format = blContext.getIgnoredSourceFormat(); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), format); } private class SemanticTextFieldValueFetcher implements ValueFetcher { diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 6810ade52044d..4de422e51b5a1 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -411,12 +411,11 @@ protected Object parseSourceValue(Object value) { return unsignedToSortableSignedLong(parseUnsignedLong(value)); } }; - IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup, name(), format); + return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup, blContext.getIgnoredSourceFormat()); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { From 07ddeaeb7c28e57beedc3f14cfb9be93b023321b Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sun, 12 Oct 2025 12:19:55 +0000 Subject: [PATCH 05/14] [CI] Auto commit changes from spotless --- .../index/mapper/extras/MatchOnlyTextFieldMapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 75b9cf309d8d7..1d60d1c2ecc99 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -48,7 +48,6 @@ import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; From d68d8731d17a87a48569207813a95625fc7dcc72 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Sun, 12 Oct 2025 15:27:41 +0200 Subject: [PATCH 06/14] iter4 --- .../extras/MatchOnlyTextFieldMapper.java | 6 +- .../mapper/extras/RankFeatureFieldMapper.java | 7 +- .../mapper/extras/ScaledFloatFieldMapper.java | 18 ++-- .../elasticsearch/index/IndexSettings.java | 8 ++ .../mapper/AbstractGeometryFieldMapper.java | 10 +-- .../index/mapper/ArraySourceValueFetcher.java | 10 ++- .../index/mapper/BlockSourceReader.java | 38 +++++---- .../index/mapper/BooleanFieldMapper.java | 16 ++-- .../index/mapper/DateFieldMapper.java | 12 +-- .../index/mapper/GeoPointFieldMapper.java | 2 +- .../index/mapper/IpFieldMapper.java | 9 +- .../index/mapper/KeywordFieldMapper.java | 17 ++-- .../index/mapper/MappedFieldType.java | 8 -- .../index/mapper/NumberFieldMapper.java | 83 +++++++------------ .../index/mapper/SourceValueFetcher.java | 26 ++++-- .../index/mapper/TextFieldMapper.java | 7 +- .../flattened/FlattenedFieldMapper.java | 10 ++- .../vectors/DenseVectorFieldMapper.java | 12 +-- .../index/termvectors/TermVectorsService.java | 4 +- .../index/mapper/BlockSourceReaderTests.java | 39 ++++++--- .../mapper/SemanticTextFieldMapper.java | 8 +- .../unsignedlong/UnsignedLongFieldMapper.java | 17 ++-- 22 files changed, 206 insertions(+), 161 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 1d60d1c2ecc99..628e84173496f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -605,10 +605,10 @@ protected String delegatingTo() { } // fallback to _source (synthetic or not) - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()), blContext.indexSettings()); // MatchOnlyText never has norms, so we have to use the field names field BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup, blContext.getIgnoredSourceFormat()); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup); } @Override @@ -636,7 +636,7 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name()), fieldDataContext.indexSettings()), fieldDataContext.lookupSupplier().get(), TextDocValuesField::new ); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index 8e4eda87abfb8..7a1a03cc537d7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -148,11 +148,12 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher(context); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(SearchExecutionContext context) { + Set sourcePaths = context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(); + return new SourceValueFetcher(sourcePaths, nullValue, context.getIndexSettings().getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { return objectToFloat(value); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 0e2dc35a73c22..3d97bba26398d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; @@ -40,6 +41,7 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -263,6 +265,7 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) { ) ); + @SuppressWarnings("checkstyle:LineLength") public static final class ScaledFloatFieldType extends SimpleMappedFieldType { private final double scalingFactor; @@ -397,12 +400,12 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name())); + ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup, blContext.getIgnoredSourceFormat()); + return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { @@ -489,7 +492,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedDoubleIndexFieldData.Builder( name(), valuesSourceType, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, ScaledFloatDocValuesField::new ); @@ -503,11 +506,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected Double parseSourceValue(Object value) { double doubleValue; diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index b396e1ca206e3..18e2194ee6d42 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -1813,6 +1813,14 @@ public SourceFieldMapper.Mode getIndexMappingSourceMode() { return indexMappingSourceMode; } + public IgnoredSourceFieldMapper.IgnoredSourceFormat getIgnoredSourceFormat() { + if (getIndexMappingSourceMode() == SourceFieldMapper.Mode.SYNTHETIC) { + return IgnoredSourceFieldMapper.ignoredSourceFormat(getIndexVersionCreated()); + } else { + return IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } + } + /** * @return Whether recovery source should be enabled if needed. * Note that this is a node setting, and this setting is not sourced from index settings. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 4c88cbf09ec20..75c7c434930ec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.geo.GeometryFormatterFactory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -172,9 +173,9 @@ protected Object parseSourceValue(Object value) { }; } - public ArraySourceValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format) { + public ArraySourceValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format, IndexSettings indexSettings) { Function, List> formatter = getFormatter(format != null ? format : GeometryFormatterFactory.GEOJSON); - return new ArraySourceValueFetcher(sourcePaths, nullValueAsSource(nullValue)) { + return new ArraySourceValueFetcher(sourcePaths, nullValueAsSource(nullValue), indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { final List values = new ArrayList<>(); @@ -185,10 +186,9 @@ protected Object parseSourceValue(Object value) { } protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { - var fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB); + var fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB, blContext.indexSettings()); // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) - var format = blContext.getIgnoredSourceFormat(); - return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), format); + return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); } protected abstract Object nullValueAsSource(T nullValue); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index 00e4fc3905ea0..c88764f6a9d4f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -10,6 +10,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; @@ -30,6 +31,7 @@ public abstract class ArraySourceValueFetcher implements ValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; + private final IgnoredSourceFormat ignoredSourceFormat; public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) { this(fieldName, context, null); @@ -43,15 +45,17 @@ public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { this.sourcePaths = context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(); this.nullValue = nullValue; + this.ignoredSourceFormat = context.getIndexSettings().getIgnoredSourceFormat(); } /** * @param sourcePaths The paths to pull source values from * @param nullValue An optional substitute value if the _source value is `null` */ - public ArraySourceValueFetcher(Set sourcePaths, Object nullValue) { + public ArraySourceValueFetcher(Set sourcePaths, Object nullValue, IgnoredSourceFormat ignoredSourceFormat) { this.sourcePaths = sourcePaths; this.nullValue = nullValue; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -83,6 +87,10 @@ public Set getSourcePaths() { return sourcePaths; } + public IgnoredSourceFormat getIgnoredSourceFormat() { + return ignoredSourceFormat; + } + /** * Given a value that has been extracted from a document's source, parse it into a standard * format. This parsing logic should closely mirror the value parsing in diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 32bf3962cb3fe..e7f864815ee45 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -99,17 +99,19 @@ private abstract static class SourceBlockLoader implements BlockLoader { private final Set sourcePaths; private final IgnoredSourceFormat ignoredSourceFormat; - private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat ignoredSourceFormat) { + private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { this.fetcher = fetcher; this.lookup = lookup; if (fetcher instanceof SourceValueFetcher sourceValueFetcher) { this.sourcePaths = sourceValueFetcher.getSourcePaths(); + this.ignoredSourceFormat = sourceValueFetcher.getIgnoredSourceFormat(); } else if (fetcher instanceof ArraySourceValueFetcher arraySourceValueFetcher) { this.sourcePaths = arraySourceValueFetcher.getSourcePaths(); + this.ignoredSourceFormat = arraySourceValueFetcher.getIgnoredSourceFormat(); } else { + // TODO: maybe introduce a common interface for SourceValueFetcher and ArraySourceValueFetcher? throw new IllegalStateException("unexpected fetcher type " + fetcher); } - this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -155,8 +157,8 @@ public final String toString() { * Load {@code boolean}s from {@code _source}. */ public static class BooleansBlockLoader extends SourceBlockLoader { - public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { - super(fetcher, lookup, format); + public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + super(fetcher, lookup); } @Override @@ -195,8 +197,8 @@ public String toString() { * Load {@link BytesRef}s from {@code _source}. */ public static class BytesRefsBlockLoader extends SourceBlockLoader { - public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { - super(fetcher, lookup, format); + public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + super(fetcher, lookup); } @Override @@ -216,8 +218,8 @@ protected String name() { } public static class GeometriesBlockLoader extends SourceBlockLoader { - public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { - super(fetcher, lookup, format); + public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + super(fetcher, lookup); } @Override @@ -279,8 +281,8 @@ public String toString() { * Load {@code double}s from {@code _source}. */ public static class DoublesBlockLoader extends SourceBlockLoader { - public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { - super(fetcher, lookup, format); + public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + super(fetcher, lookup); } @Override @@ -321,8 +323,8 @@ public String toString() { public static class DenseVectorBlockLoader extends SourceBlockLoader { private final int dimensions; - public DenseVectorBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, int dimensions, IgnoredSourceFormat format) { - super(fetcher, lookup, format); + public DenseVectorBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, int dimensions) { + super(fetcher, lookup); this.dimensions = dimensions; } @@ -362,8 +364,8 @@ public String toString() { * Load {@code int}s from {@code _source}. */ public static class IntsBlockLoader extends SourceBlockLoader { - public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { - super(fetcher, lookup, format); + public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + super(fetcher, lookup); } @Override @@ -402,8 +404,8 @@ public String toString() { * Load {@code long}s from {@code _source}. */ public static class LongsBlockLoader extends SourceBlockLoader { - public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, IgnoredSourceFormat format) { - super(fetcher, lookup, format); + public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + super(fetcher, lookup); } @Override @@ -442,8 +444,8 @@ public String toString() { * Load {@code ip}s from {@code _source}. */ public static class IpsBlockLoader extends SourceBlockLoader { - public IpsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, String name, IgnoredSourceFormat format) { - super(fetcher, lookup, format); + public IpsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + super(fetcher, lookup); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index d9b7455febe0c..4e37a77227042 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -281,11 +282,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected Boolean parseSourceValue(Object value) { if (value instanceof Boolean) { @@ -364,11 +368,11 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); + ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); BlockSourceReader.LeafIteratorLookup lookup = indexType.hasTerms() || isStored() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup, blContext.getIgnoredSourceFormat()); + return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { @@ -431,7 +435,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedBooleanIndexFieldData.Builder( name(), CoreValuesSourceType.BOOLEAN, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, BooleanDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 196647a821b42..9d7dfecfb925c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -662,8 +662,8 @@ public String parseSourceValue(Object value) { } // returns a Long to support source fallback which emulates numeric doc values for dates - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override public Long parseSourceValue(Object value) { String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString(); @@ -998,8 +998,10 @@ public Builder builder(BlockFactory factory, int expectedCount) { BlockSourceReader.LeafIteratorLookup lookup = isStored() || indexType.hasPoints() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - var format = blContext.getIgnoredSourceFormat(); - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, format); + return new BlockSourceReader.LongsBlockLoader( + sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()), + lookup + ); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { @@ -1067,7 +1069,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedNumericIndexFieldData.Builder( name(), resolution.numericType().getValuesSourceType(), - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, resolution.getDefaultToScriptFieldFactory() ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index d9865202a0592..3a297bb43ab37 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -494,7 +494,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherMultiGeoPointIndexFieldData.Builder( name(), valuesSourceType, - valueFetcher(sourcePaths, null, null), + valueFetcher(sourcePaths, null, null, fieldDataContext.indexSettings()), searchLookup, GeoPointDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 862f9ed6e2f45..644ab73a0217b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; -import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.IpFieldScript; import org.elasticsearch.script.Script; @@ -54,7 +53,6 @@ import java.util.Collections; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.function.BiFunction; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; @@ -482,12 +480,11 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (isSyntheticSource && blContext.parentField(name()) == null) { return blockLoaderFromFallbackSyntheticSource(blContext); } - IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); // see #indexValue BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && hasPoints ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, name(), format); + return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext), lookup); } private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) { @@ -504,8 +501,8 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(BlockLoaderContext blContext) { + return new SourceValueFetcher(blContext.sourcePaths(name()), nullValue, blContext.indexSettings().getIgnoredSourceFormat()) { @Override public InetAddress parseSourceValue(Object value) { return parse(value); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 0aa193168310b..b337cdd41dcb3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -820,9 +821,8 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); - var format = blContext.getIgnoredSourceFormat(); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext), format); + SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext)); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { @@ -909,7 +909,7 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), fieldDataContext.lookupSupplier().get(), KeywordDocValuesField::new ); @@ -931,11 +931,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index bcc9bc78fad3a..d5b322126cfe9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -704,14 +704,6 @@ public interface BlockLoaderContext { */ FieldNamesFieldMapper.FieldNamesFieldType fieldNames(); - default IgnoredSourceFieldMapper.IgnoredSourceFormat getIgnoredSourceFormat() { - IndexSettings indexSettings = indexSettings(); - if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) { - return IgnoredSourceFieldMapper.ignoredSourceFormat(indexSettings.getIndexVersionCreated()); - } else { - return IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } - } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 8d5cb4bfbb743..27f4ace5cc739 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -45,7 +46,6 @@ import org.elasticsearch.index.fielddata.SourceValueFetcherSortedNumericIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedDoublesIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; -import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.mapper.TimeSeriesParams.MetricType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.DoubleFieldScript; @@ -495,12 +495,8 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource( - SourceValueFetcher sourceValueFetcher, - BlockSourceReader.LeafIteratorLookup lookup, - IgnoredSourceFormat format - ) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, format); + BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); } @Override @@ -693,12 +689,8 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource( - SourceValueFetcher sourceValueFetcher, - BlockSourceReader.LeafIteratorLookup lookup, - IgnoredSourceFormat format - ) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, format); + BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); } @Override @@ -857,12 +849,8 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource( - SourceValueFetcher sourceValueFetcher, - BlockSourceReader.LeafIteratorLookup lookup, - IgnoredSourceFormat format - ) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, format); + BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); } @Override @@ -989,12 +977,8 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource( - SourceValueFetcher sourceValueFetcher, - BlockSourceReader.LeafIteratorLookup lookup, - IgnoredSourceFormat format - ) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, format); + BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); } @Override @@ -1121,12 +1105,8 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource( - SourceValueFetcher sourceValueFetcher, - BlockSourceReader.LeafIteratorLookup lookup, - IgnoredSourceFormat format - ) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, format); + BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); } @Override @@ -1327,12 +1307,8 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource( - SourceValueFetcher sourceValueFetcher, - BlockSourceReader.LeafIteratorLookup lookup, - IgnoredSourceFormat format - ) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, format); + BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); } @Override @@ -1493,12 +1469,8 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource( - SourceValueFetcher sourceValueFetcher, - BlockSourceReader.LeafIteratorLookup lookup, - IgnoredSourceFormat format - ) { - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup, format); + BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup); } @Override @@ -1808,11 +1780,7 @@ public void writeValue(XContentBuilder b, long value) throws IOException { abstract BlockLoader blockLoaderFromDocValues(String fieldName); - abstract BlockLoader blockLoaderFromSource( - SourceValueFetcher sourceValueFetcher, - BlockSourceReader.LeafIteratorLookup lookup, - IgnoredSourceFormat format - ); + abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup); abstract BlockLoader blockLoaderFromFallbackSyntheticSource( String fieldName, @@ -2088,8 +2056,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { // We only write the field names field if there aren't doc values or norms ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); - return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup, format); + return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()), lookup); } @Override @@ -2111,7 +2078,12 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation == FielddataOperation.SCRIPT) { SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); - return type.getValueFetcherFieldDataBuilder(name(), valuesSourceType, searchLookup, sourceValueFetcher(sourcePaths)); + return type.getValueFetcherFieldDataBuilder( + name(), + valuesSourceType, + searchLookup, + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()) + ); } throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); @@ -2133,11 +2105,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index 6a93d44b3168b..4a0da8ed3faf4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -10,6 +10,8 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; @@ -31,6 +33,7 @@ public abstract class SourceValueFetcher implements ValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; + private final IgnoredSourceFormat ignoredSourceFormat; public SourceValueFetcher(String fieldName, SearchExecutionContext context) { this(fieldName, context, null); @@ -41,16 +44,22 @@ public SourceValueFetcher(String fieldName, SearchExecutionContext context) { * @param nullValue An optional substitute value if the _source value is 'null'. */ public SourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { - this(context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(), nullValue); + this( + context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(), + nullValue, + context.getIndexSettings().getIgnoredSourceFormat() + ); } /** - * @param sourcePaths The paths to pull source values from - * @param nullValue An optional substitute value if the _source value is `null` + * @param sourcePaths The paths to pull source values from + * @param nullValue An optional substitute value if the _source value is `null` + * @param ignoredSourceFormat */ - public SourceValueFetcher(Set sourcePaths, Object nullValue) { + public SourceValueFetcher(Set sourcePaths, Object nullValue, IgnoredSourceFormat ignoredSourceFormat) { this.sourcePaths = sourcePaths; this.nullValue = nullValue; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -105,6 +114,10 @@ public Set getSourcePaths() { return sourcePaths; } + public IgnoredSourceFormat getIgnoredSourceFormat() { + return ignoredSourceFormat; + } + /** * Given a value that has been extracted from a document's source, parse it into a standard * format. This parsing logic should closely mirror the value parsing in @@ -144,10 +157,11 @@ protected Object parseSourceValue(Object value) { /** * Creates a {@link SourceValueFetcher} that converts source values to Strings + * * @param sourcePaths the paths to fetch values from in the source */ - public static SourceValueFetcher toString(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + public static SourceValueFetcher toString(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, null, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { return value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index f67cf6d46d921..6c5231fdfb9a1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1117,9 +1117,8 @@ protected String delegatingTo() { return fallbackSyntheticSourceBlockLoader(blContext); } // otherwise, load values from _source (synthetic or not) - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); - var format = blContext.getIgnoredSourceFormat(); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), format); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()), blContext.indexSettings()); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext)); } FallbackSyntheticSourceBlockLoader fallbackSyntheticSourceBlockLoader(BlockLoaderContext blContext) { @@ -1247,7 +1246,7 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name()), fieldDataContext.indexSettings()), fieldDataContext.lookupSupplier().get(), TextDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 69a520d2bf518..de8c9020c2f3e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -753,15 +754,18 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } public IgnoreAbove ignoreAbove() { return ignoreAbove; } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, null, indexSettings.getIgnoredSourceFormat()) { @Override @SuppressWarnings("unchecked") protected Object parseSourceValue(Object value) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index a503c0f1cee71..84d32874a17b3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -63,7 +63,6 @@ import org.elasticsearch.index.mapper.BlockSourceReader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -2692,13 +2691,16 @@ public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) { return new BlockDocValuesReader.DenseVectorFromBinaryBlockLoader(name(), dims, indexVersionCreated, element.elementType()); } - IgnoredSourceFieldMapper.IgnoredSourceFormat format = blContext.getIgnoredSourceFormat(); BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.DenseVectorBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, dims, format); + return new BlockSourceReader.DenseVectorBlockLoader( + sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()), + lookup, + dims + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, null, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index f494fdd0acfab..6eb4c10c68b11 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.DocumentParser; @@ -279,11 +280,12 @@ private static Fields generateTermVectors( } } if (source != null) { + IndexSettings indexSettings = indexShard.indexSettings(); MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); Source s = Source.fromMap(source, XContentType.JSON); for (String field : fields) { if (values.containsKey(field) == false) { - SourceValueFetcher valueFetcher = SourceValueFetcher.toString(mappingLookup.sourcePaths(field)); + SourceValueFetcher valueFetcher = SourceValueFetcher.toString(mappingLookup.sourcePaths(field), indexSettings); List ignoredValues = new ArrayList<>(); List v = valueFetcher.fetchValues(s, -1, ignoredValues); if (v.isEmpty() == false) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index f87c732208a9f..a785ae13a4a06 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -11,9 +11,15 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -49,19 +55,32 @@ public void testEmptyArray() throws IOException { } private void loadBlock(LeafReaderContext ctx, Consumer test) throws IOException { - boolean synteticSource = randomBoolean(); - IgnoredSourceFieldMapper.IgnoredSourceFormat format; - if (synteticSource) { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE; - } else { - format = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; - } - ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field")); + boolean syntheticSource = randomBoolean(); + IndexMetadata indexMetadata = IndexMetadata.builder("index") + .settings( + Settings.builder() + .put( + ESTestCase.indexSettings(IndexVersion.current(), 1, 1) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), syntheticSource ? "synthetic" : "stored") + .build() + ) + ) + .build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field"), indexSettings); BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field"); - BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup, format); + BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup); assertThat(loader.columnAtATimeReader(ctx), nullValue()); BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx); - assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.withSourcePaths(format, Set.of("field")))); + assertThat( + loader.rowStrideStoredFieldSpec(), + equalTo( + StoredFieldsSpec.withSourcePaths( + syntheticSource ? IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE : IgnoredSourceFormat.NO_IGNORED_SOURCE, + Set.of("field") + ) + ) + ); BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null), loader.rowStrideStoredFieldSpec().requiresSource() ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) : null diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 052b163e1f2f6..b1f37298c803f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -1053,9 +1053,11 @@ private String generateInvalidQueryInferenceResultsMessage(StringBuilder baseMes @Override public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { String name = useLegacyFormat ? name().concat(".text") : name(); - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name)); - var format = blContext.getIgnoredSourceFormat(); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), format); + SourceValueFetcher fetcher = SourceValueFetcher.toString( + blContext.sourcePaths(name), + blContext.indexSettings() + ); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); } private class SemanticTextFieldValueFetcher implements ValueFetcher { diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 4de422e51b5a1..6c6842f55da3b 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -398,7 +399,8 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - ValueFetcher valueFetcher = new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted) { + var ignoredSourceFormat = blContext.indexSettings().getIgnoredSourceFormat(); + ValueFetcher valueFetcher = new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted, ignoredSourceFormat) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { @@ -415,7 +417,7 @@ protected Object parseSourceValue(Object value) { // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup, blContext.getIgnoredSourceFormat()); + return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { @@ -511,7 +513,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedUnsignedLongIndexFieldData.Builder( name(), valuesSourceType, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, UnsignedLongDocValuesField::new ); @@ -525,11 +527,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValueFormatted) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValueFormatted, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { From f055a87711780cf50e8195863c1a6400beb8fa9b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Sun, 12 Oct 2025 15:33:18 +0200 Subject: [PATCH 07/14] cleanup --- .../elasticsearch/index/mapper/AbstractGeometryFieldMapper.java | 2 +- .../java/org/elasticsearch/index/mapper/MappedFieldType.java | 1 - .../java/org/elasticsearch/index/mapper/SourceValueFetcher.java | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 75c7c434930ec..2d48bf82db1ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -173,7 +173,7 @@ protected Object parseSourceValue(Object value) { }; } - public ArraySourceValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format, IndexSettings indexSettings) { + public ValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format, IndexSettings indexSettings) { Function, List> formatter = getFormatter(format != null ? format : GeometryFormatterFactory.GEOJSON); return new ArraySourceValueFetcher(sourcePaths, nullValueAsSource(nullValue), indexSettings.getIgnoredSourceFormat()) { @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index d5b322126cfe9..771a54afa267a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -703,7 +703,6 @@ public interface BlockLoaderContext { * The {@code _field_names} field mapper, mostly used to check if it is enabled. */ FieldNamesFieldMapper.FieldNamesFieldType fieldNames(); - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index 4a0da8ed3faf4..125238f9ea127 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -157,7 +157,6 @@ protected Object parseSourceValue(Object value) { /** * Creates a {@link SourceValueFetcher} that converts source values to Strings - * * @param sourcePaths the paths to fetch values from in the source */ public static SourceValueFetcher toString(Set sourcePaths, IndexSettings indexSettings) { From fa845b630c27676ba3e9c0681c6efa7941cbf2f6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sun, 12 Oct 2025 13:40:06 +0000 Subject: [PATCH 08/14] [CI] Auto commit changes from spotless --- .../index/mapper/extras/ScaledFloatFieldMapper.java | 1 - .../xpack/inference/mapper/SemanticTextFieldMapper.java | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 3d97bba26398d..3645f51db96ed 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -41,7 +41,6 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; -import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index b1f37298c803f..6e16bc4a43512 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -1053,10 +1053,7 @@ private String generateInvalidQueryInferenceResultsMessage(StringBuilder baseMes @Override public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { String name = useLegacyFormat ? name().concat(".text") : name(); - SourceValueFetcher fetcher = SourceValueFetcher.toString( - blContext.sourcePaths(name), - blContext.indexSettings() - ); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name), blContext.indexSettings()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); } From f435afddae09d1003979b417b232f78ab70b8b9e Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Sun, 12 Oct 2025 15:41:27 +0200 Subject: [PATCH 09/14] iter5 --- .../mapper/extras/ScaledFloatFieldMapper.java | 2 +- .../mapper/AbstractGeometryFieldMapper.java | 2 +- .../index/mapper/ArraySourceValueFetcher.java | 12 ++----- .../index/mapper/BlockSourceReader.java | 34 ++++++------------- .../index/mapper/BooleanFieldMapper.java | 2 +- .../index/mapper/SourceBasedValueFetcher.java | 15 ++++++++ .../index/mapper/SourceValueFetcher.java | 12 ++----- .../index/mapper/BlockSourceReaderTests.java | 2 +- .../unsignedlong/UnsignedLongFieldMapper.java | 2 +- 9 files changed, 34 insertions(+), 49 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/SourceBasedValueFetcher.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 3645f51db96ed..0d7f412d7e8f5 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -399,7 +399,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); + var valueFetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 2d48bf82db1ba..a91eb5fa229c2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -173,7 +173,7 @@ protected Object parseSourceValue(Object value) { }; } - public ValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format, IndexSettings indexSettings) { + public SourceBasedValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format, IndexSettings indexSettings) { Function, List> formatter = getFormatter(format != null ? format : GeometryFormatterFactory.GEOJSON); return new ArraySourceValueFetcher(sourcePaths, nullValueAsSource(nullValue), indexSettings.getIgnoredSourceFormat()) { @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index c88764f6a9d4f..66abde002159f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -28,7 +28,7 @@ * array values in parsing. Field types should use this class if their corresponding * mapper returns true for {@link FieldMapper#parsesArrayValue()}. */ -public abstract class ArraySourceValueFetcher implements ValueFetcher { +public abstract class ArraySourceValueFetcher implements SourceBasedValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; private final IgnoredSourceFormat ignoredSourceFormat; @@ -80,15 +80,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; - } - - public Set getSourcePaths() { - return sourcePaths; - } - - public IgnoredSourceFormat getIgnoredSourceFormat() { - return ignoredSourceFormat; + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, sourcePaths); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index e7f864815ee45..0d24bc9654673 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -18,7 +18,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.UnicodeUtil; -import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -26,7 +25,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.Set; /** * Loads values from {@code _source}. This whole process is very slow and cast-tastic, @@ -96,22 +94,10 @@ public interface LeafIteratorLookup { private abstract static class SourceBlockLoader implements BlockLoader { protected final ValueFetcher fetcher; private final LeafIteratorLookup lookup; - private final Set sourcePaths; - private final IgnoredSourceFormat ignoredSourceFormat; - private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + private SourceBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { this.fetcher = fetcher; this.lookup = lookup; - if (fetcher instanceof SourceValueFetcher sourceValueFetcher) { - this.sourcePaths = sourceValueFetcher.getSourcePaths(); - this.ignoredSourceFormat = sourceValueFetcher.getIgnoredSourceFormat(); - } else if (fetcher instanceof ArraySourceValueFetcher arraySourceValueFetcher) { - this.sourcePaths = arraySourceValueFetcher.getSourcePaths(); - this.ignoredSourceFormat = arraySourceValueFetcher.getIgnoredSourceFormat(); - } else { - // TODO: maybe introduce a common interface for SourceValueFetcher and ArraySourceValueFetcher? - throw new IllegalStateException("unexpected fetcher type " + fetcher); - } } @Override @@ -121,7 +107,7 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) @Override public final StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, sourcePaths); + return fetcher.storedFieldsSpec(); } @Override @@ -157,7 +143,7 @@ public final String toString() { * Load {@code boolean}s from {@code _source}. */ public static class BooleansBlockLoader extends SourceBlockLoader { - public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + public BooleansBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -197,7 +183,7 @@ public String toString() { * Load {@link BytesRef}s from {@code _source}. */ public static class BytesRefsBlockLoader extends SourceBlockLoader { - public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + public BytesRefsBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -218,7 +204,7 @@ protected String name() { } public static class GeometriesBlockLoader extends SourceBlockLoader { - public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + public GeometriesBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -281,7 +267,7 @@ public String toString() { * Load {@code double}s from {@code _source}. */ public static class DoublesBlockLoader extends SourceBlockLoader { - public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + public DoublesBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -323,7 +309,7 @@ public String toString() { public static class DenseVectorBlockLoader extends SourceBlockLoader { private final int dimensions; - public DenseVectorBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, int dimensions) { + public DenseVectorBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup, int dimensions) { super(fetcher, lookup); this.dimensions = dimensions; } @@ -364,7 +350,7 @@ public String toString() { * Load {@code int}s from {@code _source}. */ public static class IntsBlockLoader extends SourceBlockLoader { - public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + public IntsBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -404,7 +390,7 @@ public String toString() { * Load {@code long}s from {@code _source}. */ public static class LongsBlockLoader extends SourceBlockLoader { - public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + public LongsBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -444,7 +430,7 @@ public String toString() { * Load {@code ip}s from {@code _source}. */ public static class IpsBlockLoader extends SourceBlockLoader { - public IpsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + public IpsBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 4e37a77227042..d118683ba5942 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -368,7 +368,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); + var fetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); BlockSourceReader.LeafIteratorLookup lookup = indexType.hasTerms() || isStored() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceBasedValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceBasedValueFetcher.java new file mode 100644 index 0000000000000..5807b5cdfe996 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceBasedValueFetcher.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +/** + * Marker interface for {@link ValueFetcher} implementations that fetch values from the source. + */ +public interface SourceBasedValueFetcher extends ValueFetcher {} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index 125238f9ea127..5a92ba8120c11 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -30,7 +30,7 @@ * * Field types that handle arrays directly should instead use {@link ArraySourceValueFetcher}. */ -public abstract class SourceValueFetcher implements ValueFetcher { +public abstract class SourceValueFetcher implements SourceBasedValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; private final IgnoredSourceFormat ignoredSourceFormat; @@ -107,15 +107,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; - } - - public Set getSourcePaths() { - return sourcePaths; - } - - public IgnoredSourceFormat getIgnoredSourceFormat() { - return ignoredSourceFormat; + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, sourcePaths); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index a785ae13a4a06..fa0f4c0bd4c94 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -67,7 +67,7 @@ private void loadBlock(LeafReaderContext ctx, Consumer test) throws I ) .build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field"), indexSettings); + var valueFetcher = SourceValueFetcher.toString(Set.of("field"), indexSettings); BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field"); BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup); assertThat(loader.columnAtATimeReader(ctx), nullValue()); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 6c6842f55da3b..b11eb62b9db7d 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -400,7 +400,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { } var ignoredSourceFormat = blContext.indexSettings().getIgnoredSourceFormat(); - ValueFetcher valueFetcher = new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted, ignoredSourceFormat) { + var valueFetcher = new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted, ignoredSourceFormat) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { From d7455368076010f01a5256e321e507b12114a83d Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Sun, 12 Oct 2025 16:00:04 +0200 Subject: [PATCH 10/14] fixed unit test --- .../main/java/org/elasticsearch/index/mapper/MapperTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 98d6576f59a44..a2967973fb102 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -859,6 +859,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { From f33dabd1d71271717551f5955986c133867cb823 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Sun, 12 Oct 2025 20:33:24 +0200 Subject: [PATCH 11/14] fix unit tests --- .../java/org/elasticsearch/index/mapper/FieldTypeTestCase.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index fae33f51740f1..b3410f590033a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.Query; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.FieldLookup; @@ -24,6 +25,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; @@ -63,6 +65,7 @@ public static List fetchSourceValue(MappedFieldType fieldType, Object sourceV public static List fetchSourceValue(MappedFieldType fieldType, Object sourceValue, String format) throws IOException { String field = fieldType.name(); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY)); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); From a177f8a55cce29ad3d3b3756b518fd9050cf9980 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 13 Oct 2025 09:42:51 +0200 Subject: [PATCH 12/14] fix mocking in unit tests --- .../index/mapper/extras/MatchOnlyTextFieldTypeTests.java | 3 +++ .../elasticsearch/index/mapper/TextFieldMapperTests.java | 3 ++- .../elasticsearch/index/mapper/TextFieldTypeTests.java | 9 +++++++-- .../mapper/flattened/KeyedFlattenedFieldTypeTests.java | 3 +++ .../mapper/vectors/DenseVectorFieldMapperTests.java | 1 + .../search/fetch/subphase/FieldFetcherTests.java | 9 +++++++-- .../elasticsearch/index/mapper/FieldTypeTestCase.java | 1 + 7 files changed, 24 insertions(+), 5 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java index 74f1f9a16a6d6..88f14aa5ecc90 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java @@ -59,6 +59,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class MatchOnlyTextFieldTypeTests extends FieldTypeTestCase { @@ -315,6 +316,7 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet // when MappedFieldType.BlockLoaderContext blContext = mock(MappedFieldType.BlockLoaderContext.class); doReturn(FieldNamesFieldMapper.FieldNamesFieldType.get(false)).when(blContext).fieldNames(); + when(blContext.indexSettings()).thenReturn(indexSettings); BlockLoader blockLoader = ft.blockLoader(blContext); // then @@ -362,6 +364,7 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet // when MappedFieldType.BlockLoaderContext blContext = mock(MappedFieldType.BlockLoaderContext.class); + when(blContext.indexSettings()).thenReturn(indexSettings); doReturn(FieldNamesFieldMapper.FieldNamesFieldType.get(false)).when(blContext).fieldNames(); BlockLoader blockLoader = ft.blockLoader(blContext); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 332c727ffadde..cee149071134f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1446,8 +1446,9 @@ public void testEmpty() throws Exception { throw new IllegalArgumentException("Can't load source in scripts in synthetic mode"); } : SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()); SearchLookup searchLookup = new SearchLookup(null, null, sourceProvider); + var indexSettings = mapperService.getIndexSettings(); IndexFieldData sfd = ft.fielddataBuilder( - new FieldDataContext("", null, () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) + new FieldDataContext("", indexSettings, () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) ).build(null, null); LeafFieldData lfd = sfd.load(getOnlyLeafReader(searcher.getIndexReader()).getContext()); TextDocValuesField scriptDV = (TextDocValuesField) lfd.getScriptFieldFactory("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index fcab101da9baa..4787b444d732f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -55,6 +55,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TextFieldTypeTests extends FieldTypeTestCase { @@ -374,7 +375,9 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet ); // when - BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + var context = mock(MappedFieldType.BlockLoaderContext.class); + when(context.indexSettings()).thenReturn(indexSettings); + BlockLoader blockLoader = ft.blockLoader(context); // then // verify that we don't delegate anything @@ -423,7 +426,9 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet ); // when - BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + var context = mock(MappedFieldType.BlockLoaderContext.class); + when(context.indexSettings()).thenReturn(indexSettings); + BlockLoader blockLoader = ft.blockLoader(context); // then // verify that we don't delegate anything diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java index c8d7ad8127b55..dadeae3cc654c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java @@ -18,12 +18,14 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.search.AutomatonQueries; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; @@ -186,6 +188,7 @@ public void testFetchSourceValue() throws IOException { Map sourceValue = Map.of("key", "value"); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY)); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath("field.key")).thenReturn(Set.of("field.key")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 5932180ac3c03..8d2787ab76dc9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -2403,6 +2403,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; SourceToParse source = source(b -> b.field(ft.name(), value)); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 5f26b046e0c8e..5311c1664c9c3 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import static java.util.Collections.emptyMap; @@ -1585,9 +1586,13 @@ public void testFetchMetadataFieldWithSourceDisabled() throws IOException { } public void testStoredFieldsSpec() throws IOException { + var mapperService = createMapperService(); List fields = List.of(new FieldAndFormat("field", null)); - FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(createMapperService()), fields); - assertEquals(StoredFieldsSpec.NEEDS_SOURCE, fieldFetcher.storedFieldsSpec()); + FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(mapperService), fields); + assertEquals( + StoredFieldsSpec.withSourcePaths(mapperService.getIndexSettings().getIgnoredSourceFormat(), Set.of("field")), + fieldFetcher.storedFieldsSpec() + ); } private List fieldAndFormatList(String name, String format, boolean includeUnmapped) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index b3410f590033a..757ff99e0d508 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -77,6 +77,7 @@ public static List fetchSourceValue(MappedFieldType fieldType, Object sourceV public static List fetchSourceValues(MappedFieldType fieldType, Object... values) throws IOException { String field = fieldType.name(); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY)); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); From 77415664de3ca0e225a57325201eccb0dec52678 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 13 Oct 2025 11:50:59 +0200 Subject: [PATCH 13/14] fix mocking in more unit tests --- .../index/mapper/murmur3/Murmur3FieldMapperTests.java | 1 + .../xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 039f8c049e0c5..d04f2067d8aed 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -99,6 +99,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java index ad29a191aace3..5834aca5fa0a5 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java @@ -374,6 +374,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; SourceToParse source = source(b -> b.field(ft.name(), value)); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { From ae0b28d70f1d3af2a0f88708d511edd5e84a28f1 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 13 Oct 2025 16:41:54 +0200 Subject: [PATCH 14/14] Remove SourceBasedValueFetcher --- .../mapper/AbstractGeometryFieldMapper.java | 2 +- .../index/mapper/ArraySourceValueFetcher.java | 2 +- .../index/mapper/BlockSourceReader.java | 18 +++++++++--------- .../index/mapper/SourceBasedValueFetcher.java | 15 --------------- .../index/mapper/SourceValueFetcher.java | 2 +- 5 files changed, 12 insertions(+), 27 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/mapper/SourceBasedValueFetcher.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index a91eb5fa229c2..2d48bf82db1ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -173,7 +173,7 @@ protected Object parseSourceValue(Object value) { }; } - public SourceBasedValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format, IndexSettings indexSettings) { + public ValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format, IndexSettings indexSettings) { Function, List> formatter = getFormatter(format != null ? format : GeometryFormatterFactory.GEOJSON); return new ArraySourceValueFetcher(sourcePaths, nullValueAsSource(nullValue), indexSettings.getIgnoredSourceFormat()) { @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index 66abde002159f..c73b4fefa8a4a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -28,7 +28,7 @@ * array values in parsing. Field types should use this class if their corresponding * mapper returns true for {@link FieldMapper#parsesArrayValue()}. */ -public abstract class ArraySourceValueFetcher implements SourceBasedValueFetcher { +public abstract class ArraySourceValueFetcher implements ValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; private final IgnoredSourceFormat ignoredSourceFormat; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 0d24bc9654673..858b054012fe2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -95,7 +95,7 @@ private abstract static class SourceBlockLoader implements BlockLoader { protected final ValueFetcher fetcher; private final LeafIteratorLookup lookup; - private SourceBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { + private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { this.fetcher = fetcher; this.lookup = lookup; } @@ -143,7 +143,7 @@ public final String toString() { * Load {@code boolean}s from {@code _source}. */ public static class BooleansBlockLoader extends SourceBlockLoader { - public BooleansBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { + public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -183,7 +183,7 @@ public String toString() { * Load {@link BytesRef}s from {@code _source}. */ public static class BytesRefsBlockLoader extends SourceBlockLoader { - public BytesRefsBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { + public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -204,7 +204,7 @@ protected String name() { } public static class GeometriesBlockLoader extends SourceBlockLoader { - public GeometriesBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { + public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -267,7 +267,7 @@ public String toString() { * Load {@code double}s from {@code _source}. */ public static class DoublesBlockLoader extends SourceBlockLoader { - public DoublesBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { + public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -309,7 +309,7 @@ public String toString() { public static class DenseVectorBlockLoader extends SourceBlockLoader { private final int dimensions; - public DenseVectorBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup, int dimensions) { + public DenseVectorBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, int dimensions) { super(fetcher, lookup); this.dimensions = dimensions; } @@ -350,7 +350,7 @@ public String toString() { * Load {@code int}s from {@code _source}. */ public static class IntsBlockLoader extends SourceBlockLoader { - public IntsBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { + public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -390,7 +390,7 @@ public String toString() { * Load {@code long}s from {@code _source}. */ public static class LongsBlockLoader extends SourceBlockLoader { - public LongsBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { + public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } @@ -430,7 +430,7 @@ public String toString() { * Load {@code ip}s from {@code _source}. */ public static class IpsBlockLoader extends SourceBlockLoader { - public IpsBlockLoader(SourceBasedValueFetcher fetcher, LeafIteratorLookup lookup) { + public IpsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { super(fetcher, lookup); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceBasedValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceBasedValueFetcher.java deleted file mode 100644 index 5807b5cdfe996..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceBasedValueFetcher.java +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper; - -/** - * Marker interface for {@link ValueFetcher} implementations that fetch values from the source. - */ -public interface SourceBasedValueFetcher extends ValueFetcher {} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index 5a92ba8120c11..a35215310a85e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -30,7 +30,7 @@ * * Field types that handle arrays directly should instead use {@link ArraySourceValueFetcher}. */ -public abstract class SourceValueFetcher implements SourceBasedValueFetcher { +public abstract class SourceValueFetcher implements ValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; private final IgnoredSourceFormat ignoredSourceFormat;