diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java index bbd58d3309ab6..2cf7cafa3dd74 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java @@ -368,7 +368,7 @@ public void benchmark() { blockFactory, ByteSizeValue.ofMb(1).getBytes(), fields(name), - List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException("can't load _source here"); }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))), 0 diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 70c28ee18184f..628e84173496f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -605,7 +605,7 @@ protected String delegatingTo() { } // fallback to _source (synthetic or not) - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()), blContext.indexSettings()); // MatchOnlyText never has norms, so we have to use the field names field BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup); @@ -636,7 +636,7 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name()), fieldDataContext.indexSettings()), fieldDataContext.lookupSupplier().get(), TextDocValuesField::new ); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index 8e4eda87abfb8..7a1a03cc537d7 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -148,11 +148,12 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher(context); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(SearchExecutionContext context) { + Set sourcePaths = context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(); + return new SourceValueFetcher(sourcePaths, nullValue, context.getIndexSettings().getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { return objectToFloat(value); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index d0bbf3059395d..35eb54ae6843a 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; @@ -262,6 +263,7 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) { ) ); + @SuppressWarnings("checkstyle:LineLength") public static final class ScaledFloatFieldType extends SimpleMappedFieldType { private final double scalingFactor; @@ -396,8 +398,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - - ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name())); + var valueFetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) @@ -489,7 +490,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedDoubleIndexFieldData.Builder( name(), valuesSourceType, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, ScaledFloatDocValuesField::new ); @@ -503,11 +504,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected Double parseSourceValue(Object value) { double doubleValue; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java index 74f1f9a16a6d6..88f14aa5ecc90 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java @@ -59,6 +59,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class MatchOnlyTextFieldTypeTests extends FieldTypeTestCase { @@ -315,6 +316,7 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet // when MappedFieldType.BlockLoaderContext blContext = mock(MappedFieldType.BlockLoaderContext.class); doReturn(FieldNamesFieldMapper.FieldNamesFieldType.get(false)).when(blContext).fieldNames(); + when(blContext.indexSettings()).thenReturn(indexSettings); BlockLoader blockLoader = ft.blockLoader(blContext); // then @@ -362,6 +364,7 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet // when MappedFieldType.BlockLoaderContext blContext = mock(MappedFieldType.BlockLoaderContext.class); + when(blContext.indexSettings()).thenReturn(indexSettings); doReturn(FieldNamesFieldMapper.FieldNamesFieldType.get(false)).when(blContext).fieldNames(); BlockLoader blockLoader = ft.blockLoader(blContext); diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 039f8c049e0c5..d04f2067d8aed 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -99,6 +99,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index b396e1ca206e3..18e2194ee6d42 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -1813,6 +1813,14 @@ public SourceFieldMapper.Mode getIndexMappingSourceMode() { return indexMappingSourceMode; } + public IgnoredSourceFieldMapper.IgnoredSourceFormat getIgnoredSourceFormat() { + if (getIndexMappingSourceMode() == SourceFieldMapper.Mode.SYNTHETIC) { + return IgnoredSourceFieldMapper.ignoredSourceFormat(getIndexVersionCreated()); + } else { + return IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } + } + /** * @return Whether recovery source should be enabled if needed. * Note that this is a node setting, and this setting is not sourced from index settings. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 78a8f85f5cf66..2d48bf82db1ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.geo.GeometryFormatterFactory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -172,9 +173,9 @@ protected Object parseSourceValue(Object value) { }; } - public ValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format) { + public ValueFetcher valueFetcher(Set sourcePaths, T nullValue, String format, IndexSettings indexSettings) { Function, List> formatter = getFormatter(format != null ? format : GeometryFormatterFactory.GEOJSON); - return new ArraySourceValueFetcher(sourcePaths, nullValueAsSource(nullValue)) { + return new ArraySourceValueFetcher(sourcePaths, nullValueAsSource(nullValue), indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { final List values = new ArrayList<>(); @@ -185,7 +186,7 @@ protected Object parseSourceValue(Object value) { } protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { - ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB); + var fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB, blContext.indexSettings()); // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index fa835e4356667..c73b4fefa8a4a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -10,6 +10,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; @@ -30,6 +31,7 @@ public abstract class ArraySourceValueFetcher implements ValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; + private final IgnoredSourceFormat ignoredSourceFormat; public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) { this(fieldName, context, null); @@ -43,15 +45,17 @@ public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { this.sourcePaths = context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(); this.nullValue = nullValue; + this.ignoredSourceFormat = context.getIndexSettings().getIgnoredSourceFormat(); } /** * @param sourcePaths The paths to pull source values from * @param nullValue An optional substitute value if the _source value is `null` */ - public ArraySourceValueFetcher(Set sourcePaths, Object nullValue) { + public ArraySourceValueFetcher(Set sourcePaths, Object nullValue, IgnoredSourceFormat ignoredSourceFormat) { this.sourcePaths = sourcePaths; this.nullValue = nullValue; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -76,7 +80,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, sourcePaths); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 3b467e383f20d..858b054012fe2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -107,7 +107,7 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) @Override public final StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return fetcher.storedFieldsSpec(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 56ff91eef69f0..d118683ba5942 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -281,11 +282,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected Boolean parseSourceValue(Object value) { if (value instanceof Boolean) { @@ -364,7 +368,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); + var fetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); BlockSourceReader.LeafIteratorLookup lookup = indexType.hasTerms() || isStored() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); @@ -431,7 +435,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedBooleanIndexFieldData.Builder( name(), CoreValuesSourceType.BOOLEAN, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, BooleanDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 9991abba7ba6a..e69793fb799cf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -662,8 +662,8 @@ public String parseSourceValue(Object value) { } // returns a Long to support source fallback which emulates numeric doc values for dates - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override public Long parseSourceValue(Object value) { String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString(); @@ -995,11 +995,13 @@ public Builder builder(BlockFactory factory, int expectedCount) { } }; } - BlockSourceReader.LeafIteratorLookup lookup = isStored() || indexType.hasPoints() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return new BlockSourceReader.LongsBlockLoader( + sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()), + lookup + ); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { @@ -1067,7 +1069,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedNumericIndexFieldData.Builder( name(), resolution.numericType().getValuesSourceType(), - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, resolution.getDefaultToScriptFieldFactory() ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index b66730d6704e0..59d36e58fd3af 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -65,7 +65,7 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep @Override public StoredFieldsSpec rowStrideStoredFieldSpec() { - return new StoredFieldsSpec(false, false, Set.of(), new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat)); + return new StoredFieldsSpec(false, false, Set.of(), new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat), null); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index d9865202a0592..3a297bb43ab37 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -494,7 +494,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherMultiGeoPointIndexFieldData.Builder( name(), valuesSourceType, - valueFetcher(sourcePaths, null, null), + valueFetcher(sourcePaths, null, null, fieldDataContext.indexSettings()), searchLookup, GeoPointDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 6db87007747ad..644ab73a0217b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -53,7 +53,6 @@ import java.util.Collections; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.function.BiFunction; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; @@ -481,12 +480,11 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (isSyntheticSource && blContext.parentField(name()) == null) { return blockLoaderFromFallbackSyntheticSource(blContext); } - // see #indexValue BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && hasPoints ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext), lookup); } private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) { @@ -503,8 +501,8 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(BlockLoaderContext blContext) { + return new SourceValueFetcher(blContext.sourcePaths(name()), nullValue, blContext.indexSettings().getIgnoredSourceFormat()) { @Override public InetAddress parseSourceValue(Object value) { return parse(value); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 89a158cf3bdb4..b337cdd41dcb3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -820,7 +821,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); + SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext)); } @@ -908,7 +909,7 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), fieldDataContext.lookupSupplier().get(), KeywordDocValuesField::new ); @@ -930,11 +931,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 6ca488e137d60..27f4ace5cc739 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -2055,7 +2056,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { // We only write the field names field if there aren't doc values or norms ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()), lookup); } @Override @@ -2077,7 +2078,12 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation == FielddataOperation.SCRIPT) { SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); - return type.getValueFetcherFieldDataBuilder(name(), valuesSourceType, searchLookup, sourceValueFetcher(sourcePaths)); + return type.getValueFetcherFieldDataBuilder( + name(), + valuesSourceType, + searchLookup, + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()) + ); } throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); @@ -2099,11 +2105,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValue, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index 7d1e722200ba3..a35215310a85e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -10,6 +10,8 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.Source; @@ -31,6 +33,7 @@ public abstract class SourceValueFetcher implements ValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; + private final IgnoredSourceFormat ignoredSourceFormat; public SourceValueFetcher(String fieldName, SearchExecutionContext context) { this(fieldName, context, null); @@ -41,16 +44,22 @@ public SourceValueFetcher(String fieldName, SearchExecutionContext context) { * @param nullValue An optional substitute value if the _source value is 'null'. */ public SourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { - this(context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(), nullValue); + this( + context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(), + nullValue, + context.getIndexSettings().getIgnoredSourceFormat() + ); } /** - * @param sourcePaths The paths to pull source values from - * @param nullValue An optional substitute value if the _source value is `null` + * @param sourcePaths The paths to pull source values from + * @param nullValue An optional substitute value if the _source value is `null` + * @param ignoredSourceFormat */ - public SourceValueFetcher(Set sourcePaths, Object nullValue) { + public SourceValueFetcher(Set sourcePaths, Object nullValue, IgnoredSourceFormat ignoredSourceFormat) { this.sourcePaths = sourcePaths; this.nullValue = nullValue; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -98,7 +107,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, sourcePaths); } /** @@ -142,8 +151,8 @@ protected Object parseSourceValue(Object value) { * Creates a {@link SourceValueFetcher} that converts source values to Strings * @param sourcePaths the paths to fetch values from in the source */ - public static SourceValueFetcher toString(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + public static SourceValueFetcher toString(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, null, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { return value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 5acb088040485..6c5231fdfb9a1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1116,9 +1116,8 @@ protected String delegatingTo() { if (isSyntheticSourceEnabled() && syntheticSourceDelegate.isEmpty() && parentField == null) { return fallbackSyntheticSourceBlockLoader(blContext); } - // otherwise, load values from _source (synthetic or not) - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()), blContext.indexSettings()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext)); } @@ -1247,7 +1246,7 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name()), fieldDataContext.indexSettings()), fieldDataContext.lookupSupplier().get(), TextDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 69a520d2bf518..de8c9020c2f3e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -753,15 +754,18 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } public IgnoreAbove ignoreAbove() { return ignoreAbove; } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, null, indexSettings.getIgnoredSourceFormat()) { @Override @SuppressWarnings("unchecked") protected Object parseSourceValue(Object value) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 33d99c5628732..84d32874a17b3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2691,13 +2691,16 @@ public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) { return new BlockDocValuesReader.DenseVectorFromBinaryBlockLoader(name(), dims, indexVersionCreated, element.elementType()); } - BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.DenseVectorBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, dims); + return new BlockSourceReader.DenseVectorBlockLoader( + sourceValueFetcher(blContext.sourcePaths(name()), blContext.indexSettings()), + lookup, + dims + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, null, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index f494fdd0acfab..6eb4c10c68b11 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.get.GetResult; import org.elasticsearch.index.mapper.DocumentParser; @@ -279,11 +280,12 @@ private static Fields generateTermVectors( } } if (source != null) { + IndexSettings indexSettings = indexShard.indexSettings(); MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); Source s = Source.fromMap(source, XContentType.JSON); for (String field : fields) { if (values.containsKey(field) == false) { - SourceValueFetcher valueFetcher = SourceValueFetcher.toString(mappingLookup.sourcePaths(field)); + SourceValueFetcher valueFetcher = SourceValueFetcher.toString(mappingLookup.sourcePaths(field), indexSettings); List ignoredValues = new ArrayList<>(); List v = valueFetcher.fetchValues(s, -1, ignoredValues); if (v.isEmpty() == false) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java index ce52fabb20049..384c98390e612 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java @@ -10,6 +10,7 @@ package org.elasticsearch.search.fetch; import org.elasticsearch.index.mapper.IgnoredFieldsSpec; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import java.util.Collection; import java.util.HashSet; @@ -25,10 +26,11 @@ public record StoredFieldsSpec( boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields, - IgnoredFieldsSpec ignoredFieldsSpec + IgnoredFieldsSpec ignoredFieldsSpec, + Set sourcePaths ) { public StoredFieldsSpec(boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields) { - this(requiresSource, requiresMetadata, requiredStoredFields, IgnoredFieldsSpec.NONE); + this(requiresSource, requiresMetadata, requiredStoredFields, IgnoredFieldsSpec.NONE, null); } public boolean noRequirements() { @@ -52,6 +54,14 @@ public boolean onlyRequiresIgnoredFields() { */ public static final StoredFieldsSpec NEEDS_SOURCE = new StoredFieldsSpec(true, false, Set.of()); + public static StoredFieldsSpec withSourcePaths(IgnoredSourceFormat ignoredSourceFormat, Set sourcePaths) { + // The fields in source paths might also be in ignored source, so include source paths there as well. + IgnoredFieldsSpec ignoredFieldsSpec = ignoredSourceFormat == IgnoredSourceFormat.NO_IGNORED_SOURCE + ? new IgnoredFieldsSpec(sourcePaths, ignoredSourceFormat) + : IgnoredFieldsSpec.NONE; + return new StoredFieldsSpec(true, false, Set.of(), ignoredFieldsSpec, sourcePaths); + } + /** * Combine these stored field requirements with those from another StoredFieldsSpec */ @@ -70,11 +80,23 @@ public StoredFieldsSpec merge(StoredFieldsSpec other) { mergedFields = new HashSet<>(this.requiredStoredFields); mergedFields.addAll(other.requiredStoredFields); } + Set mergedSourcePaths; + if (this.sourcePaths != null && other.sourcePaths != null) { + mergedSourcePaths = new HashSet<>(this.sourcePaths); + mergedSourcePaths.addAll(other.sourcePaths); + } else if (this.sourcePaths != null) { + mergedSourcePaths = this.sourcePaths; + } else if (other.sourcePaths != null) { + mergedSourcePaths = other.sourcePaths; + } else { + mergedSourcePaths = null; + } return new StoredFieldsSpec( this.requiresSource || other.requiresSource, this.requiresMetadata || other.requiresMetadata, mergedFields, - ignoredFieldsSpec.merge(other.ignoredFieldsSpec) + ignoredFieldsSpec.merge(other.ignoredFieldsSpec), + mergedSourcePaths ); } diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java index e460b5a460194..46d12f2a65e8d 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java @@ -39,7 +39,8 @@ public void testSupports() { false, false, Set.of(), - new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ) ) ); @@ -50,7 +51,8 @@ public void testSupports() { false, false, Set.of(), - new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ) ) ); @@ -61,7 +63,8 @@ public void testSupports() { true, false, Set.of(), - new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ) ) ); @@ -129,7 +132,8 @@ private void testLoader( false, false, Set.of(), - new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ); assertTrue(IgnoredSourceFieldLoader.supports(spec)); iw.addDocument(doc); diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java index 0255506816799..bff84ec345522 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java @@ -48,7 +48,7 @@ private StoredFieldsSpec fieldsSpec( Set ignoredFields, IgnoredSourceFieldMapper.IgnoredSourceFormat format ) { - return new StoredFieldsSpec(false, false, storedFields, new IgnoredFieldsSpec(ignoredFields, format)); + return new StoredFieldsSpec(false, false, storedFields, new IgnoredFieldsSpec(ignoredFields, format), null); } public void testEmpty() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index 1fa9c85a5c738..fa0f4c0bd4c94 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -11,9 +11,15 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -49,12 +55,32 @@ public void testEmptyArray() throws IOException { } private void loadBlock(LeafReaderContext ctx, Consumer test) throws IOException { - ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field")); + boolean syntheticSource = randomBoolean(); + IndexMetadata indexMetadata = IndexMetadata.builder("index") + .settings( + Settings.builder() + .put( + ESTestCase.indexSettings(IndexVersion.current(), 1, 1) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), syntheticSource ? "synthetic" : "stored") + .build() + ) + ) + .build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + var valueFetcher = SourceValueFetcher.toString(Set.of("field"), indexSettings); BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field"); BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup); assertThat(loader.columnAtATimeReader(ctx), nullValue()); BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx); - assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.NEEDS_SOURCE)); + assertThat( + loader.rowStrideStoredFieldSpec(), + equalTo( + StoredFieldsSpec.withSourcePaths( + syntheticSource ? IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE : IgnoredSourceFormat.NO_IGNORED_SOURCE, + Set.of("field") + ) + ) + ); BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null), loader.rowStrideStoredFieldSpec().requiresSource() ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) : null diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 332c727ffadde..cee149071134f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1446,8 +1446,9 @@ public void testEmpty() throws Exception { throw new IllegalArgumentException("Can't load source in scripts in synthetic mode"); } : SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()); SearchLookup searchLookup = new SearchLookup(null, null, sourceProvider); + var indexSettings = mapperService.getIndexSettings(); IndexFieldData sfd = ft.fielddataBuilder( - new FieldDataContext("", null, () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) + new FieldDataContext("", indexSettings, () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) ).build(null, null); LeafFieldData lfd = sfd.load(getOnlyLeafReader(searcher.getIndexReader()).getContext()); TextDocValuesField scriptDV = (TextDocValuesField) lfd.getScriptFieldFactory("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index fcab101da9baa..4787b444d732f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -55,6 +55,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TextFieldTypeTests extends FieldTypeTestCase { @@ -374,7 +375,9 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet ); // when - BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + var context = mock(MappedFieldType.BlockLoaderContext.class); + when(context.indexSettings()).thenReturn(indexSettings); + BlockLoader blockLoader = ft.blockLoader(context); // then // verify that we don't delegate anything @@ -423,7 +426,9 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet ); // when - BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + var context = mock(MappedFieldType.BlockLoaderContext.class); + when(context.indexSettings()).thenReturn(indexSettings); + BlockLoader blockLoader = ft.blockLoader(context); // then // verify that we don't delegate anything diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java index c8d7ad8127b55..dadeae3cc654c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java @@ -18,12 +18,14 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.search.AutomatonQueries; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; @@ -186,6 +188,7 @@ public void testFetchSourceValue() throws IOException { Map sourceValue = Map.of("key", "value"); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY)); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath("field.key")).thenReturn(Set.of("field.key")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 5932180ac3c03..8d2787ab76dc9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -2403,6 +2403,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; SourceToParse source = source(b -> b.field(ft.name(), value)); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 5f26b046e0c8e..5311c1664c9c3 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import static java.util.Collections.emptyMap; @@ -1585,9 +1586,13 @@ public void testFetchMetadataFieldWithSourceDisabled() throws IOException { } public void testStoredFieldsSpec() throws IOException { + var mapperService = createMapperService(); List fields = List.of(new FieldAndFormat("field", null)); - FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(createMapperService()), fields); - assertEquals(StoredFieldsSpec.NEEDS_SOURCE, fieldFetcher.storedFieldsSpec()); + FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(mapperService), fields); + assertEquals( + StoredFieldsSpec.withSourcePaths(mapperService.getIndexSettings().getIgnoredSourceFormat(), Set.of("field")), + fieldFetcher.storedFieldsSpec() + ); } private List fieldAndFormatList(String name, String format, boolean includeUnmapped) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index fae33f51740f1..757ff99e0d508 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.Query; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.FieldLookup; @@ -24,6 +25,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; @@ -63,6 +65,7 @@ public static List fetchSourceValue(MappedFieldType fieldType, Object sourceV public static List fetchSourceValue(MappedFieldType fieldType, Object sourceValue, String format) throws IOException { String field = fieldType.name(); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY)); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); @@ -74,6 +77,7 @@ public static List fetchSourceValue(MappedFieldType fieldType, Object sourceV public static List fetchSourceValues(MappedFieldType fieldType, Object... values) throws IOException { String field = fieldType.name(); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY)); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 98d6576f59a44..a2967973fb102 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -859,6 +859,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java index d20a002407be6..965d1a5037205 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.Set; /** * Context of each shard we're operating against. @@ -48,7 +49,7 @@ public interface ShardContext extends RefCounted { /** * Build something to load source {@code _source}. */ - SourceLoader newSourceLoader(); + SourceLoader newSourceLoader(Set sourcePaths); /** * Returns something to load values from this field into a {@link Block}. diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java index 6f00e97a1f9f2..8acc4d2a4cc93 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java @@ -181,7 +181,7 @@ private void fieldsMoved(LeafReaderContext ctx, int shard) throws IOException { } SourceLoader sourceLoader = null; if (storedFieldsSpec.requiresSource()) { - sourceLoader = operator.shardContexts.get(shard).newSourceLoader().get(); + sourceLoader = operator.shardContexts.get(shard).newSourceLoader().apply(storedFieldsSpec.sourcePaths()); storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } storedFields = new BlockLoaderStoredFieldsFromLeafLoader( diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java index 59e54103eb310..6203842dc9e0d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java @@ -139,7 +139,7 @@ private void loadFromRowStrideReaders( SourceLoader sourceLoader = null; ValuesSourceReaderOperator.ShardContext shardContext = operator.shardContexts.get(shard); if (storedFieldsSpec.requiresSource()) { - sourceLoader = shardContext.newSourceLoader().get(); + sourceLoader = shardContext.newSourceLoader().apply(storedFieldsSpec.sourcePaths()); storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } if (storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java index 3f3d73eec88ae..0a230f1bbe609 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java @@ -28,9 +28,10 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.function.Function; import java.util.function.IntFunction; -import java.util.function.Supplier; /** * Operator that extracts doc_values from a Lucene index out of pages that have been produced by {@link LuceneSourceOperator} @@ -91,7 +92,11 @@ public String describe() { */ public record FieldInfo(String name, ElementType type, boolean nullsFiltered, IntFunction blockLoader) {} - public record ShardContext(IndexReader reader, Supplier newSourceLoader, double storedFieldsSequentialProportion) {} + public record ShardContext( + IndexReader reader, + Function, SourceLoader> newSourceLoader, + double storedFieldsSequentialProportion + ) {} final BlockFactory blockFactory; /** diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java index 56acaa86c299d..758a9f55d5655 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java @@ -190,7 +190,7 @@ public void testPushRoundToToQuery() throws IOException { f -> new BlockDocValuesReader.LongsBlockLoader("v") ) ), - List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException(); }, 0.8)), 0 diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java index 5b25fb6fb64af..505cd7c212bb9 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java @@ -210,7 +210,7 @@ private List runQuery(Set values, Query query, boolean shuffleDocs unused -> new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(FIELD) ) ), - List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException(); }, 0.2)), 0 diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java index 3249031824210..609cc5875d3ff 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @@ -456,7 +457,7 @@ public IndexSearcher searcher() { } @Override - public SourceLoader newSourceLoader() { + public SourceLoader newSourceLoader(Set sourcePaths) { return SourceLoader.FROM_STORED_SOURCE; } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java index 4e562217a00e3..15c43ce16acfa 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java @@ -208,7 +208,7 @@ private MapperService mapperService(String indexKey) { private List initShardContexts() { return INDICES.keySet() .stream() - .map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), () -> SourceLoader.FROM_STORED_SOURCE, 0.2)) + .map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2)) .toList(); } @@ -1325,7 +1325,11 @@ public void testWithNulls() throws IOException { LuceneOperator.NO_LIMIT, false // no scoring ); - var vsShardContext = new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE, 0.2); + var vsShardContext = new ValuesSourceReaderOperator.ShardContext( + reader(indexKey), + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, + 0.2 + ); try ( Driver driver = TestDriverFactory.create( driverContext, @@ -1455,7 +1459,7 @@ public void testDescriptionOfMany() throws IOException { ValuesSourceReaderOperator.Factory factory = new ValuesSourceReaderOperator.Factory( ByteSizeValue.ofGb(1), cases.stream().map(c -> c.info).toList(), - List.of(new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE, 0.2)), + List.of(new ValuesSourceReaderOperator.ShardContext(reader(indexKey), (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2)), 0 ); assertThat(factory.describe(), equalTo("ValuesSourceReaderOperator[fields = [" + cases.size() + " fields]]")); @@ -1484,7 +1488,7 @@ public void testManyShards() throws IOException { for (int s = 0; s < shardCount; s++) { contexts.add(new LuceneSourceOperatorTests.MockShardContext(readers[s], s)); readerShardContexts.add( - new ValuesSourceReaderOperator.ShardContext(readers[s], () -> SourceLoader.FROM_STORED_SOURCE, 0.2) + new ValuesSourceReaderOperator.ShardContext(readers[s], (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2) ); } var luceneFactory = new LuceneSourceOperator.Factory( diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java index 3a83d365f6f57..801a4e7fc9056 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java @@ -164,7 +164,7 @@ static Operator.OperatorFactory factory(IndexReader reader, String name, Element List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -500,7 +500,7 @@ public void testManySingleDocPages() { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -620,7 +620,7 @@ private void loadSimpleAndAssert( List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -639,7 +639,7 @@ private void loadSimpleAndAssert( List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -738,7 +738,7 @@ private void testLoadAllStatus(boolean allInOnePage) { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -972,7 +972,7 @@ private void testLoadLong(boolean shuffle, boolean manySegments) throws IOExcept List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1641,7 +1641,7 @@ public void testNullsShared() { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1693,7 +1693,7 @@ private void testSequentialStoredFields(boolean sequential, int docCount) throws List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1728,7 +1728,7 @@ public void testDescriptionOfMany() throws IOException { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1761,7 +1761,7 @@ public void testManyShards() throws IOException { readerShardContexts.add( new ValuesSourceReaderOperator.ShardContext( readers[s], - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java index 7da6db1b5235e..9a551093bb7a0 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java @@ -360,7 +360,7 @@ private void runLookup(List keyTypes, PopulateIndices populateIndices, ValuesSourceReaderOperator.Factory reader = new ValuesSourceReaderOperator.Factory( PlannerSettings.VALUES_LOADING_JUMBO_SIZE.getDefault(Settings.EMPTY), fieldInfos, - List.of(new ValuesSourceReaderOperator.ShardContext(searchContext.getSearchExecutionContext().getIndexReader(), () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(searchContext.getSearchExecutionContext().getIndexReader(), (paths) -> { throw new IllegalStateException("can't load source here"); }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))), 0 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index cd5cfab6eb102..0ce74047affc4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -59,6 +59,7 @@ import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.xpack.esql.core.expression.Attribute; @@ -453,8 +454,9 @@ public String shardIdentifier() { } @Override - public SourceLoader newSourceLoader() { - return ctx.newSourceLoader(null, false); + public SourceLoader newSourceLoader(Set sourcePaths) { + var filter = sourcePaths != null ? new SourceFilter(sourcePaths.toArray(new String[0]), null) : null; + return ctx.newSourceLoader(filter, false); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 1a8b162eb1b46..6e16bc4a43512 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -1053,7 +1053,7 @@ private String generateInvalidQueryInferenceResultsMessage(StringBuilder baseMes @Override public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { String name = useLegacyFormat ? name().concat(".text") : name(); - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name)); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name), blContext.indexSettings()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 74459072b8979..b11eb62b9db7d 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -398,7 +399,8 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - ValueFetcher valueFetcher = new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted) { + var ignoredSourceFormat = blContext.indexSettings().getIgnoredSourceFormat(); + var valueFetcher = new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted, ignoredSourceFormat) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { @@ -511,7 +513,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedUnsignedLongIndexFieldData.Builder( name(), valuesSourceType, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, fieldDataContext.indexSettings()), searchLookup, UnsignedLongDocValuesField::new ); @@ -525,11 +527,14 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.getIndexSettings() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValueFormatted) { + private SourceValueFetcher sourceValueFetcher(Set sourcePaths, IndexSettings indexSettings) { + return new SourceValueFetcher(sourcePaths, nullValueFormatted, indexSettings.getIgnoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java index ad29a191aace3..5834aca5fa0a5 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java @@ -374,6 +374,7 @@ protected void assertFetch(MapperService mapperService, String field, Object val MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; SourceToParse source = source(b -> b.field(ft.name(), value)); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); when(searchExecutionContext.getForField(ft, fdt)).thenAnswer(inv -> fieldDataLookup(mapperService).apply(ft, () -> {