diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java index bbd58d3309ab6..9a22ac68a6521 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java @@ -51,6 +51,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -259,6 +260,11 @@ public String parentField(String field) { public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { return FieldNamesFieldMapper.FieldNamesFieldType.get(true); } + + @Override + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + throw new UnsupportedOperationException(); + } }); } throw new IllegalArgumentException("can't read [" + name + "]"); @@ -368,7 +374,7 @@ public void benchmark() { blockFactory, ByteSizeValue.ofMb(1).getBytes(), fields(name), - List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException("can't load _source here"); }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))), 0 diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 70c28ee18184f..d9e5b677548e8 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; @@ -605,7 +606,7 @@ protected String delegatingTo() { } // fallback to _source (synthetic or not) - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()), blContext.ignoredSourceFormat()); // MatchOnlyText never has norms, so we have to use the field names field BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup); @@ -636,7 +637,10 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + SourceValueFetcher.toString( + fieldDataContext.sourcePathsLookup().apply(name()), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ), fieldDataContext.lookupSupplier().get(), TextDocValuesField::new ); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index 8e4eda87abfb8..c9003e4a5769c 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -148,11 +148,12 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher(context); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher(SearchExecutionContext context) { + Set sourcePaths = context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(); + return new SourceValueFetcher(sourcePaths, nullValue, context.ignoredSourceFormat()) { @Override protected Object parseSourceValue(Object value) { return objectToFloat(value); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index d2bae840b905d..c749f46053deb 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -398,7 +398,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name())); + ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.ignoredSourceFormat()); BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isStored() // We only write the field names field if there aren't doc values ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) @@ -487,10 +487,18 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; + if (isSyntheticSource) { + ignoredSourceFormat = IgnoredSourceFieldMapper.ignoredSourceFormat( + fieldDataContext.indexSettings().getIndexVersionCreated() + ); + } else { + ignoredSourceFormat = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } return new SourceValueFetcherSortedDoubleIndexFieldData.Builder( name(), valuesSourceType, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, ignoredSourceFormat), searchLookup, ScaledFloatDocValuesField::new ); @@ -504,11 +512,17 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.ignoredSourceFormat() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher( + Set sourcePaths, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) { + return new SourceValueFetcher(sourcePaths, nullValue, ignoredSourceFormat) { @Override protected Double parseSourceValue(Object value) { double doubleValue; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index 76d45d682a49f..9921bbdf909ad 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -37,6 +37,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; @@ -194,7 +195,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) return new DocValueFetcher( docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH), - StoredFieldsSpec.NEEDS_SOURCE // for now we assume runtime fields need source + StoredFieldsSpec.withSourcePaths(context.ignoredSourceFormat(), Set.of(name())) // for now we assume runtime fields need source ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index fa835e4356667..0a034aeca02e0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -30,6 +30,7 @@ public abstract class ArraySourceValueFetcher implements ValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; + private final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) { this(fieldName, context, null); @@ -43,6 +44,7 @@ public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context) public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { this.sourcePaths = context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(); this.nullValue = nullValue; + this.ignoredSourceFormat = context.ignoredSourceFormat(); } /** @@ -52,6 +54,7 @@ public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context, public ArraySourceValueFetcher(Set sourcePaths, Object nullValue) { this.sourcePaths = sourcePaths; this.nullValue = nullValue; + this.ignoredSourceFormat = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; } @Override @@ -76,7 +79,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, sourcePaths); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 3b467e383f20d..858b054012fe2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -107,7 +107,7 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) @Override public final StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return fetcher.storedFieldsSpec(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 56ff91eef69f0..b50ba6dc4d90e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -56,6 +56,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Consumer; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; @@ -203,11 +204,22 @@ private FieldValues scriptValues() { return null; } BooleanFieldScript.Factory scriptFactory = scriptCompiler.compile(script.get(), BooleanFieldScript.CONTEXT); - return scriptFactory == null - ? null - : (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) - .newInstance(ctx) - .runForDoc(doc, consumer); + if (scriptFactory == null) { + return null; + } + return new FieldValues<>() { + @Override + public void valuesForDoc(SearchLookup lookup, LeafReaderContext ctx, int doc, Consumer consumer) { + scriptFactory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) + .newInstance(ctx) + .runForDoc(doc, consumer); + } + + @Override + public String name() { + return leafName(); + } + }; } } @@ -281,11 +293,17 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.ignoredSourceFormat() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher( + Set sourcePaths, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) { + return new SourceValueFetcher(sourcePaths, nullValue, ignoredSourceFormat) { @Override protected Boolean parseSourceValue(Object value) { if (value instanceof Boolean) { @@ -364,7 +382,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); + ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.ignoredSourceFormat()); BlockSourceReader.LeafIteratorLookup lookup = indexType.hasTerms() || isStored() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); @@ -427,11 +445,18 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation == FielddataOperation.SCRIPT) { SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); - + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; + if (isSyntheticSource) { + ignoredSourceFormat = IgnoredSourceFieldMapper.ignoredSourceFormat( + fieldDataContext.indexSettings().getIndexVersionCreated() + ); + } else { + ignoredSourceFormat = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } return new SourceValueFetcherSortedBooleanIndexFieldData.Builder( name(), CoreValuesSourceType.BOOLEAN, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, ignoredSourceFormat), searchLookup, BooleanDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index cc4ec0c2f8296..4446b39a41e93 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -79,6 +79,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.LongSupplier; @@ -368,15 +369,22 @@ private FieldValues scriptValues() { return null; } DateFieldScript.Factory factory = scriptCompiler.compile(script.get(), DateFieldScript.CONTEXT); - return factory == null - ? null - : (lookup, ctx, doc, consumer) -> factory.newFactory( - leafName(), - script.get().getParams(), - lookup, - buildFormatter(), - OnScriptError.FAIL - ).newInstance(ctx).runForDoc(doc, consumer::accept); + if (factory == null) { + return null; + } + return new FieldValues<>() { + @Override + public void valuesForDoc(SearchLookup lookup, LeafReaderContext ctx, int doc, Consumer consumer) { + factory.newFactory(leafName(), script.get().getParams(), lookup, buildFormatter(), OnScriptError.FAIL) + .newInstance(ctx) + .runForDoc(doc, consumer::accept); + } + + @Override + public String name() { + return leafName(); + } + }; } @Override @@ -662,8 +670,11 @@ public String parseSourceValue(Object value) { } // returns a Long to support source fallback which emulates numeric doc values for dates - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher( + Set sourcePaths, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) { + return new SourceValueFetcher(sourcePaths, nullValue, ignoredSourceFormat) { @Override public Long parseSourceValue(Object value) { String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString(); @@ -999,7 +1010,10 @@ public Builder builder(BlockFactory factory, int expectedCount) { BlockSourceReader.LeafIteratorLookup lookup = isStored() || indexType.hasPoints() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return new BlockSourceReader.LongsBlockLoader( + sourceValueFetcher(blContext.sourcePaths(name()), blContext.ignoredSourceFormat()), + lookup + ); } private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { @@ -1063,11 +1077,18 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation == FielddataOperation.SCRIPT) { SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); - + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; + if (isSyntheticSource) { + ignoredSourceFormat = IgnoredSourceFieldMapper.ignoredSourceFormat( + fieldDataContext.indexSettings().getIndexVersionCreated() + ); + } else { + ignoredSourceFormat = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } return new SourceValueFetcherSortedNumericIndexFieldData.Builder( name(), resolution.numericType().getValuesSourceType(), - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, ignoredSourceFormat), searchLookup, resolution.getDefaultToScriptFieldFactory() ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index b66730d6704e0..59d36e58fd3af 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -65,7 +65,7 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep @Override public StoredFieldsSpec rowStrideStoredFieldSpec() { - return new StoredFieldsSpec(false, false, Set.of(), new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat)); + return new StoredFieldsSpec(false, false, Set.of(), new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat), null); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index d9865202a0592..1693f77ee569d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -67,6 +67,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Consumer; import java.util.function.Function; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES; @@ -199,11 +200,22 @@ private FieldValues scriptValues() { return null; } GeoPointFieldScript.Factory factory = scriptCompiler.compile(this.script.get(), GeoPointFieldScript.CONTEXT); - return factory == null - ? null - : (lookup, ctx, doc, consumer) -> factory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) - .newInstance(ctx) - .runForDoc(doc, consumer); + if (factory == null) { + return null; + } + return new FieldValues<>() { + @Override + public void valuesForDoc(SearchLookup lookup, LeafReaderContext ctx, int doc, Consumer consumer) { + factory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) + .newInstance(ctx) + .runForDoc(doc, consumer); + } + + @Override + public String name() { + return leafName(); + } + }; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java index cf6aa6b521308..f32137bc0b3c3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java @@ -42,6 +42,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; public final class GeoPointScriptFieldType extends AbstractScriptFieldType implements GeoShapeQueryable { @@ -223,7 +224,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths(context.ignoredSourceFormat(), Set.of(name())); } }; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 742f9751ce280..1a05d67b975e2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -483,7 +483,12 @@ public BytesRef filterValue(BytesRef value, Function, Map scriptValues() { return null; } IpFieldScript.Factory factory = scriptCompiler.compile(this.script.get(), IpFieldScript.CONTEXT); - return factory == null - ? null - : (lookup, ctx, doc, consumer) -> factory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) - .newInstance(ctx) - .runForDoc(doc, consumer); + if (factory == null) { + return null; + } + return new FieldValues<>() { + @Override + public void valuesForDoc(SearchLookup lookup, LeafReaderContext ctx, int doc, Consumer consumer) { + factory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) + .newInstance(ctx) + .runForDoc(doc, consumer); + } + + @Override + public String name() { + return leafName(); + } + }; } @Override @@ -486,7 +498,10 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && hasPoints ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return new BlockSourceReader.IpsBlockLoader( + sourceValueFetcher(blContext.sourcePaths(name()), blContext.ignoredSourceFormat()), + lookup + ); } private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) { @@ -503,8 +518,11 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher( + Set sourcePaths, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) { + return new SourceValueFetcher(sourcePaths, nullValue, ignoredSourceFormat) { @Override public InetAddress parseSourceValue(Object value) { return parse(value); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 89a158cf3bdb4..f8d599815a156 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -85,6 +85,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Consumer; import java.util.function.Supplier; import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; @@ -386,11 +387,22 @@ private FieldValues scriptValues() { return null; } StringFieldScript.Factory scriptFactory = scriptCompiler.compile(script.get(), StringFieldScript.CONTEXT); - return scriptFactory == null - ? null - : (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) - .newInstance(ctx) - .runForDoc(doc, consumer); + if (scriptFactory == null) { + return null; + } + return new FieldValues<>() { + @Override + public void valuesForDoc(SearchLookup lookup, LeafReaderContext ctx, int doc, Consumer consumer) { + scriptFactory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) + .newInstance(ctx) + .runForDoc(doc, consumer); + } + + @Override + public String name() { + return leafName(); + } + }; } @Override @@ -820,7 +832,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); + SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()), blContext.ignoredSourceFormat()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext)); } @@ -903,12 +915,11 @@ protected BytesRef storedToBytesRef(Object stored) { } }; } - Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE), fieldDataContext.lookupSupplier().get(), KeywordDocValuesField::new ); @@ -930,11 +941,17 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.ignoredSourceFormat() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher( + Set sourcePaths, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) { + return new SourceValueFetcher(sourcePaths, nullValue, ignoredSourceFormat) { @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 771a54afa267a..597840bde8f75 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -703,6 +703,8 @@ public interface BlockLoaderContext { * The {@code _field_names} field mapper, mostly used to check if it is enabled. */ FieldNamesFieldMapper.FieldNamesFieldType fieldNames(); + + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 93a157f435111..947280b99502f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -489,6 +489,10 @@ public boolean isSourceSynthetic() { return sfm != null && sfm.isSynthetic(); } + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + return mapping.ignoredSourceFormat(); + } + /** * Build something to load source {@code _source}. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java index 008bbdbd0005b..3c0c9a3090dc4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedValueFetcher.java @@ -18,8 +18,10 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -31,13 +33,19 @@ public class NestedValueFetcher implements ValueFetcher { // the name of the nested field without the full path, i.e. in foo.bar.baz it would be baz private final String nestedFieldName; private final String[] nestedPathParts; + private final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; - public NestedValueFetcher(String nestedField, FieldFetcher nestedFieldFetcher) { + public NestedValueFetcher( + String nestedField, + FieldFetcher nestedFieldFetcher, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) { assert nestedField != null && nestedField.isEmpty() == false; this.nestedFieldPath = nestedField; this.nestedFieldFetcher = nestedFieldFetcher; this.nestedPathParts = nestedFieldPath.split("\\."); this.nestedFieldName = nestedPathParts[nestedPathParts.length - 1]; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -92,6 +100,6 @@ public void setNextReader(LeafReaderContext context) { @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, new HashSet<>(Arrays.asList(nestedPathParts))); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 6ca488e137d60..9c7626f3edd90 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -80,6 +80,7 @@ import java.util.Objects; import java.util.Set; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Function; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; @@ -725,9 +726,19 @@ public Double parse(XContentParser parser, boolean coerce) throws IOException { @Override public FieldValues compile(String fieldName, Script script, ScriptCompiler compiler) { DoubleFieldScript.Factory scriptFactory = compiler.compile(script, DoubleFieldScript.CONTEXT); - return (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(fieldName, script.getParams(), lookup, OnScriptError.FAIL) - .newInstance(ctx) - .runForDoc(doc, consumer::accept); + return new FieldValues<>() { + @Override + public void valuesForDoc(SearchLookup lookup, LeafReaderContext ctx, int doc, Consumer consumer) { + scriptFactory.newFactory(fieldName, script.getParams(), lookup, OnScriptError.FAIL) + .newInstance(ctx) + .runForDoc(doc, consumer::accept); + } + + @Override + public String name() { + return fieldName; + } + }; } @Override @@ -1339,9 +1350,19 @@ public Long parse(XContentParser parser, boolean coerce) throws IOException { @Override public FieldValues compile(String fieldName, Script script, ScriptCompiler compiler) { final LongFieldScript.Factory scriptFactory = compiler.compile(script, LongFieldScript.CONTEXT); - return (lookup, ctx, doc, consumer) -> scriptFactory.newFactory(fieldName, script.getParams(), lookup, OnScriptError.FAIL) - .newInstance(ctx) - .runForDoc(doc, consumer::accept); + return new FieldValues<>() { + @Override + public void valuesForDoc(SearchLookup lookup, LeafReaderContext ctx, int doc, Consumer consumer) { + scriptFactory.newFactory(fieldName, script.getParams(), lookup, OnScriptError.FAIL) + .newInstance(ctx) + .runForDoc(doc, consumer::accept); + } + + @Override + public String name() { + return fieldName; + } + }; } @Override @@ -2055,7 +2076,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { // We only write the field names field if there aren't doc values or norms ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name()), blContext.ignoredSourceFormat()), lookup); } @Override @@ -2077,7 +2098,20 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation == FielddataOperation.SCRIPT) { SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); - return type.getValueFetcherFieldDataBuilder(name(), valuesSourceType, searchLookup, sourceValueFetcher(sourcePaths)); + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; + if (isSyntheticSource) { + ignoredSourceFormat = IgnoredSourceFieldMapper.ignoredSourceFormat( + fieldDataContext.indexSettings().getIndexVersionCreated() + ); + } else { + ignoredSourceFormat = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } + return type.getValueFetcherFieldDataBuilder( + name(), + valuesSourceType, + searchLookup, + sourceValueFetcher(sourcePaths, ignoredSourceFormat) + ); } throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); @@ -2099,11 +2133,17 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.ignoredSourceFormat() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValue) { + private SourceValueFetcher sourceValueFetcher( + Set sourcePaths, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) { + return new SourceValueFetcher(sourcePaths, nullValue, ignoredSourceFormat) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index 7d1e722200ba3..e0de62dc398fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -31,6 +31,7 @@ public abstract class SourceValueFetcher implements ValueFetcher { private final Set sourcePaths; private final @Nullable Object nullValue; + private final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; public SourceValueFetcher(String fieldName, SearchExecutionContext context) { this(fieldName, context, null); @@ -41,16 +42,17 @@ public SourceValueFetcher(String fieldName, SearchExecutionContext context) { * @param nullValue An optional substitute value if the _source value is 'null'. */ public SourceValueFetcher(String fieldName, SearchExecutionContext context, Object nullValue) { - this(context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(), nullValue); + this(context.isSourceEnabled() ? context.sourcePath(fieldName) : Collections.emptySet(), nullValue, context.ignoredSourceFormat()); } /** - * @param sourcePaths The paths to pull source values from - * @param nullValue An optional substitute value if the _source value is `null` + * @param sourcePaths The paths to pull source values from + * @param nullValue An optional substitute value if the _source value is `null` */ - public SourceValueFetcher(Set sourcePaths, Object nullValue) { + public SourceValueFetcher(Set sourcePaths, Object nullValue, IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat) { this.sourcePaths = sourcePaths; this.nullValue = nullValue; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -98,7 +100,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, sourcePaths); } /** @@ -140,10 +142,12 @@ protected Object parseSourceValue(Object value) { /** * Creates a {@link SourceValueFetcher} that converts source values to Strings - * @param sourcePaths the paths to fetch values from in the source + * + * @param sourcePaths the paths to fetch values from in the source + * @param ignoredSourceFormat */ - public static SourceValueFetcher toString(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + public static SourceValueFetcher toString(Set sourcePaths, IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat) { + return new SourceValueFetcher(sourcePaths, null, ignoredSourceFormat) { @Override protected Object parseSourceValue(Object value) { return value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 5acb088040485..6368c284a5873 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1118,7 +1118,7 @@ protected String delegatingTo() { } // otherwise, load values from _source (synthetic or not) - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()), blContext.ignoredSourceFormat()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext)); } @@ -1247,7 +1247,10 @@ protected BytesRef storedToBytesRef(Object stored) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - SourceValueFetcher.toString(fieldDataContext.sourcePathsLookup().apply(name())), + SourceValueFetcher.toString( + fieldDataContext.sourcePathsLookup().apply(name()), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ), fieldDataContext.lookupSupplier().get(), TextDocValuesField::new ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 69a520d2bf518..7e87c7f834658 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -753,15 +753,16 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher(context); } public IgnoreAbove ignoreAbove() { return ignoreAbove; } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + private SourceValueFetcher sourceValueFetcher(SearchExecutionContext context) { + Set sourcePaths = context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(); + return new SourceValueFetcher(sourcePaths, null, context.ignoredSourceFormat()) { @Override @SuppressWarnings("unchecked") protected Object parseSourceValue(Object value) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 33d99c5628732..8609ae9ec3c1d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -63,6 +63,7 @@ import org.elasticsearch.index.mapper.BlockSourceReader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -2693,11 +2694,18 @@ public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { } BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.DenseVectorBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, dims); + return new BlockSourceReader.DenseVectorBlockLoader( + sourceValueFetcher(blContext.sourcePaths(name()), blContext.ignoredSourceFormat()), + lookup, + dims + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, null) { + private SourceValueFetcher sourceValueFetcher( + Set sourcePaths, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) { + return new SourceValueFetcher(sourcePaths, null, ignoredSourceFormat) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 6919bd725133c..efbf41cca466f 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -446,6 +446,10 @@ public boolean isSourceSynthetic() { return mappingLookup.isSourceSynthetic(); } + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + return mappingLookup.ignoredSourceFormat(); + } + /** * Build something to load source {@code _source}. */ diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index f494fdd0acfab..99f508a99ce50 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -283,7 +283,10 @@ private static Fields generateTermVectors( Source s = Source.fromMap(source, XContentType.JSON); for (String field : fields) { if (values.containsKey(field) == false) { - SourceValueFetcher valueFetcher = SourceValueFetcher.toString(mappingLookup.sourcePaths(field)); + SourceValueFetcher valueFetcher = SourceValueFetcher.toString( + mappingLookup.sourcePaths(field), + mappingLookup.ignoredSourceFormat() + ); List ignoredValues = new ArrayList<>(); List v = valueFetcher.fetchValues(s, -1, ignoredValues); if (v.isEmpty() == false) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java index ce52fabb20049..384c98390e612 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java @@ -10,6 +10,7 @@ package org.elasticsearch.search.fetch; import org.elasticsearch.index.mapper.IgnoredFieldsSpec; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.IgnoredSourceFormat; import java.util.Collection; import java.util.HashSet; @@ -25,10 +26,11 @@ public record StoredFieldsSpec( boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields, - IgnoredFieldsSpec ignoredFieldsSpec + IgnoredFieldsSpec ignoredFieldsSpec, + Set sourcePaths ) { public StoredFieldsSpec(boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields) { - this(requiresSource, requiresMetadata, requiredStoredFields, IgnoredFieldsSpec.NONE); + this(requiresSource, requiresMetadata, requiredStoredFields, IgnoredFieldsSpec.NONE, null); } public boolean noRequirements() { @@ -52,6 +54,14 @@ public boolean onlyRequiresIgnoredFields() { */ public static final StoredFieldsSpec NEEDS_SOURCE = new StoredFieldsSpec(true, false, Set.of()); + public static StoredFieldsSpec withSourcePaths(IgnoredSourceFormat ignoredSourceFormat, Set sourcePaths) { + // The fields in source paths might also be in ignored source, so include source paths there as well. + IgnoredFieldsSpec ignoredFieldsSpec = ignoredSourceFormat == IgnoredSourceFormat.NO_IGNORED_SOURCE + ? new IgnoredFieldsSpec(sourcePaths, ignoredSourceFormat) + : IgnoredFieldsSpec.NONE; + return new StoredFieldsSpec(true, false, Set.of(), ignoredFieldsSpec, sourcePaths); + } + /** * Combine these stored field requirements with those from another StoredFieldsSpec */ @@ -70,11 +80,23 @@ public StoredFieldsSpec merge(StoredFieldsSpec other) { mergedFields = new HashSet<>(this.requiredStoredFields); mergedFields.addAll(other.requiredStoredFields); } + Set mergedSourcePaths; + if (this.sourcePaths != null && other.sourcePaths != null) { + mergedSourcePaths = new HashSet<>(this.sourcePaths); + mergedSourcePaths.addAll(other.sourcePaths); + } else if (this.sourcePaths != null) { + mergedSourcePaths = this.sourcePaths; + } else if (other.sourcePaths != null) { + mergedSourcePaths = other.sourcePaths; + } else { + mergedSourcePaths = null; + } return new StoredFieldsSpec( this.requiresSource || other.requiresSource, this.requiresMetadata || other.requiresMetadata, mergedFields, - ignoredFieldsSpec.merge(other.ignoredFieldsSpec) + ignoredFieldsSpec.merge(other.ignoredFieldsSpec), + mergedSourcePaths ); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java index 96def8dae1423..fa14f89ee8527 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java @@ -19,7 +19,9 @@ import org.elasticsearch.search.lookup.Source; import org.elasticsearch.search.lookup.SourceFilter; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; public final class FetchSourcePhase implements FetchSubPhase { @@ -42,7 +44,12 @@ public void setNextReader(LeafReaderContext readerContext) { @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + if (sourceFilter != null) { + var ignoredSourceFormat = fetchContext.getSearchExecutionContext().ignoredSourceFormat(); + return StoredFieldsSpec.withSourcePaths(ignoredSourceFormat, new HashSet<>(Arrays.asList(sourceFilter.getIncludes()))); + } else { + return StoredFieldsSpec.NEEDS_SOURCE; + } } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java index 8e3813f8f76ba..4c18d263def43 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java @@ -147,7 +147,11 @@ private static Map buildFieldContexts( scope, unmappedFetchPatterns ); - NestedValueFetcher nvf = new NestedValueFetcher(scope, new FieldFetcher(scopedFields, unmappedFieldFetcher)); + NestedValueFetcher nvf = new NestedValueFetcher( + scope, + new FieldFetcher(scopedFields, unmappedFieldFetcher), + context.ignoredSourceFormat() + ); output.put(scope, new FieldContext(scope, nvf)); } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/FieldValues.java b/server/src/main/java/org/elasticsearch/search/lookup/FieldValues.java index ceed4f48e7711..7f6594aa14951 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/FieldValues.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/FieldValues.java @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -33,6 +34,8 @@ public interface FieldValues { */ void valuesForDoc(SearchLookup lookup, LeafReaderContext ctx, int doc, Consumer consumer); + String name(); + /** * Creates a {@link ValueFetcher} that fetches values from a {@link FieldValues} instance * @param fieldValues the source of the values @@ -72,7 +75,7 @@ public List fetchValues(Source lookup, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; // TODO can we get more information from the script + return StoredFieldsSpec.withSourcePaths(context.ignoredSourceFormat(), Set.of(fieldValues.name())); } }; } @@ -110,7 +113,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; // TODO can we get more info from the script? + return StoredFieldsSpec.withSourcePaths(context.ignoredSourceFormat(), Set.of(fieldValues.name())); } }; } diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java index b6ca12368f762..6b819d97aa9fc 100644 --- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.mapper.IdLoader; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.query.SearchExecutionContext; @@ -78,6 +79,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; @@ -864,7 +866,10 @@ public void process(FetchSubPhase.HitContext hitContext) throws IOException { @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths( + searchContext.getSearchExecutionContext().ignoredSourceFormat(), + Set.of("thefield") + ); } })); fetchPhase.execute(searchContext, IntStream.range(0, 100).toArray(), null); @@ -918,7 +923,11 @@ public void process(FetchSubPhase.HitContext hitContext) throws IOException { @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + boolean sourceSynthetic = fetchContext.getSearchExecutionContext().isSourceSynthetic(); + return StoredFieldsSpec.withSourcePaths( + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + Set.of("thefield") + ); } })); FetchPhaseExecutionException fetchPhaseExecutionException = assertThrows( diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java index e460b5a460194..46d12f2a65e8d 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java @@ -39,7 +39,8 @@ public void testSupports() { false, false, Set.of(), - new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ) ) ); @@ -50,7 +51,8 @@ public void testSupports() { false, false, Set.of(), - new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ) ) ); @@ -61,7 +63,8 @@ public void testSupports() { true, false, Set.of(), - new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ) ) ); @@ -129,7 +132,8 @@ private void testLoader( false, false, Set.of(), - new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) + new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + null ); assertTrue(IgnoredSourceFieldLoader.supports(spec)); iw.addDocument(doc); diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java index 0255506816799..bff84ec345522 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java @@ -48,7 +48,7 @@ private StoredFieldsSpec fieldsSpec( Set ignoredFields, IgnoredSourceFieldMapper.IgnoredSourceFormat format ) { - return new StoredFieldsSpec(false, false, storedFields, new IgnoredFieldsSpec(ignoredFields, format)); + return new StoredFieldsSpec(false, false, storedFields, new IgnoredFieldsSpec(ignoredFields, format), null); } public void testEmpty() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index 1fa9c85a5c738..21ed4d029ea02 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -49,12 +49,16 @@ public void testEmptyArray() throws IOException { } private void loadBlock(LeafReaderContext ctx, Consumer test) throws IOException { - ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field")); + boolean syntheticSourceEnabled = randomBoolean(); + IgnoredSourceFieldMapper.IgnoredSourceFormat format = syntheticSourceEnabled + ? IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE + : IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field"), format); BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field"); BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup); assertThat(loader.columnAtATimeReader(ctx), nullValue()); BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx); - assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.NEEDS_SOURCE)); + assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.withSourcePaths(format, Set.of("field")))); BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null), loader.rowStrideStoredFieldSpec().requiresSource() ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) : null diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RuntimeFieldSourceProviderOptimizationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RuntimeFieldSourceProviderOptimizationTests.java index 95da43177406c..d32dcac82dfc8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RuntimeFieldSourceProviderOptimizationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RuntimeFieldSourceProviderOptimizationTests.java @@ -223,6 +223,11 @@ public String parentField(String field) { public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { return FieldNamesFieldMapper.FieldNamesFieldType.get(true); } + + @Override + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + throw new UnsupportedOperationException(); + } }; } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 5f26b046e0c8e..ddd913bae2bd7 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import static java.util.Collections.emptyMap; @@ -1586,8 +1587,9 @@ public void testFetchMetadataFieldWithSourceDisabled() throws IOException { public void testStoredFieldsSpec() throws IOException { List fields = List.of(new FieldAndFormat("field", null)); - FieldFetcher fieldFetcher = FieldFetcher.create(newSearchExecutionContext(createMapperService()), fields); - assertEquals(StoredFieldsSpec.NEEDS_SOURCE, fieldFetcher.storedFieldsSpec()); + SearchExecutionContext context = newSearchExecutionContext(createMapperService()); + FieldFetcher fieldFetcher = FieldFetcher.create(context, fields); + assertEquals(StoredFieldsSpec.withSourcePaths(context.ignoredSourceFormat(), Set.of("field")), fieldFetcher.storedFieldsSpec()); } private List fieldAndFormatList(String name, String format, boolean includeUnmapped) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index eb68672c065d1..77c3b69da5737 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -575,6 +575,11 @@ public String parentField(String field) { public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { return FieldNamesFieldMapper.FieldNamesFieldType.get(true); } + + @Override + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + throw new UnsupportedOperationException(); + } }; } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java index 7a82e30ebc04c..8efb735b3a17b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java @@ -180,6 +180,11 @@ public String parentField(String field) { public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { return (FieldNamesFieldMapper.FieldNamesFieldType) mapperService.fieldType(FieldNamesFieldMapper.NAME); } + + @Override + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + return mapperService.mappingLookup().ignoredSourceFormat(); + } }); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java index d20a002407be6..965d1a5037205 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.Set; /** * Context of each shard we're operating against. @@ -48,7 +49,7 @@ public interface ShardContext extends RefCounted { /** * Build something to load source {@code _source}. */ - SourceLoader newSourceLoader(); + SourceLoader newSourceLoader(Set sourcePaths); /** * Returns something to load values from this field into a {@link Block}. diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java index 6f00e97a1f9f2..8acc4d2a4cc93 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java @@ -181,7 +181,7 @@ private void fieldsMoved(LeafReaderContext ctx, int shard) throws IOException { } SourceLoader sourceLoader = null; if (storedFieldsSpec.requiresSource()) { - sourceLoader = operator.shardContexts.get(shard).newSourceLoader().get(); + sourceLoader = operator.shardContexts.get(shard).newSourceLoader().apply(storedFieldsSpec.sourcePaths()); storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } storedFields = new BlockLoaderStoredFieldsFromLeafLoader( diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java index 59e54103eb310..6203842dc9e0d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java @@ -139,7 +139,7 @@ private void loadFromRowStrideReaders( SourceLoader sourceLoader = null; ValuesSourceReaderOperator.ShardContext shardContext = operator.shardContexts.get(shard); if (storedFieldsSpec.requiresSource()) { - sourceLoader = shardContext.newSourceLoader().get(); + sourceLoader = shardContext.newSourceLoader().apply(storedFieldsSpec.sourcePaths()); storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } if (storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java index 3f3d73eec88ae..0a230f1bbe609 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java @@ -28,9 +28,10 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; +import java.util.function.Function; import java.util.function.IntFunction; -import java.util.function.Supplier; /** * Operator that extracts doc_values from a Lucene index out of pages that have been produced by {@link LuceneSourceOperator} @@ -91,7 +92,11 @@ public String describe() { */ public record FieldInfo(String name, ElementType type, boolean nullsFiltered, IntFunction blockLoader) {} - public record ShardContext(IndexReader reader, Supplier newSourceLoader, double storedFieldsSequentialProportion) {} + public record ShardContext( + IndexReader reader, + Function, SourceLoader> newSourceLoader, + double storedFieldsSequentialProportion + ) {} final BlockFactory blockFactory; /** diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java index 56acaa86c299d..b98f6ce25163f 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java @@ -67,6 +67,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperServiceTestCase; @@ -190,7 +191,7 @@ public void testPushRoundToToQuery() throws IOException { f -> new BlockDocValuesReader.LongsBlockLoader("v") ) ), - List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException(); }, 0.8)), 0 @@ -553,6 +554,11 @@ public String parentField(String field) { public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { throw new UnsupportedOperationException(); } + + @Override + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + throw new UnsupportedOperationException(); + } }; } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java index 5b25fb6fb64af..505cd7c212bb9 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryEvaluatorTests.java @@ -210,7 +210,7 @@ private List runQuery(Set values, Query query, boolean shuffleDocs unused -> new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(FIELD) ) ), - List.of(new ValuesSourceReaderOperator.ShardContext(reader, () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> { throw new UnsupportedOperationException(); }, 0.2)), 0 diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java index 3249031824210..609cc5875d3ff 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @@ -456,7 +457,7 @@ public IndexSearcher searcher() { } @Override - public SourceLoader newSourceLoader() { + public SourceLoader newSourceLoader(Set sourcePaths) { return SourceLoader.FROM_STORED_SOURCE; } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java index 4e562217a00e3..a20b4a52f5d6c 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java @@ -77,6 +77,7 @@ import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; @@ -208,7 +209,7 @@ private MapperService mapperService(String indexKey) { private List initShardContexts() { return INDICES.keySet() .stream() - .map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), () -> SourceLoader.FROM_STORED_SOURCE, 0.2)) + .map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2)) .toList(); } @@ -599,6 +600,11 @@ public String parentField(String field) { public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { return FieldNamesFieldMapper.FieldNamesFieldType.get(true); } + + @Override + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + return IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } }; } @@ -1325,7 +1331,7 @@ public void testWithNulls() throws IOException { LuceneOperator.NO_LIMIT, false // no scoring ); - var vsShardContext = new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE, 0.2); + var vsShardContext = new ValuesSourceReaderOperator.ShardContext(reader(indexKey), (p) -> SourceLoader.FROM_STORED_SOURCE, 0.2); try ( Driver driver = TestDriverFactory.create( driverContext, @@ -1455,7 +1461,7 @@ public void testDescriptionOfMany() throws IOException { ValuesSourceReaderOperator.Factory factory = new ValuesSourceReaderOperator.Factory( ByteSizeValue.ofGb(1), cases.stream().map(c -> c.info).toList(), - List.of(new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE, 0.2)), + List.of(new ValuesSourceReaderOperator.ShardContext(reader(indexKey), (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2)), 0 ); assertThat(factory.describe(), equalTo("ValuesSourceReaderOperator[fields = [" + cases.size() + " fields]]")); @@ -1484,7 +1490,7 @@ public void testManyShards() throws IOException { for (int s = 0; s < shardCount; s++) { contexts.add(new LuceneSourceOperatorTests.MockShardContext(readers[s], s)); readerShardContexts.add( - new ValuesSourceReaderOperator.ShardContext(readers[s], () -> SourceLoader.FROM_STORED_SOURCE, 0.2) + new ValuesSourceReaderOperator.ShardContext(readers[s], (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, 0.2) ); } var luceneFactory = new LuceneSourceOperator.Factory( diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java index 3a83d365f6f57..be9667e9e9616 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java @@ -67,6 +67,7 @@ import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; @@ -164,7 +165,7 @@ static Operator.OperatorFactory factory(IndexReader reader, String name, Element List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -500,7 +501,7 @@ public void testManySingleDocPages() { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -601,6 +602,11 @@ public String parentField(String field) { public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { return FieldNamesFieldMapper.FieldNamesFieldType.get(true); } + + @Override + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + return IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } }; } @@ -620,7 +626,7 @@ private void loadSimpleAndAssert( List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -639,7 +645,7 @@ private void loadSimpleAndAssert( List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -738,7 +744,7 @@ private void testLoadAllStatus(boolean allInOnePage) { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -972,7 +978,7 @@ private void testLoadLong(boolean shuffle, boolean manySegments) throws IOExcept List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1641,7 +1647,7 @@ public void testNullsShared() { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1693,7 +1699,7 @@ private void testSequentialStoredFields(boolean sequential, int docCount) throws List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1728,7 +1734,7 @@ public void testDescriptionOfMany() throws IOException { List.of( new ValuesSourceReaderOperator.ShardContext( reader, - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ), @@ -1761,7 +1767,7 @@ public void testManyShards() throws IOException { readerShardContexts.add( new ValuesSourceReaderOperator.ShardContext( readers[s], - () -> SourceLoader.FROM_STORED_SOURCE, + (sourcePaths) -> SourceLoader.FROM_STORED_SOURCE, STORED_FIELDS_SEQUENTIAL_PROPORTIONS ) ); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java index 7da6db1b5235e..866d545baadb0 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -360,7 +361,7 @@ private void runLookup(List keyTypes, PopulateIndices populateIndices, ValuesSourceReaderOperator.Factory reader = new ValuesSourceReaderOperator.Factory( PlannerSettings.VALUES_LOADING_JUMBO_SIZE.getDefault(Settings.EMPTY), fieldInfos, - List.of(new ValuesSourceReaderOperator.ShardContext(searchContext.getSearchExecutionContext().getIndexReader(), () -> { + List.of(new ValuesSourceReaderOperator.ShardContext(searchContext.getSearchExecutionContext().getIndexReader(), (p) -> { throw new IllegalStateException("can't load source here"); }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))), 0 @@ -534,6 +535,11 @@ public String parentField(String field) { public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { return FieldNamesFieldMapper.FieldNamesFieldType.get(true); } + + @Override + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + throw new UnsupportedOperationException(); + } }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index cd5cfab6eb102..6a488182e84a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; @@ -59,6 +60,7 @@ import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.xpack.esql.core.expression.Attribute; @@ -453,8 +455,9 @@ public String shardIdentifier() { } @Override - public SourceLoader newSourceLoader() { - return ctx.newSourceLoader(null, false); + public SourceLoader newSourceLoader(Set sourcePaths) { + var filter = sourcePaths != null ? new SourceFilter(sourcePaths.toArray(new String[0]), null) : null; + return ctx.newSourceLoader(filter, false); } @Override @@ -524,6 +527,11 @@ public String parentField(String field) { public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { return (FieldNamesFieldMapper.FieldNamesFieldType) ctx.lookup().fieldType(FieldNamesFieldMapper.NAME); } + + @Override + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + return ctx.ignoredSourceFormat(); + } }); if (loader == null) { HeaderWarning.addWarning("Field [{}] cannot be retrieved, it is unsupported or not indexed; returning null", name); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 1a8b162eb1b46..0f8c2d05f076d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -1053,7 +1053,7 @@ private String generateInvalidQueryInferenceResultsMessage(StringBuilder baseMes @Override public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) { String name = useLegacyFormat ? name().concat(".text") : name(); - SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name)); + SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name), blContext.ignoredSourceFormat()); return new BlockSourceReader.BytesRefsBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index d5405018dd64c..4195ebb7b106e 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; @@ -268,6 +269,11 @@ public String parentField(String field) { public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { return FieldNamesFieldMapper.FieldNamesFieldType.get(true); } + + @Override + public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() { + throw new UnsupportedOperationException(); + } }); try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 74459072b8979..16613fd54ea5c 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -398,7 +398,11 @@ public Builder builder(BlockFactory factory, int expectedCount) { }; } - ValueFetcher valueFetcher = new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted) { + ValueFetcher valueFetcher = new SourceValueFetcher( + blContext.sourcePaths(name()), + nullValueFormatted, + blContext.ignoredSourceFormat() + ) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { @@ -507,11 +511,18 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext if (operation == FielddataOperation.SCRIPT) { SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); - + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; + if (isSyntheticSource) { + ignoredSourceFormat = IgnoredSourceFieldMapper.ignoredSourceFormat( + fieldDataContext.indexSettings().getIndexVersionCreated() + ); + } else { + ignoredSourceFormat = IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + } return new SourceValueFetcherSortedUnsignedLongIndexFieldData.Builder( name(), valuesSourceType, - sourceValueFetcher(sourcePaths), + sourceValueFetcher(sourcePaths, ignoredSourceFormat), searchLookup, UnsignedLongDocValuesField::new ); @@ -525,11 +536,17 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + return sourceValueFetcher( + context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet(), + context.ignoredSourceFormat() + ); } - private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { - return new SourceValueFetcher(sourcePaths, nullValueFormatted) { + private SourceValueFetcher sourceValueFetcher( + Set sourcePaths, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) { + return new SourceValueFetcher(sourcePaths, nullValueFormatted, ignoredSourceFormat) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeScriptFieldType.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeScriptFieldType.java index c11f77b8da74e..939cebf8f117a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeScriptFieldType.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeScriptFieldType.java @@ -36,6 +36,7 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; public final class GeoShapeScriptFieldType extends AbstractScriptFieldType implements GeoShapeQueryable { @@ -160,7 +161,7 @@ public List fetchValues(Source source, int doc, List ignoredValu @Override public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return StoredFieldsSpec.withSourcePaths(context.ignoredSourceFormat(), Set.of(name())); } }; } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 8e63a97678602..69f5426cee24f 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -77,6 +77,7 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; +import java.util.function.Consumer; import java.util.function.Function; /** @@ -163,11 +164,22 @@ private FieldValues scriptValues() { return null; } GeometryFieldScript.Factory factory = scriptCompiler.compile(this.script.get(), GeometryFieldScript.CONTEXT); - return factory == null - ? null - : (lookup, ctx, doc, consumer) -> factory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) - .newInstance(ctx) - .runForDoc(doc, consumer); + if (factory == null) { + return null; + } + return new FieldValues<>() { + @Override + public void valuesForDoc(SearchLookup lookup, LeafReaderContext ctx, int doc, Consumer consumer) { + factory.newFactory(leafName(), script.get().getParams(), lookup, OnScriptError.FAIL) + .newInstance(ctx) + .runForDoc(doc, consumer); + } + + @Override + public String name() { + return leafName(); + } + }; } @Override