From 609ee45b8a7656129db3601b794be1c54378659c Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Mon, 4 Aug 2025 15:55:15 -0700 Subject: [PATCH 1/7] Track ignored fields in BlockLoader#rowStrideStoredFieldSpec --- .../index/fieldvisitor/StoredFieldLoader.java | 22 ++++++++ .../index/mapper/BlockDocValuesReader.java | 5 +- .../index/mapper/BlockLoader.java | 33 ++++++++--- .../index/mapper/BlockSourceReader.java | 4 +- .../index/mapper/BlockStoredFieldsReader.java | 4 +- .../FallbackSyntheticSourceBlockLoader.java | 12 +--- .../index/mapper/IgnoredFieldsSpec.java | 55 +++++++++++++++++++ .../mapper/IgnoredSourceFieldMapper.java | 24 ++++++++ .../index/mapper/SourceFieldBlockLoader.java | 5 +- .../index/mapper/BlockSourceReaderTests.java | 8 ++- .../index/mapper/BlockLoaderTestRunner.java | 10 ++-- .../read/TimeSeriesExtractFieldOperator.java | 18 +++--- .../lucene/read/ValuesFromManyReader.java | 14 ++--- .../lucene/read/ValuesFromSingleReader.java | 22 ++++---- .../read/ValuesSourceReaderOperator.java | 7 +-- .../ValueSourceReaderTypeConversionTests.java | 5 +- .../planner/EsPhysicalOperationProviders.java | 5 +- 17 files changed, 181 insertions(+), 72 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java index 0d3c314cee352..15cfa89fe1cd4 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.search.fetch.StoredFieldsSpec; @@ -55,6 +56,27 @@ public static StoredFieldLoader fromSpec(StoredFieldsSpec spec) { return create(spec.requiresSource(), spec.requiredStoredFields()); } + /** + * Crates a new StoredFieldLaoader using a BlockLoader.FieldsSpec + */ + public static StoredFieldLoader fromSpec(BlockLoader.FieldsSpec spec, boolean forceSequentialReader) { + if (spec.noRequirements()) { + return StoredFieldLoader.empty(); + } + + // TODO + // if (IgnoredSourceFieldLoader.supports(spec)) { + // return new IgnoredSourceFieldLoader(spec); + // } + + StoredFieldsSpec mergedSpec = spec.storedFieldsSpec().merge(spec.ignoredFieldsSpec().requiredStoredFields()); + if (forceSequentialReader) { + return fromSpecSequential(mergedSpec); + } else { + return fromSpec(mergedSpec); + } + } + public static StoredFieldLoader create(boolean loadSource, Set fields) { return create(loadSource, fields, false); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index 017e713fe09fe..b84600a4d6397 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -34,7 +34,6 @@ import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; import org.elasticsearch.index.mapper.vectors.VectorEncoderDecoder; -import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -78,8 +77,8 @@ public final RowStrideReader rowStrideReader(LeafReaderContext context) throws I } @Override - public final StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.NO_REQUIREMENTS; + public final FieldsSpec rowStrideFieldSpec() { + return FieldsSpec.NO_REQUIREMENTS; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index 36c0b8bfb062f..90a4d04ee180f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -81,7 +81,26 @@ interface StoredFields { RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException; - StoredFieldsSpec rowStrideStoredFieldSpec(); + record FieldsSpec(StoredFieldsSpec storedFieldsSpec, IgnoredFieldsSpec ignoredFieldsSpec) { + public static FieldsSpec NO_REQUIREMENTS = new FieldsSpec(StoredFieldsSpec.NO_REQUIREMENTS, IgnoredFieldsSpec.NONE); + + public FieldsSpec merge(FieldsSpec other) { + return new FieldsSpec( + this.storedFieldsSpec.merge(other.storedFieldsSpec), + this.ignoredFieldsSpec.merge(other.ignoredFieldsSpec) + ); + } + + public FieldsSpec merge(StoredFieldsSpec other) { + return new FieldsSpec(this.storedFieldsSpec.merge(other), this.ignoredFieldsSpec); + } + + public boolean noRequirements() { + return storedFieldsSpec.noRequirements() && ignoredFieldsSpec.requiredIgnoredFields().isEmpty(); + } + } + + FieldsSpec rowStrideFieldSpec(); /** * Does this loader support loading bytes via calling {@link #ordinals}. @@ -123,8 +142,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) { } @Override - public StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.NO_REQUIREMENTS; + public FieldsSpec rowStrideFieldSpec() { + return FieldsSpec.NO_REQUIREMENTS; } @Override @@ -220,8 +239,8 @@ public String toString() { } @Override - public StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.NO_REQUIREMENTS; + public FieldsSpec rowStrideFieldSpec() { + return FieldsSpec.NO_REQUIREMENTS; } @Override @@ -302,8 +321,8 @@ public String toString() { } @Override - public StoredFieldsSpec rowStrideStoredFieldSpec() { - return delegate.rowStrideStoredFieldSpec(); + public FieldsSpec rowStrideFieldSpec() { + return delegate.rowStrideFieldSpec(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 9d65f0ed8ba2b..ed66dd423da89 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -106,8 +106,8 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) } @Override - public final StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + public final FieldsSpec rowStrideFieldSpec() { + return new FieldsSpec(StoredFieldsSpec.NEEDS_SOURCE, IgnoredFieldsSpec.NONE); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java index a1f5dc4381f50..457291e05216c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java @@ -48,8 +48,8 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) } @Override - public final StoredFieldsSpec rowStrideStoredFieldSpec() { - return new StoredFieldsSpec(false, false, Set.of(field)); + public final FieldsSpec rowStrideFieldSpec() { + return new FieldsSpec(new StoredFieldsSpec(false, false, Set.of(field)), IgnoredFieldsSpec.NONE); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index a05eac9a72b9f..fe30f2cfc96c2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -23,7 +23,6 @@ import java.util.Optional; import java.util.Set; import java.util.Stack; -import java.util.stream.Collectors; /** * Block loader for fields that use fallback synthetic source implementation. @@ -65,15 +64,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep } @Override - public StoredFieldsSpec rowStrideStoredFieldSpec() { - Set ignoredFieldNames; - if (ignoredSourceFormat == IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) { - ignoredFieldNames = fieldPaths.stream().map(IgnoredSourceFieldMapper::ignoredFieldName).collect(Collectors.toSet()); - } else { - ignoredFieldNames = Set.of(IgnoredSourceFieldMapper.NAME); - } - - return new StoredFieldsSpec(false, false, ignoredFieldNames); + public FieldsSpec rowStrideFieldSpec() { + return new FieldsSpec(StoredFieldsSpec.NO_REQUIREMENTS, new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat)); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java new file mode 100644 index 0000000000000..db21c1bd0acec --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.search.fetch.StoredFieldsSpec; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Defines which fields need to be loaded from _ignored_source during a fetch + */ +public record IgnoredFieldsSpec(Set requiredIgnoredFields, IgnoredSourceFieldMapper.IgnoredSourceFormat format) { + public static IgnoredFieldsSpec NONE = new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE); + + public IgnoredFieldsSpec merge(IgnoredFieldsSpec other) { + if (this.format == IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE) { + return other; + } + if (other.format == IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE) { + return this; + } + if (other.requiredIgnoredFields.isEmpty()) { + return this; + } + + if (this.format != other.format) { + throw new ElasticsearchException( + "failed to merge IgnoredFieldsSpec with differing formats " + this.format.name() + "," + other.format.name() + ); + } + + Set mergedFields = new HashSet<>(requiredIgnoredFields); + mergedFields.addAll(other.requiredIgnoredFields); + return new IgnoredFieldsSpec(mergedFields, format); + } + + public StoredFieldsSpec requiredStoredFields() { + return new StoredFieldsSpec( + false, + false, + requiredIgnoredFields.stream().flatMap(field -> format.requiredStoredFields(field).stream()).collect(Collectors.toSet()) + ); + + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 2809a5e802433..24f667d38bbc7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -277,6 +278,11 @@ public Map> loadSingleIgnoredFi public void writeIgnoredFields(Collection ignoredFieldValues) { assert false : "cannot write " + ignoredFieldValues.size() + " values with format NO_IGNORED_SOURCE"; } + + @Override + public Set requiredStoredFields(String fieldName) { + return Set.of(); + } }, SINGLE_IGNORED_SOURCE { @Override @@ -327,6 +333,11 @@ public void writeIgnoredFields(Collection ignoredFieldValues) { nameValue.doc().add(new StoredField(NAME, encode(nameValue))); } } + + @Override + public Set requiredStoredFields(String fieldName) { + return Set.of(IgnoredSourceFieldMapper.NAME); + } }, PER_FIELD_IGNORED_SOURCE { @Override @@ -403,6 +414,14 @@ public void writeIgnoredFields(Collection ignoredFieldValues) { } } } + + @Override + public Set requiredStoredFields(String fieldName) { + return FallbackSyntheticSourceBlockLoader.splitIntoFieldPaths(fieldName) + .stream() + .map(IgnoredSourceFieldMapper::ignoredFieldName) + .collect(Collectors.toSet()); + } }; public abstract Map> loadAllIgnoredFields( @@ -416,6 +435,11 @@ public abstract Map> loadSingle ); public abstract void writeIgnoredFields(Collection ignoredFieldValues); + + /** + * Get the set of stored fields needed to retrieve the value for fieldName + */ + public abstract Set requiredStoredFields(String fieldName); } public IgnoredSourceFormat ignoredSourceFormat() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldBlockLoader.java index 24bb30a3ac6e8..f160551413333 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldBlockLoader.java @@ -14,7 +14,6 @@ import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; -import java.util.Set; /** * Load {@code _source} into blocks. @@ -36,8 +35,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep } @Override - public StoredFieldsSpec rowStrideStoredFieldSpec() { - return new StoredFieldsSpec(true, false, Set.of()); + public FieldsSpec rowStrideFieldSpec() { + return new FieldsSpec(StoredFieldsSpec.NEEDS_SOURCE, IgnoredFieldsSpec.NONE); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index 1fa9c85a5c738..1f5ab7281fb6f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -54,10 +54,12 @@ private void loadBlock(LeafReaderContext ctx, Consumer test) throws I BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup); assertThat(loader.columnAtATimeReader(ctx), nullValue()); BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx); - assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.NEEDS_SOURCE)); + assertThat(loader.rowStrideFieldSpec(), equalTo(new BlockLoader.FieldsSpec(StoredFieldsSpec.NEEDS_SOURCE, IgnoredFieldsSpec.NONE))); BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( - StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null), - loader.rowStrideStoredFieldSpec().requiresSource() ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) : null + StoredFieldLoader.fromSpec(loader.rowStrideFieldSpec(), false).getLoader(ctx, null), + loader.rowStrideFieldSpec().storedFieldsSpec().requiresSource() + ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) + : null ); BlockLoader.Builder builder = loader.builder(TestBlock.factory(), 1); storedFields.advanceTo(0); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java index eeb1a349d8bbc..5e4aee8727e72 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java @@ -118,17 +118,17 @@ private Object load(BlockLoader blockLoader, LeafReaderContext context, MapperSe return block.get(0); } - StoredFieldsSpec storedFieldsSpec = blockLoader.rowStrideStoredFieldSpec(); + BlockLoader.FieldsSpec fieldsSpec = blockLoader.rowStrideFieldSpec(); SourceLoader.Leaf leafSourceLoader = null; - if (storedFieldsSpec.requiresSource()) { + if (fieldsSpec.storedFieldsSpec().requiresSource()) { var sourceLoader = mapperService.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP); leafSourceLoader = sourceLoader.leaf(context.reader(), null); - storedFieldsSpec = storedFieldsSpec.merge( - new StoredFieldsSpec(true, storedFieldsSpec.requiresMetadata(), sourceLoader.requiredStoredFields()) + fieldsSpec = fieldsSpec.merge( + new StoredFieldsSpec(true, fieldsSpec.storedFieldsSpec().requiresMetadata(), sourceLoader.requiredStoredFields()) ); } BlockLoaderStoredFieldsFromLeafLoader storedFieldsLoader = new BlockLoaderStoredFieldsFromLeafLoader( - StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(context, null), + StoredFieldLoader.fromSpec(fieldsSpec, false).getLoader(context, null), leafSourceLoader ); storedFieldsLoader.advanceTo(1); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/TimeSeriesExtractFieldOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/TimeSeriesExtractFieldOperator.java index e197861e9b701..193892c3710e3 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/TimeSeriesExtractFieldOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/TimeSeriesExtractFieldOperator.java @@ -220,7 +220,7 @@ static final class ShardLevelFieldsReader implements Releasable { private final BlockLoader[] loaders; private final boolean[] dimensions; private final Block.Builder[] builders; - private final StoredFieldsSpec storedFieldsSpec; + private final BlockLoader.FieldsSpec fieldsSpec; private final SourceLoader sourceLoader; ShardLevelFieldsReader( @@ -235,23 +235,23 @@ static final class ShardLevelFieldsReader implements Releasable { this.segments = new SegmentLevelFieldsReader[indexReader.leaves().size()]; this.loaders = new BlockLoader[fields.size()]; this.builders = new Block.Builder[loaders.length]; - StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; + BlockLoader.FieldsSpec fieldsSpec = BlockLoader.FieldsSpec.NO_REQUIREMENTS; for (int i = 0; i < fields.size(); i++) { BlockLoader loader = fields.get(i).blockLoader().apply(shardIndex); - storedFieldsSpec = storedFieldsSpec.merge(loader.rowStrideStoredFieldSpec()); + fieldsSpec = fieldsSpec.merge(loader.rowStrideFieldSpec()); loaders[i] = loader; } for (int i = 0; i < indexReader.leaves().size(); i++) { LeafReaderContext leafReaderContext = indexReader.leaves().get(i); segments[i] = new SegmentLevelFieldsReader(leafReaderContext, loaders); } - if (storedFieldsSpec.requiresSource()) { + if (fieldsSpec.storedFieldsSpec().requiresSource()) { sourceLoader = shardContext.newSourceLoader(); - storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(false, false, sourceLoader.requiredStoredFields())); + fieldsSpec = fieldsSpec.merge(new StoredFieldsSpec(false, false, sourceLoader.requiredStoredFields())); } else { sourceLoader = null; } - this.storedFieldsSpec = storedFieldsSpec; + this.fieldsSpec = fieldsSpec; this.dimensions = new boolean[fields.size()]; for (int i = 0; i < fields.size(); i++) { final var mappedFieldType = shardContext.fieldType(fields.get(i).name()); @@ -274,7 +274,7 @@ void prepareForReading(int estimatedSize) throws IOException { } } for (SegmentLevelFieldsReader segment : segments) { - segment.reinitializeIfNeeded(sourceLoader, storedFieldsSpec); + segment.reinitializeIfNeeded(sourceLoader, fieldsSpec); } } @@ -329,7 +329,7 @@ static final class SegmentLevelFieldsReader { this.rowStride = new BlockLoader.RowStrideReader[loaders.length]; } - private void reinitializeIfNeeded(SourceLoader sourceLoader, StoredFieldsSpec storedFieldsSpec) throws IOException { + private void reinitializeIfNeeded(SourceLoader sourceLoader, BlockLoader.FieldsSpec fieldsSpec) throws IOException { final Thread currentThread = Thread.currentThread(); if (loadedThread != currentThread) { loadedThread = currentThread; @@ -337,7 +337,7 @@ private void reinitializeIfNeeded(SourceLoader sourceLoader, StoredFieldsSpec st rowStride[f] = loaders[f].rowStrideReader(leafContext); } storedFields = new BlockLoaderStoredFieldsFromLeafLoader( - StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(leafContext, null), + StoredFieldLoader.fromSpec(fieldsSpec, false).getLoader(leafContext, null), sourceLoader != null ? sourceLoader.leaf(leafContext.reader(), null) : null ); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java index 6f00e97a1f9f2..edbc51f1fd35c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java @@ -173,23 +173,23 @@ private long estimatedRamBytesUsed() { } private void fieldsMoved(LeafReaderContext ctx, int shard) throws IOException { - StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; + BlockLoader.FieldsSpec fieldsSpec = BlockLoader.FieldsSpec.NO_REQUIREMENTS; for (int f = 0; f < operator.fields.length; f++) { ValuesSourceReaderOperator.FieldWork field = operator.fields[f]; rowStride[f] = field.rowStride(ctx); - storedFieldsSpec = storedFieldsSpec.merge(field.loader.rowStrideStoredFieldSpec()); + fieldsSpec = fieldsSpec.merge(field.loader.rowStrideFieldSpec()); } SourceLoader sourceLoader = null; - if (storedFieldsSpec.requiresSource()) { + if (fieldsSpec.storedFieldsSpec().requiresSource()) { sourceLoader = operator.shardContexts.get(shard).newSourceLoader().get(); - storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); + fieldsSpec = fieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } storedFields = new BlockLoaderStoredFieldsFromLeafLoader( - StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(ctx, null), + StoredFieldLoader.fromSpec(fieldsSpec, false).getLoader(ctx, null), sourceLoader != null ? sourceLoader.leaf(ctx.reader(), null) : null ); - if (false == storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { - operator.trackStoredFields(storedFieldsSpec, false); + if (false == fieldsSpec.equals(BlockLoader.FieldsSpec.NO_REQUIREMENTS)) { + operator.trackStoredFields(fieldsSpec, false); } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java index de7136c2fade3..81dac376b29a1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java @@ -84,7 +84,7 @@ protected void load(Block[] target, int offset) throws IOException { private void loadFromSingleLeaf(long jumboBytes, Block[] target, ValuesReaderDocs docs, int offset) throws IOException { int firstDoc = docs.get(offset); operator.positionFieldWork(shard, segment, firstDoc); - StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; + BlockLoader.FieldsSpec storedFieldsSpec = BlockLoader.FieldsSpec.NO_REQUIREMENTS; LeafReaderContext ctx = operator.ctx(shard, segment); List columnAtATimeReaders = new ArrayList<>(operator.fields.length); @@ -104,7 +104,7 @@ private void loadFromSingleLeaf(long jumboBytes, Block[] target, ValuesReaderDoc f ) ); - storedFieldsSpec = storedFieldsSpec.merge(field.loader.rowStrideStoredFieldSpec()); + storedFieldsSpec = storedFieldsSpec.merge(field.loader.rowStrideFieldSpec()); } } @@ -130,7 +130,7 @@ private void loadFromSingleLeaf(long jumboBytes, Block[] target, ValuesReaderDoc private void loadFromRowStrideReaders( long jumboBytes, Block[] target, - StoredFieldsSpec storedFieldsSpec, + BlockLoader.FieldsSpec fieldsSpec, List rowStrideReaders, LeafReaderContext ctx, ValuesReaderDocs docs, @@ -138,22 +138,22 @@ private void loadFromRowStrideReaders( ) throws IOException { SourceLoader sourceLoader = null; ValuesSourceReaderOperator.ShardContext shardContext = operator.shardContexts.get(shard); - if (storedFieldsSpec.requiresSource()) { + if (fieldsSpec.storedFieldsSpec().requiresSource()) { sourceLoader = shardContext.newSourceLoader().get(); - storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); + fieldsSpec = fieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } - if (storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { + if (fieldsSpec.equals(BlockLoader.FieldsSpec.NO_REQUIREMENTS)) { throw new IllegalStateException( - "found row stride readers [" + rowStrideReaders + "] without stored fields [" + storedFieldsSpec + "]" + "found row stride readers [" + rowStrideReaders + "] without stored fields [" + fieldsSpec + "]" ); } StoredFieldLoader storedFieldLoader; if (useSequentialStoredFieldsReader(docs, shardContext.storedFieldsSequentialProportion())) { - storedFieldLoader = StoredFieldLoader.fromSpecSequential(storedFieldsSpec); - operator.trackStoredFields(storedFieldsSpec, true); + storedFieldLoader = StoredFieldLoader.fromSpec(fieldsSpec, true); + operator.trackStoredFields(fieldsSpec, true); } else { - storedFieldLoader = StoredFieldLoader.fromSpec(storedFieldsSpec); - operator.trackStoredFields(storedFieldsSpec, false); + storedFieldLoader = StoredFieldLoader.fromSpec(fieldsSpec, false); + operator.trackStoredFields(fieldsSpec, false); } BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( storedFieldLoader.getLoader(ctx, null), diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java index 6d0ebb9c312d0..abf9caecbf9c2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java @@ -23,7 +23,6 @@ import org.elasticsearch.core.ReleasableIterator; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.SourceLoader; -import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.List; @@ -181,13 +180,13 @@ boolean positionFieldWorkDocGuaranteedAscending(int shard, int segment) { return true; } - void trackStoredFields(StoredFieldsSpec spec, boolean sequential) { + void trackStoredFields(BlockLoader.FieldsSpec spec, boolean sequential) { readersBuilt.merge( "stored_fields[" + "requires_source:" - + spec.requiresSource() + + spec.storedFieldsSpec().requiresSource() + ", fields:" - + spec.requiredStoredFields().size() + + (spec.storedFieldsSpec().requiredStoredFields().size() + spec.ignoredFieldsSpec().requiredIgnoredFields().size()) + ", sequential: " + sequential + "]", diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java index 5a1f2ee7cc949..016b20b7792fc 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java @@ -86,7 +86,6 @@ import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.TestThreadPool; @@ -1715,8 +1714,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep } @Override - public StoredFieldsSpec rowStrideStoredFieldSpec() { - return delegate.rowStrideStoredFieldSpec(); + public FieldsSpec rowStrideFieldSpec() { + return delegate.rowStrideFieldSpec(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index e0b570267899b..b00e0e0f0aea0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -51,7 +51,6 @@ import org.elasticsearch.index.search.NestedHelper; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; -import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortAndFormats; @@ -557,8 +556,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep } @Override - public StoredFieldsSpec rowStrideStoredFieldSpec() { - return delegate.rowStrideStoredFieldSpec(); + public FieldsSpec rowStrideFieldSpec() { + return delegate.rowStrideFieldSpec(); } @Override From 0d59c1dd0b33c7a74d0340aaedb03f537fa6ab59 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Mon, 4 Aug 2025 17:36:25 -0700 Subject: [PATCH 2/7] Add IgnoredSourceFieldLoader --- .../IgnoredSourceFieldLoader.java | 136 ++++++++++++++++++ .../index/fieldvisitor/StoredFieldLoader.java | 22 +-- .../mapper/IgnoredSourceFieldMapper.java | 2 +- 3 files changed, 151 insertions(+), 9 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java new file mode 100644 index 0000000000000..946e37e357be6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.fieldvisitor; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +class IgnoredSourceFieldLoader extends StoredFieldLoader { + private final boolean forceSequentialReader; + private final Map> potentialFieldsInIgnoreSource; + private final Set fieldNames; + + IgnoredSourceFieldLoader(BlockLoader.FieldsSpec spec, boolean forceSequentialReader) { + fieldNames = new HashSet<>(spec.ignoredFieldsSpec().requiredIgnoredFields()); + this.forceSequentialReader = forceSequentialReader; + + HashMap> potentialFieldsInIgnoreSource = new HashMap<>(); + for (String requiredIgnoredField : spec.ignoredFieldsSpec().requiredIgnoredFields()) { + for (String potentialStoredField : spec.ignoredFieldsSpec().format().requiredStoredFields(requiredIgnoredField)) { + potentialFieldsInIgnoreSource.computeIfAbsent(potentialStoredField, k -> new HashSet<>()).add(requiredIgnoredField); + } + } + this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource; + } + + @Override + public LeafStoredFieldLoader getLoader(LeafReaderContext ctx, int[] docs) throws IOException { + var reader = forceSequentialReader ? sequentialReader(ctx) : reader(ctx, docs); + var visitor = new SFV(fieldNames, potentialFieldsInIgnoreSource); + return new LeafStoredFieldLoader() { + + private int doc = -1; + + @Override + public void advanceTo(int doc) throws IOException { + if (doc != this.doc) { + visitor.reset(); + reader.accept(doc, visitor); + this.doc = doc; + } + } + + @Override + public BytesReference source() { + assert false : "source() is not supported by IgnoredSourceFieldLoader"; + return null; + } + + @Override + public String id() { + assert false : "id() is not supported by IgnoredSourceFieldLoader"; + return null; + } + + @Override + public String routing() { + assert false : "routing() is not supported by IgnoredSourceFieldLoader"; + return null; + } + + @Override + public Map> storedFields() { + return visitor.values; + } + }; + } + + @Override + public List fieldsToLoad() { + return potentialFieldsInIgnoreSource.keySet().stream().toList(); + } + + static class SFV extends StoredFieldVisitor { + final Map> values = new HashMap<>(); + final Set fieldNames; + private final Set unvisitedFields; + final Map> potentialFieldsInIgnoreSource; + + SFV(Set fieldNames, Map> potentialFieldsInIgnoreSource) { + this.fieldNames = fieldNames; + this.unvisitedFields = new HashSet<>(fieldNames); + this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource; + } + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (unvisitedFields.isEmpty()) { + return Status.STOP; + } + + Set foundFields = potentialFieldsInIgnoreSource.get(fieldInfo.name); + if (foundFields == null) { + return Status.NO; + } + + unvisitedFields.removeAll(foundFields); + return Status.YES; + } + + @Override + public void binaryField(FieldInfo fieldInfo, byte[] value) { + values.computeIfAbsent(fieldInfo.name, k -> new ArrayList<>()).add(new BytesRef(value)); + } + + void reset() { + values.clear(); + unvisitedFields.addAll(fieldNames); + } + + } + + static boolean supports(BlockLoader.FieldsSpec spec) { + return spec.storedFieldsSpec().noRequirements() + && spec.ignoredFieldsSpec().format() == IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java index 15cfa89fe1cd4..41db10ffdd883 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.StoredFields; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.bytes.BytesReference; @@ -64,10 +65,9 @@ public static StoredFieldLoader fromSpec(BlockLoader.FieldsSpec spec, boolean fo return StoredFieldLoader.empty(); } - // TODO - // if (IgnoredSourceFieldLoader.supports(spec)) { - // return new IgnoredSourceFieldLoader(spec); - // } + if (IgnoredSourceFieldLoader.supports(spec)) { + return new IgnoredSourceFieldLoader(spec, forceSequentialReader); + } StoredFieldsSpec mergedSpec = spec.storedFieldsSpec().merge(spec.ignoredFieldsSpec().requiredStoredFields()); if (forceSequentialReader) { @@ -163,7 +163,8 @@ public List fieldsToLoad() { }; } - private static CheckedBiConsumer reader(LeafReaderContext ctx, int[] docs) throws IOException { + protected static CheckedBiConsumer reader(LeafReaderContext ctx, int[] docs) + throws IOException { LeafReader leafReader = ctx.reader(); if (docs != null && docs.length > 10 && hasSequentialDocs(docs)) { return sequentialReader(ctx); @@ -172,7 +173,8 @@ private static CheckedBiConsumer reader(Lea return storedFields::document; } - private static CheckedBiConsumer sequentialReader(LeafReaderContext ctx) throws IOException { + protected static CheckedBiConsumer sequentialReader(LeafReaderContext ctx) + throws IOException { LeafReader leafReader = ctx.reader(); if (leafReader instanceof SequentialStoredFieldsLeafReader lf) { return lf.getSequentialStoredFieldsReader()::document; @@ -223,7 +225,7 @@ public Map> storedFields() { private static class ReaderStoredFieldLoader implements LeafStoredFieldLoader { - private final CheckedBiConsumer reader; + private final CheckedBiConsumer reader; private final CustomFieldsVisitor visitor; private int doc = -1; @@ -243,7 +245,11 @@ public Status needsField(FieldInfo fieldInfo) { return new CustomFieldsVisitor(fields, loadSource); } - ReaderStoredFieldLoader(CheckedBiConsumer reader, boolean loadSource, Set fields) { + ReaderStoredFieldLoader( + CheckedBiConsumer reader, + boolean loadSource, + Set fields + ) { this.reader = reader; this.visitor = getFieldsVisitor(fields, loadSource); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 24f667d38bbc7..6e65978aa5bc0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -200,7 +200,7 @@ static BytesRef encodeMultipleValuesForField(List values) { } } - static List decodeMultipleValuesForField(BytesRef value) { + public static List decodeMultipleValuesForField(BytesRef value) { try { StreamInput stream = new BytesArray(value).streamInput(); var count = stream.readVInt(); From c05fdb353a284ebe07222fae50fbdc546e9caf24 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Wed, 27 Aug 2025 17:03:32 -0400 Subject: [PATCH 3/7] Add IgnoredSourceFieldLoaderTests --- .../IgnoredSourceFieldLoaderTests.java | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java new file mode 100644 index 0000000000000..ef95a349853d3 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.fieldvisitor; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.IgnoredFieldsSpec; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; + +public class IgnoredSourceFieldLoaderTests extends ESTestCase { + public void testSupports() { + assertTrue( + IgnoredSourceFieldLoader.supports( + new BlockLoader.FieldsSpec( + StoredFieldsSpec.NO_REQUIREMENTS, + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) + ) + ) + ); + + assertTrue( + IgnoredSourceFieldLoader.supports( + new BlockLoader.FieldsSpec( + StoredFieldsSpec.NO_REQUIREMENTS, + new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) + ) + ) + ); + + assertFalse( + IgnoredSourceFieldLoader.supports( + new BlockLoader.FieldsSpec( + StoredFieldsSpec.NEEDS_SOURCE, + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) + ) + ) + ); + + assertFalse( + IgnoredSourceFieldLoader.supports(new BlockLoader.FieldsSpec(StoredFieldsSpec.NO_REQUIREMENTS, IgnoredFieldsSpec.NONE)) + ); + } + + public void testLoadSingle() throws IOException { + BytesRef value = new BytesRef("lorem ipsum"); + Document doc = new Document(); + doc.add(new StoredField("_ignored_source.foo", value)); + testLoader(doc, Set.of("foo"), storedFields -> { + assertThat(storedFields, hasEntry(equalTo("_ignored_source.foo"), containsInAnyOrder(value))); + }); + } + + public void testLoaderNone() throws IOException { + testLoader(new Document(), Set.of(), storedFields -> { assertThat(storedFields, anEmptyMap()); }); + } + + public void testLoadMultiple() throws IOException { + BytesRef fooValue = new BytesRef("lorem ipsum"); + BytesRef barValue = new BytesRef("dolor sit amet"); + Document doc = new Document(); + doc.add(new StoredField("_ignored_source.foo", fooValue)); + doc.add(new StoredField("_ignored_source.bar", barValue)); + testLoader(doc, Set.of("foo", "bar"), storedFields -> { + assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(fooValue))); + assertThat(storedFields, hasEntry(equalTo("bar"), containsInAnyOrder(barValue))); + }); + } + + public void testLoadSubset() throws IOException { + BytesRef fooValue = new BytesRef("lorem ipsum"); + BytesRef barValue = new BytesRef("dolor sit amet"); + + Document doc = new Document(); + doc.add(new StoredField("_ignored_source.foo", fooValue)); + doc.add(new StoredField("_ignored_source.bar", barValue)); + + testLoader(doc, Set.of("foo"), storedFields -> { + assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(fooValue))); + assertThat(storedFields, not(hasKey("bar"))); + }); + } + + private void testLoader(Document doc, Set fieldsToLoad, Consumer>> storedFieldsConsumer) + throws IOException { + try (Directory dir = newDirectory(); IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { + BlockLoader.FieldsSpec spec = new BlockLoader.FieldsSpec( + StoredFieldsSpec.NO_REQUIREMENTS, + new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) + ); + assertTrue(IgnoredSourceFieldLoader.supports(spec)); + iw.addDocument(doc); + try (DirectoryReader reader = DirectoryReader.open(iw)) { + IgnoredSourceFieldLoader loader = new IgnoredSourceFieldLoader(spec, false); + var leafLoader = loader.getLoader(reader.leaves().getFirst(), new int[] { 0 }); + leafLoader.advanceTo(0); + } + } + } +} From ac52d06c22fc82d57df8ceda7c12481854842f97 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Wed, 27 Aug 2025 17:17:12 -0400 Subject: [PATCH 4/7] Fix typo --- .../index/fieldvisitor/StoredFieldLoader.java | 2 +- .../IgnoredSourceFieldLoaderTests.java | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java index 41db10ffdd883..ba61847071afc 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java @@ -58,7 +58,7 @@ public static StoredFieldLoader fromSpec(StoredFieldsSpec spec) { } /** - * Crates a new StoredFieldLaoader using a BlockLoader.FieldsSpec + * Crates a new StoredFieldLoader using a BlockLoader.FieldsSpec */ public static StoredFieldLoader fromSpec(BlockLoader.FieldsSpec spec, boolean forceSequentialReader) { if (spec.noRequirements()) { diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java index ef95a349853d3..324368e8a89d9 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java @@ -35,6 +35,9 @@ import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; +/** + * Test that the {@link IgnoredSourceFieldLoader} loads the correct stored values. + */ public class IgnoredSourceFieldLoaderTests extends ESTestCase { public void testSupports() { assertTrue( @@ -70,6 +73,8 @@ public void testSupports() { } public void testLoadSingle() throws IOException { + // Note: normally the stored value is encoded in the ignored source format + // (see IgnoredSourceFieldMapper#encodeMultipleValuesForField), but these tests are only verifying the loader, not the encoding. BytesRef value = new BytesRef("lorem ipsum"); Document doc = new Document(); doc.add(new StoredField("_ignored_source.foo", value)); @@ -89,8 +94,8 @@ public void testLoadMultiple() throws IOException { doc.add(new StoredField("_ignored_source.foo", fooValue)); doc.add(new StoredField("_ignored_source.bar", barValue)); testLoader(doc, Set.of("foo", "bar"), storedFields -> { - assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(fooValue))); - assertThat(storedFields, hasEntry(equalTo("bar"), containsInAnyOrder(barValue))); + assertThat(storedFields, hasEntry(equalTo("_ignored_source.foo"), containsInAnyOrder(fooValue))); + assertThat(storedFields, hasEntry(equalTo("_ignored_source.bar"), containsInAnyOrder(barValue))); }); } @@ -103,8 +108,8 @@ public void testLoadSubset() throws IOException { doc.add(new StoredField("_ignored_source.bar", barValue)); testLoader(doc, Set.of("foo"), storedFields -> { - assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(fooValue))); - assertThat(storedFields, not(hasKey("bar"))); + assertThat(storedFields, hasEntry(equalTo("_ignored_source.foo"), containsInAnyOrder(fooValue))); + assertThat(storedFields, not(hasKey("_ignored_source.bar"))); }); } @@ -121,6 +126,7 @@ private void testLoader(Document doc, Set fieldsToLoad, Consumer Date: Wed, 27 Aug 2025 18:13:41 -0400 Subject: [PATCH 5/7] Add StoredFieldLoaderTests --- .../IgnoredSourceFieldLoaderTests.java | 13 +- .../fieldvisitor/StoredFieldLoaderTests.java | 191 ++++++++++++++++++ 2 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java index 324368e8a89d9..8a41e9e6eb7df 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java @@ -113,7 +113,16 @@ public void testLoadSubset() throws IOException { }); } - private void testLoader(Document doc, Set fieldsToLoad, Consumer>> storedFieldsConsumer) + public void testLoadFromParent() throws IOException { + BytesRef fooValue = new BytesRef("lorem ipsum"); + Document doc = new Document(); + doc.add(new StoredField("_ignored_source.parent", fooValue)); + testLoader(doc, Set.of("parent.foo"), storedFields -> { + assertThat(storedFields, hasEntry(equalTo("_ignored_source.parent"), containsInAnyOrder(fooValue))); + }); + } + + private void testLoader(Document doc, Set fieldsToLoad, Consumer>> storedFieldsTest) throws IOException { try (Directory dir = newDirectory(); IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { BlockLoader.FieldsSpec spec = new BlockLoader.FieldsSpec( @@ -126,7 +135,7 @@ private void testLoader(Document doc, Set fieldsToLoad, Consumer storedFields, + Set ignoredFields, + IgnoredSourceFieldMapper.IgnoredSourceFormat format + ) { + return new BlockLoader.FieldsSpec(new StoredFieldsSpec(false, false, storedFields), new IgnoredFieldsSpec(ignoredFields, format)); + } + + public void testEmpty() throws IOException { + testStoredFieldLoader( + doc("foo", "lorem ipsum", "_ignored_source.bar", "dolor sit amet"), + fieldsSpec(Set.of(), Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + storedFields -> { + assertThat(storedFields, anEmptyMap()); + } + ); + } + + public void testSingleIgnoredSourceNewFormat() throws IOException { + testIgnoredSourceLoader( + doc("_ignored_source.foo", "lorem ipsum"), + fieldsSpec(Set.of(), Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + storedFields -> { + assertThat(storedFields, hasEntry(equalTo("_ignored_source.foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); + } + ); + } + + public void testSingleIgnoredSourceOldFormat() throws IOException { + testStoredFieldLoader( + doc("_ignored_source", "lorem ipsum"), + fieldsSpec(Set.of(), Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.SINGLE_IGNORED_SOURCE), + storedFields -> { + assertThat(storedFields, hasEntry(equalTo("_ignored_source"), containsInAnyOrder(new BytesRef("lorem ipsum")))); + } + ); + } + + public void testMultiValueIgnoredSourceNewFormat() throws IOException { + testIgnoredSourceLoader( + doc("_ignored_source.foo", "lorem ipsum", "_ignored_source.bar", "dolor sit amet"), + fieldsSpec(Set.of(), Set.of("foo", "bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + storedFields -> { + assertThat(storedFields, hasEntry(equalTo("_ignored_source.foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); + assertThat(storedFields, hasEntry(equalTo("_ignored_source.bar"), containsInAnyOrder(new BytesRef("dolor sit amet")))); + } + ); + } + + public void testMultiValueIgnoredSourceOldFormat() throws IOException { + testStoredFieldLoader( + doc("_ignored_source", "lorem ipsum", "_ignored_source", "dolor sit amet"), + fieldsSpec(Set.of(), Set.of("foo", "bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.SINGLE_IGNORED_SOURCE), + storedFields -> { + assertThat( + storedFields, + hasEntry(equalTo("_ignored_source"), containsInAnyOrder(new BytesRef("lorem ipsum"), new BytesRef("dolor sit amet"))) + ); + } + ); + } + + public void testSingleStoredField() throws IOException { + testStoredFieldLoader( + doc("foo", "lorem ipsum"), + fieldsSpec(Set.of("foo"), Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + storedFields -> { + assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); + } + ); + } + + public void testMultiValueStoredField() throws IOException { + testStoredFieldLoader( + doc("foo", "lorem ipsum", "bar", "dolor sit amet"), + fieldsSpec(Set.of("foo", "bar"), Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + storedFields -> { + assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); + assertThat(storedFields, hasEntry(equalTo("bar"), containsInAnyOrder(new BytesRef("dolor sit amet")))); + } + ); + } + + public void testMixedStoredAndIgnoredFieldsNewFormat() throws IOException { + testStoredFieldLoader( + doc("foo", "lorem ipsum", "_ignored_source.bar", "dolor sit amet"), + fieldsSpec(Set.of("foo"), Set.of("bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + storedFields -> { + assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); + assertThat(storedFields, hasEntry(equalTo("_ignored_source.bar"), containsInAnyOrder(new BytesRef("dolor sit amet")))); + } + ); + } + + public void testMixedStoredAndIgnoredFieldsOldFormat() throws IOException { + testStoredFieldLoader( + doc("foo", "lorem ipsum", "_ignored_source", "dolor sit amet"), + fieldsSpec(Set.of("foo"), Set.of("bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.SINGLE_IGNORED_SOURCE), + storedFields -> { + assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); + assertThat(storedFields, hasEntry(equalTo("_ignored_source"), containsInAnyOrder(new BytesRef("dolor sit amet")))); + } + ); + } + + public void testMixedStoredAndIgnoredFieldsLoadParent() throws IOException { + testStoredFieldLoader( + doc("foo", "lorem ipsum", "_ignored_source.parent", "dolor sit amet"), + fieldsSpec(Set.of("foo"), Set.of("parent.bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + storedFields -> { + assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); + assertThat(storedFields, hasEntry(equalTo("_ignored_source.parent"), containsInAnyOrder(new BytesRef("dolor sit amet")))); + } + ); + } + + private void testStoredFieldLoader(Document doc, BlockLoader.FieldsSpec spec, Consumer>> storedFieldsTest) + throws IOException { + testLoader(doc, spec, StoredFieldLoader.class, storedFieldsTest); + } + + private void testIgnoredSourceLoader(Document doc, BlockLoader.FieldsSpec spec, Consumer>> storedFieldsTest) + throws IOException { + testLoader(doc, spec, IgnoredSourceFieldLoader.class, storedFieldsTest); + } + + private void testLoader( + Document doc, + BlockLoader.FieldsSpec spec, + Class expectedLoaderClass, + Consumer>> storedFieldsTest + ) throws IOException { + try (Directory dir = newDirectory(); IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { + iw.addDocument(doc); + try (DirectoryReader reader = DirectoryReader.open(iw)) { + StoredFieldLoader loader = StoredFieldLoader.fromSpec(spec, false); + assertThat(loader, Matchers.isA(expectedLoaderClass)); + var leafLoader = loader.getLoader(reader.leaves().getFirst(), new int[] { 0 }); + leafLoader.advanceTo(0); + storedFieldsTest.accept(leafLoader.storedFields()); + } + } + } + +} From 9fb1125693bdd52d9a70717e7d730ceb030b2fc4 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Thu, 28 Aug 2025 12:10:24 -0400 Subject: [PATCH 6/7] Move IgnoredFieldsSpec into StoredFieldsSpec --- .../IgnoredSourceFieldLoader.java | 10 +++-- .../index/fieldvisitor/StoredFieldLoader.java | 45 ++++--------------- .../index/mapper/BlockDocValuesReader.java | 5 ++- .../index/mapper/BlockLoader.java | 33 +++----------- .../index/mapper/BlockSourceReader.java | 4 +- .../index/mapper/BlockStoredFieldsReader.java | 4 +- .../FallbackSyntheticSourceBlockLoader.java | 4 +- .../index/mapper/IgnoredFieldsSpec.java | 21 +++++---- .../index/mapper/SourceFieldBlockLoader.java | 5 ++- .../search/fetch/StoredFieldsSpec.java | 36 +++++++++++++-- .../IgnoredSourceFieldLoaderTests.java | 36 +++++++-------- .../fieldvisitor/StoredFieldLoaderTests.java | 13 +++--- .../index/mapper/BlockSourceReaderTests.java | 8 ++-- .../index/mapper/BlockLoaderTestRunner.java | 10 ++--- .../read/TimeSeriesExtractFieldOperator.java | 18 ++++---- .../lucene/read/ValuesFromManyReader.java | 14 +++--- .../lucene/read/ValuesFromSingleReader.java | 22 ++++----- .../read/ValuesSourceReaderOperator.java | 7 +-- .../ValueSourceReaderTypeConversionTests.java | 5 ++- .../planner/EsPhysicalOperationProviders.java | 5 ++- 20 files changed, 149 insertions(+), 156 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java index 946e37e357be6..0675632c0cc10 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java @@ -14,8 +14,8 @@ import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; @@ -30,7 +30,9 @@ class IgnoredSourceFieldLoader extends StoredFieldLoader { private final Map> potentialFieldsInIgnoreSource; private final Set fieldNames; - IgnoredSourceFieldLoader(BlockLoader.FieldsSpec spec, boolean forceSequentialReader) { + IgnoredSourceFieldLoader(StoredFieldsSpec spec, boolean forceSequentialReader) { + assert IgnoredSourceFieldLoader.supports(spec); + fieldNames = new HashSet<>(spec.ignoredFieldsSpec().requiredIgnoredFields()); this.forceSequentialReader = forceSequentialReader; @@ -129,8 +131,8 @@ void reset() { } - static boolean supports(BlockLoader.FieldsSpec spec) { - return spec.storedFieldsSpec().noRequirements() + static boolean supports(StoredFieldsSpec spec) { + return spec.onlyRequiresIgnoredFields() && spec.ignoredFieldsSpec().format() == IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE; } } diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java index ba61847071afc..effb3ce8f6fff 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; -import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.search.fetch.StoredFieldsSpec; @@ -51,16 +50,18 @@ public abstract class StoredFieldLoader { * Creates a new StoredFieldLoader using a StoredFieldsSpec */ public static StoredFieldLoader fromSpec(StoredFieldsSpec spec) { - if (spec.noRequirements()) { - return StoredFieldLoader.empty(); - } - return create(spec.requiresSource(), spec.requiredStoredFields()); + return fromSpec(spec, false); } /** - * Crates a new StoredFieldLoader using a BlockLoader.FieldsSpec + * Creates a new StoredFieldLoader using a StoredFieldsSpec that is optimized + * for loading documents in order. */ - public static StoredFieldLoader fromSpec(BlockLoader.FieldsSpec spec, boolean forceSequentialReader) { + public static StoredFieldLoader fromSpecSequential(StoredFieldsSpec spec) { + return fromSpec(spec, true); + } + + private static StoredFieldLoader fromSpec(StoredFieldsSpec spec, boolean forceSequentialReader) { if (spec.noRequirements()) { return StoredFieldLoader.empty(); } @@ -68,13 +69,7 @@ public static StoredFieldLoader fromSpec(BlockLoader.FieldsSpec spec, boolean fo if (IgnoredSourceFieldLoader.supports(spec)) { return new IgnoredSourceFieldLoader(spec, forceSequentialReader); } - - StoredFieldsSpec mergedSpec = spec.storedFieldsSpec().merge(spec.ignoredFieldsSpec().requiredStoredFields()); - if (forceSequentialReader) { - return fromSpecSequential(mergedSpec); - } else { - return fromSpec(mergedSpec); - } + return create(spec.requiresSource(), spec.requiredStoredFields(), forceSequentialReader); } public static StoredFieldLoader create(boolean loadSource, Set fields) { @@ -107,28 +102,6 @@ public List fieldsToLoad() { }; } - /** - * Creates a new StoredFieldLoader using a StoredFieldsSpec that is optimized - * for loading documents in order. - */ - public static StoredFieldLoader fromSpecSequential(StoredFieldsSpec spec) { - if (spec.noRequirements()) { - return StoredFieldLoader.empty(); - } - List fieldsToLoad = fieldsToLoad(spec.requiresSource(), spec.requiredStoredFields()); - return new StoredFieldLoader() { - @Override - public LeafStoredFieldLoader getLoader(LeafReaderContext ctx, int[] docs) throws IOException { - return new ReaderStoredFieldLoader(sequentialReader(ctx), spec.requiresSource(), spec.requiredStoredFields()); - } - - @Override - public List fieldsToLoad() { - return fieldsToLoad; - } - }; - } - /** * Creates a StoredFieldLoader tuned for sequential reads of _source */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index eaaf9cadcd7c6..06513444737f3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; import org.elasticsearch.index.mapper.vectors.VectorEncoderDecoder; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -75,8 +76,8 @@ public final RowStrideReader rowStrideReader(LeafReaderContext context) throws I } @Override - public final FieldsSpec rowStrideFieldSpec() { - return FieldsSpec.NO_REQUIREMENTS; + public final StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index 0c7515f1ace6a..b4fae3fd824de 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -107,26 +107,7 @@ interface StoredFields { RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException; - record FieldsSpec(StoredFieldsSpec storedFieldsSpec, IgnoredFieldsSpec ignoredFieldsSpec) { - public static FieldsSpec NO_REQUIREMENTS = new FieldsSpec(StoredFieldsSpec.NO_REQUIREMENTS, IgnoredFieldsSpec.NONE); - - public FieldsSpec merge(FieldsSpec other) { - return new FieldsSpec( - this.storedFieldsSpec.merge(other.storedFieldsSpec), - this.ignoredFieldsSpec.merge(other.ignoredFieldsSpec) - ); - } - - public FieldsSpec merge(StoredFieldsSpec other) { - return new FieldsSpec(this.storedFieldsSpec.merge(other), this.ignoredFieldsSpec); - } - - public boolean noRequirements() { - return storedFieldsSpec.noRequirements() && ignoredFieldsSpec.requiredIgnoredFields().isEmpty(); - } - } - - FieldsSpec rowStrideFieldSpec(); + StoredFieldsSpec rowStrideStoredFieldSpec(); /** * Does this loader support loading bytes via calling {@link #ordinals}. @@ -168,8 +149,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) { } @Override - public FieldsSpec rowStrideFieldSpec() { - return FieldsSpec.NO_REQUIREMENTS; + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; } @Override @@ -265,8 +246,8 @@ public String toString() { } @Override - public FieldsSpec rowStrideFieldSpec() { - return FieldsSpec.NO_REQUIREMENTS; + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; } @Override @@ -347,8 +328,8 @@ public String toString() { } @Override - public FieldsSpec rowStrideFieldSpec() { - return delegate.rowStrideFieldSpec(); + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return delegate.rowStrideStoredFieldSpec(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index ed66dd423da89..9d65f0ed8ba2b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -106,8 +106,8 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) } @Override - public final FieldsSpec rowStrideFieldSpec() { - return new FieldsSpec(StoredFieldsSpec.NEEDS_SOURCE, IgnoredFieldsSpec.NONE); + public final StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java index 457291e05216c..a1f5dc4381f50 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java @@ -48,8 +48,8 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) } @Override - public final FieldsSpec rowStrideFieldSpec() { - return new FieldsSpec(new StoredFieldsSpec(false, false, Set.of(field)), IgnoredFieldsSpec.NONE); + public final StoredFieldsSpec rowStrideStoredFieldSpec() { + return new StoredFieldsSpec(false, false, Set.of(field)); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index fe30f2cfc96c2..b66730d6704e0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -64,8 +64,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep } @Override - public FieldsSpec rowStrideFieldSpec() { - return new FieldsSpec(StoredFieldsSpec.NO_REQUIREMENTS, new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat)); + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return new StoredFieldsSpec(false, false, Set.of(), new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat)); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java index db21c1bd0acec..cdbc7621024c0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java @@ -10,18 +10,21 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; /** - * Defines which fields need to be loaded from _ignored_source during a fetch + * Defines which fields need to be loaded from _ignored_source during a fetch. */ public record IgnoredFieldsSpec(Set requiredIgnoredFields, IgnoredSourceFieldMapper.IgnoredSourceFormat format) { public static IgnoredFieldsSpec NONE = new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE); + public boolean noRequirements() { + return requiredIgnoredFields.isEmpty(); + } + public IgnoredFieldsSpec merge(IgnoredFieldsSpec other) { if (this.format == IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE) { return other; @@ -32,6 +35,9 @@ public IgnoredFieldsSpec merge(IgnoredFieldsSpec other) { if (other.requiredIgnoredFields.isEmpty()) { return this; } + if (this.requiredIgnoredFields.isEmpty()) { + return other; + } if (this.format != other.format) { throw new ElasticsearchException( @@ -44,12 +50,11 @@ public IgnoredFieldsSpec merge(IgnoredFieldsSpec other) { return new IgnoredFieldsSpec(mergedFields, format); } - public StoredFieldsSpec requiredStoredFields() { - return new StoredFieldsSpec( - false, - false, - requiredIgnoredFields.stream().flatMap(field -> format.requiredStoredFields(field).stream()).collect(Collectors.toSet()) - ); + /** + * Get the set of stored fields required to load the specified fields from _ignored_source. + */ + public Set requiredStoredFields() { + return requiredIgnoredFields.stream().flatMap(field -> format.requiredStoredFields(field).stream()).collect(Collectors.toSet()); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldBlockLoader.java index f160551413333..24bb30a3ac6e8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldBlockLoader.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; +import java.util.Set; /** * Load {@code _source} into blocks. @@ -35,8 +36,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep } @Override - public FieldsSpec rowStrideFieldSpec() { - return new FieldsSpec(StoredFieldsSpec.NEEDS_SOURCE, IgnoredFieldsSpec.NONE); + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return new StoredFieldsSpec(true, false, Set.of()); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java index 529a6c5eaac7c..ce52fabb20049 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java @@ -9,6 +9,8 @@ package org.elasticsearch.search.fetch; +import org.elasticsearch.index.mapper.IgnoredFieldsSpec; + import java.util.Collection; import java.util.HashSet; import java.util.Set; @@ -19,10 +21,25 @@ * @param requiresSource should source be loaded * @param requiredStoredFields a set of stored fields to load */ -public record StoredFieldsSpec(boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields) { +public record StoredFieldsSpec( + boolean requiresSource, + boolean requiresMetadata, + Set requiredStoredFields, + IgnoredFieldsSpec ignoredFieldsSpec +) { + public StoredFieldsSpec(boolean requiresSource, boolean requiresMetadata, Set requiredStoredFields) { + this(requiresSource, requiresMetadata, requiredStoredFields, IgnoredFieldsSpec.NONE); + } public boolean noRequirements() { - return requiresSource == false && requiresMetadata == false && requiredStoredFields.isEmpty(); + return requiresSource == false && requiresMetadata == false && requiredStoredFields.isEmpty() && ignoredFieldsSpec.noRequirements(); + } + + public boolean onlyRequiresIgnoredFields() { + return requiresSource == false + && requiresMetadata == false + && requiredStoredFields.isEmpty() + && ignoredFieldsSpec.noRequirements() == false; } /** @@ -56,10 +73,23 @@ public StoredFieldsSpec merge(StoredFieldsSpec other) { return new StoredFieldsSpec( this.requiresSource || other.requiresSource, this.requiresMetadata || other.requiresMetadata, - mergedFields + mergedFields, + ignoredFieldsSpec.merge(other.ignoredFieldsSpec) ); } + public Set requiredStoredFields() { + if (ignoredFieldsSpec.noRequirements()) { + return requiredStoredFields; + } + if (requiredStoredFields.isEmpty()) { + return ignoredFieldsSpec.requiredStoredFields(); + } + Set mergedFields = new HashSet<>(requiredStoredFields); + mergedFields.addAll(ignoredFieldsSpec.requiredStoredFields()); + return mergedFields; + } + public static StoredFieldsSpec build(Collection sources, Function converter) { StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; for (T source : sources) { diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java index 8a41e9e6eb7df..37c6f4f5658a3 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java @@ -16,7 +16,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.IgnoredFieldsSpec; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.search.fetch.StoredFieldsSpec; @@ -28,7 +27,6 @@ import java.util.Set; import java.util.function.Consumer; -import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; @@ -42,17 +40,21 @@ public class IgnoredSourceFieldLoaderTests extends ESTestCase { public void testSupports() { assertTrue( IgnoredSourceFieldLoader.supports( - new BlockLoader.FieldsSpec( - StoredFieldsSpec.NO_REQUIREMENTS, + new StoredFieldsSpec( + false, + false, + Set.of(), new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) ) ) ); - assertTrue( + assertFalse( IgnoredSourceFieldLoader.supports( - new BlockLoader.FieldsSpec( - StoredFieldsSpec.NO_REQUIREMENTS, + new StoredFieldsSpec( + false, + false, + Set.of(), new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) ) ) @@ -60,16 +62,16 @@ public void testSupports() { assertFalse( IgnoredSourceFieldLoader.supports( - new BlockLoader.FieldsSpec( - StoredFieldsSpec.NEEDS_SOURCE, + new StoredFieldsSpec( + true, + false, + Set.of(), new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) ) ) ); - assertFalse( - IgnoredSourceFieldLoader.supports(new BlockLoader.FieldsSpec(StoredFieldsSpec.NO_REQUIREMENTS, IgnoredFieldsSpec.NONE)) - ); + assertFalse(IgnoredSourceFieldLoader.supports(StoredFieldsSpec.NO_REQUIREMENTS)); } public void testLoadSingle() throws IOException { @@ -83,10 +85,6 @@ public void testLoadSingle() throws IOException { }); } - public void testLoaderNone() throws IOException { - testLoader(new Document(), Set.of(), storedFields -> { assertThat(storedFields, anEmptyMap()); }); - } - public void testLoadMultiple() throws IOException { BytesRef fooValue = new BytesRef("lorem ipsum"); BytesRef barValue = new BytesRef("dolor sit amet"); @@ -125,8 +123,10 @@ public void testLoadFromParent() throws IOException { private void testLoader(Document doc, Set fieldsToLoad, Consumer>> storedFieldsTest) throws IOException { try (Directory dir = newDirectory(); IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { - BlockLoader.FieldsSpec spec = new BlockLoader.FieldsSpec( - StoredFieldsSpec.NO_REQUIREMENTS, + StoredFieldsSpec spec = new StoredFieldsSpec( + false, + false, + Set.of(), new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) ); assertTrue(IgnoredSourceFieldLoader.supports(spec)); diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java index f952c000d0f09..1a3b62ed74be2 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java @@ -16,7 +16,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.IgnoredFieldsSpec; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.search.fetch.StoredFieldsSpec; @@ -44,12 +43,12 @@ private Document doc(String... values) { return doc; } - private BlockLoader.FieldsSpec fieldsSpec( + private StoredFieldsSpec fieldsSpec( Set storedFields, Set ignoredFields, IgnoredSourceFieldMapper.IgnoredSourceFormat format ) { - return new BlockLoader.FieldsSpec(new StoredFieldsSpec(false, false, storedFields), new IgnoredFieldsSpec(ignoredFields, format)); + return new StoredFieldsSpec(false, false, storedFields, new IgnoredFieldsSpec(ignoredFields, format)); } public void testEmpty() throws IOException { @@ -160,26 +159,26 @@ public void testMixedStoredAndIgnoredFieldsLoadParent() throws IOException { ); } - private void testStoredFieldLoader(Document doc, BlockLoader.FieldsSpec spec, Consumer>> storedFieldsTest) + private void testStoredFieldLoader(Document doc, StoredFieldsSpec spec, Consumer>> storedFieldsTest) throws IOException { testLoader(doc, spec, StoredFieldLoader.class, storedFieldsTest); } - private void testIgnoredSourceLoader(Document doc, BlockLoader.FieldsSpec spec, Consumer>> storedFieldsTest) + private void testIgnoredSourceLoader(Document doc, StoredFieldsSpec spec, Consumer>> storedFieldsTest) throws IOException { testLoader(doc, spec, IgnoredSourceFieldLoader.class, storedFieldsTest); } private void testLoader( Document doc, - BlockLoader.FieldsSpec spec, + StoredFieldsSpec spec, Class expectedLoaderClass, Consumer>> storedFieldsTest ) throws IOException { try (Directory dir = newDirectory(); IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { iw.addDocument(doc); try (DirectoryReader reader = DirectoryReader.open(iw)) { - StoredFieldLoader loader = StoredFieldLoader.fromSpec(spec, false); + StoredFieldLoader loader = StoredFieldLoader.fromSpec(spec); assertThat(loader, Matchers.isA(expectedLoaderClass)); var leafLoader = loader.getLoader(reader.leaves().getFirst(), new int[] { 0 }); leafLoader.advanceTo(0); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index 1f5ab7281fb6f..1fa9c85a5c738 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -54,12 +54,10 @@ private void loadBlock(LeafReaderContext ctx, Consumer test) throws I BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup); assertThat(loader.columnAtATimeReader(ctx), nullValue()); BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx); - assertThat(loader.rowStrideFieldSpec(), equalTo(new BlockLoader.FieldsSpec(StoredFieldsSpec.NEEDS_SOURCE, IgnoredFieldsSpec.NONE))); + assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.NEEDS_SOURCE)); BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( - StoredFieldLoader.fromSpec(loader.rowStrideFieldSpec(), false).getLoader(ctx, null), - loader.rowStrideFieldSpec().storedFieldsSpec().requiresSource() - ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) - : null + StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null), + loader.rowStrideStoredFieldSpec().requiresSource() ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) : null ); BlockLoader.Builder builder = loader.builder(TestBlock.factory(), 1); storedFields.advanceTo(0); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java index 48af9e7020bc9..3f4b863532233 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java @@ -118,17 +118,17 @@ private Object load(BlockLoader blockLoader, LeafReaderContext context, MapperSe return block.get(0); } - BlockLoader.FieldsSpec fieldsSpec = blockLoader.rowStrideFieldSpec(); + StoredFieldsSpec storedFieldsSpec = blockLoader.rowStrideStoredFieldSpec(); SourceLoader.Leaf leafSourceLoader = null; - if (fieldsSpec.storedFieldsSpec().requiresSource()) { + if (storedFieldsSpec.requiresSource()) { var sourceLoader = mapperService.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP); leafSourceLoader = sourceLoader.leaf(context.reader(), null); - fieldsSpec = fieldsSpec.merge( - new StoredFieldsSpec(true, fieldsSpec.storedFieldsSpec().requiresMetadata(), sourceLoader.requiredStoredFields()) + storedFieldsSpec = storedFieldsSpec.merge( + new StoredFieldsSpec(true, storedFieldsSpec.requiresMetadata(), sourceLoader.requiredStoredFields()) ); } BlockLoaderStoredFieldsFromLeafLoader storedFieldsLoader = new BlockLoaderStoredFieldsFromLeafLoader( - StoredFieldLoader.fromSpec(fieldsSpec, false).getLoader(context, null), + StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(context, null), leafSourceLoader ); storedFieldsLoader.advanceTo(1); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/TimeSeriesExtractFieldOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/TimeSeriesExtractFieldOperator.java index b838d9cf187e9..f6712bf651da1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/TimeSeriesExtractFieldOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/TimeSeriesExtractFieldOperator.java @@ -214,7 +214,7 @@ static final class ShardLevelFieldsReader implements Releasable { private final BlockLoader[] loaders; private final boolean[] dimensions; private final Block.Builder[] builders; - private final BlockLoader.FieldsSpec fieldsSpec; + private final StoredFieldsSpec storedFieldsSpec; private final SourceLoader sourceLoader; ShardLevelFieldsReader( @@ -229,23 +229,23 @@ static final class ShardLevelFieldsReader implements Releasable { this.segments = new SegmentLevelFieldsReader[indexReader.leaves().size()]; this.loaders = new BlockLoader[fields.size()]; this.builders = new Block.Builder[loaders.length]; - BlockLoader.FieldsSpec fieldsSpec = BlockLoader.FieldsSpec.NO_REQUIREMENTS; + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; for (int i = 0; i < fields.size(); i++) { BlockLoader loader = fields.get(i).blockLoader().apply(shardIndex); - fieldsSpec = fieldsSpec.merge(loader.rowStrideFieldSpec()); + storedFieldsSpec = storedFieldsSpec.merge(loader.rowStrideStoredFieldSpec()); loaders[i] = loader; } for (int i = 0; i < indexReader.leaves().size(); i++) { LeafReaderContext leafReaderContext = indexReader.leaves().get(i); segments[i] = new SegmentLevelFieldsReader(leafReaderContext, loaders); } - if (fieldsSpec.storedFieldsSpec().requiresSource()) { + if (storedFieldsSpec.requiresSource()) { sourceLoader = shardContext.newSourceLoader(); - fieldsSpec = fieldsSpec.merge(new StoredFieldsSpec(false, false, sourceLoader.requiredStoredFields())); + storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(false, false, sourceLoader.requiredStoredFields())); } else { sourceLoader = null; } - this.fieldsSpec = fieldsSpec; + this.storedFieldsSpec = storedFieldsSpec; this.dimensions = new boolean[fields.size()]; for (int i = 0; i < fields.size(); i++) { final var mappedFieldType = shardContext.fieldType(fields.get(i).name()); @@ -268,7 +268,7 @@ void prepareForReading(int estimatedSize) throws IOException { } } for (SegmentLevelFieldsReader segment : segments) { - segment.reinitializeIfNeeded(sourceLoader, fieldsSpec); + segment.reinitializeIfNeeded(sourceLoader, storedFieldsSpec); } } @@ -323,7 +323,7 @@ static final class SegmentLevelFieldsReader { this.rowStride = new BlockLoader.RowStrideReader[loaders.length]; } - private void reinitializeIfNeeded(SourceLoader sourceLoader, BlockLoader.FieldsSpec fieldsSpec) throws IOException { + private void reinitializeIfNeeded(SourceLoader sourceLoader, StoredFieldsSpec storedFieldsSpec) throws IOException { final Thread currentThread = Thread.currentThread(); if (loadedThread != currentThread) { loadedThread = currentThread; @@ -331,7 +331,7 @@ private void reinitializeIfNeeded(SourceLoader sourceLoader, BlockLoader.FieldsS rowStride[f] = loaders[f].rowStrideReader(leafContext); } storedFields = new BlockLoaderStoredFieldsFromLeafLoader( - StoredFieldLoader.fromSpec(fieldsSpec, false).getLoader(leafContext, null), + StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(leafContext, null), sourceLoader != null ? sourceLoader.leaf(leafContext.reader(), null) : null ); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java index edbc51f1fd35c..6f00e97a1f9f2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java @@ -173,23 +173,23 @@ private long estimatedRamBytesUsed() { } private void fieldsMoved(LeafReaderContext ctx, int shard) throws IOException { - BlockLoader.FieldsSpec fieldsSpec = BlockLoader.FieldsSpec.NO_REQUIREMENTS; + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; for (int f = 0; f < operator.fields.length; f++) { ValuesSourceReaderOperator.FieldWork field = operator.fields[f]; rowStride[f] = field.rowStride(ctx); - fieldsSpec = fieldsSpec.merge(field.loader.rowStrideFieldSpec()); + storedFieldsSpec = storedFieldsSpec.merge(field.loader.rowStrideStoredFieldSpec()); } SourceLoader sourceLoader = null; - if (fieldsSpec.storedFieldsSpec().requiresSource()) { + if (storedFieldsSpec.requiresSource()) { sourceLoader = operator.shardContexts.get(shard).newSourceLoader().get(); - fieldsSpec = fieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); + storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } storedFields = new BlockLoaderStoredFieldsFromLeafLoader( - StoredFieldLoader.fromSpec(fieldsSpec, false).getLoader(ctx, null), + StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(ctx, null), sourceLoader != null ? sourceLoader.leaf(ctx.reader(), null) : null ); - if (false == fieldsSpec.equals(BlockLoader.FieldsSpec.NO_REQUIREMENTS)) { - operator.trackStoredFields(fieldsSpec, false); + if (false == storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { + operator.trackStoredFields(storedFieldsSpec, false); } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java index cc4859ca44a4a..59e54103eb310 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java @@ -84,7 +84,7 @@ protected void load(Block[] target, int offset) throws IOException { private void loadFromSingleLeaf(long jumboBytes, Block[] target, ValuesReaderDocs docs, int offset) throws IOException { int firstDoc = docs.get(offset); operator.positionFieldWork(shard, segment, firstDoc); - BlockLoader.FieldsSpec storedFieldsSpec = BlockLoader.FieldsSpec.NO_REQUIREMENTS; + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; LeafReaderContext ctx = operator.ctx(shard, segment); List columnAtATimeReaders = new ArrayList<>(operator.fields.length); @@ -104,7 +104,7 @@ private void loadFromSingleLeaf(long jumboBytes, Block[] target, ValuesReaderDoc f ) ); - storedFieldsSpec = storedFieldsSpec.merge(field.loader.rowStrideFieldSpec()); + storedFieldsSpec = storedFieldsSpec.merge(field.loader.rowStrideStoredFieldSpec()); } } @@ -130,7 +130,7 @@ private void loadFromSingleLeaf(long jumboBytes, Block[] target, ValuesReaderDoc private void loadFromRowStrideReaders( long jumboBytes, Block[] target, - BlockLoader.FieldsSpec fieldsSpec, + StoredFieldsSpec storedFieldsSpec, List rowStrideReaders, LeafReaderContext ctx, ValuesReaderDocs docs, @@ -138,22 +138,22 @@ private void loadFromRowStrideReaders( ) throws IOException { SourceLoader sourceLoader = null; ValuesSourceReaderOperator.ShardContext shardContext = operator.shardContexts.get(shard); - if (fieldsSpec.storedFieldsSpec().requiresSource()) { + if (storedFieldsSpec.requiresSource()) { sourceLoader = shardContext.newSourceLoader().get(); - fieldsSpec = fieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); + storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(true, false, sourceLoader.requiredStoredFields())); } - if (fieldsSpec.equals(BlockLoader.FieldsSpec.NO_REQUIREMENTS)) { + if (storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { throw new IllegalStateException( - "found row stride readers [" + rowStrideReaders + "] without stored fields [" + fieldsSpec + "]" + "found row stride readers [" + rowStrideReaders + "] without stored fields [" + storedFieldsSpec + "]" ); } StoredFieldLoader storedFieldLoader; if (useSequentialStoredFieldsReader(docs, shardContext.storedFieldsSequentialProportion())) { - storedFieldLoader = StoredFieldLoader.fromSpec(fieldsSpec, true); - operator.trackStoredFields(fieldsSpec, true); + storedFieldLoader = StoredFieldLoader.fromSpecSequential(storedFieldsSpec); + operator.trackStoredFields(storedFieldsSpec, true); } else { - storedFieldLoader = StoredFieldLoader.fromSpec(fieldsSpec, false); - operator.trackStoredFields(fieldsSpec, false); + storedFieldLoader = StoredFieldLoader.fromSpec(storedFieldsSpec); + operator.trackStoredFields(storedFieldsSpec, false); } BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( storedFieldLoader.getLoader(ctx, null), diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java index 0c675a34ea005..3f3d73eec88ae 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperator.java @@ -23,6 +23,7 @@ import org.elasticsearch.core.ReleasableIterator; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.List; @@ -185,13 +186,13 @@ boolean positionFieldWorkDocGuaranteedAscending(int shard, int segment) { return true; } - void trackStoredFields(BlockLoader.FieldsSpec spec, boolean sequential) { + void trackStoredFields(StoredFieldsSpec spec, boolean sequential) { readersBuilt.merge( "stored_fields[" + "requires_source:" - + spec.storedFieldsSpec().requiresSource() + + spec.requiresSource() + ", fields:" - + (spec.storedFieldsSpec().requiredStoredFields().size() + spec.ignoredFieldsSpec().requiredIgnoredFields().size()) + + spec.requiredStoredFields().size() + ", sequential: " + sequential + "]", diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java index a8812021831ea..4e562217a00e3 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java @@ -86,6 +86,7 @@ import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.TestThreadPool; @@ -1738,8 +1739,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep } @Override - public FieldsSpec rowStrideFieldSpec() { - return delegate.rowStrideFieldSpec(); + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return delegate.rowStrideStoredFieldSpec(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 9060c9fe419f3..093dbca0ae51f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -57,6 +57,7 @@ import org.elasticsearch.index.search.NestedHelper; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortAndFormats; @@ -612,8 +613,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep } @Override - public FieldsSpec rowStrideFieldSpec() { - return delegate.rowStrideFieldSpec(); + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return delegate.rowStrideStoredFieldSpec(); } @Override From 94bc6479fa1aafe002945f89422c881b5850a959 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Thu, 28 Aug 2025 12:27:05 -0400 Subject: [PATCH 7/7] Make decodeMultipleValuesForField package-private --- .../elasticsearch/index/mapper/IgnoredSourceFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 6e65978aa5bc0..24f667d38bbc7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -200,7 +200,7 @@ static BytesRef encodeMultipleValuesForField(List values) { } } - public static List decodeMultipleValuesForField(BytesRef value) { + static List decodeMultipleValuesForField(BytesRef value) { try { StreamInput stream = new BytesArray(value).streamInput(); var count = stream.readVInt();