Skip to content

Commit 609ee45

Browse files
jordan-powersjordan-powers
committed
Track ignored fields in BlockLoader#rowStrideStoredFieldSpec
1 parent 1075553 commit 609ee45

File tree

17 files changed

+181
-72
lines changed

17 files changed

+181
-72
lines changed

‎server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.elasticsearch.common.CheckedBiConsumer;
1717
import org.elasticsearch.common.bytes.BytesReference;
1818
import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
19+
import org.elasticsearch.index.mapper.BlockLoader;
1920
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
2021
import org.elasticsearch.search.fetch.StoredFieldsSpec;
2122

@@ -55,6 +56,27 @@ public static StoredFieldLoader fromSpec(StoredFieldsSpec spec) {
5556
return create(spec.requiresSource(), spec.requiredStoredFields());
5657
}
5758

59+
/**
60+
* Crates a new StoredFieldLaoader using a BlockLoader.FieldsSpec
61+
*/
62+
public static StoredFieldLoader fromSpec(BlockLoader.FieldsSpec spec, boolean forceSequentialReader) {
63+
if (spec.noRequirements()) {
64+
return StoredFieldLoader.empty();
65+
}
66+
67+
// TODO
68+
// if (IgnoredSourceFieldLoader.supports(spec)) {
69+
// return new IgnoredSourceFieldLoader(spec);
70+
// }
71+
72+
StoredFieldsSpec mergedSpec = spec.storedFieldsSpec().merge(spec.ignoredFieldsSpec().requiredStoredFields());
73+
if (forceSequentialReader) {
74+
return fromSpecSequential(mergedSpec);
75+
} else {
76+
return fromSpec(mergedSpec);
77+
}
78+
}
79+
5880
public static StoredFieldLoader create(boolean loadSource, Set<String> fields) {
5981
return create(loadSource, fields, false);
6082
}

‎server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
3535
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType;
3636
import org.elasticsearch.index.mapper.vectors.VectorEncoderDecoder;
37-
import org.elasticsearch.search.fetch.StoredFieldsSpec;
3837

3938
import java.io.IOException;
4039

@@ -78,8 +77,8 @@ public final RowStrideReader rowStrideReader(LeafReaderContext context) throws I
7877
}
7978

8079
@Override
81-
public final StoredFieldsSpec rowStrideStoredFieldSpec() {
82-
return StoredFieldsSpec.NO_REQUIREMENTS;
80+
public final FieldsSpec rowStrideFieldSpec() {
81+
return FieldsSpec.NO_REQUIREMENTS;
8382
}
8483

8584
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,26 @@ interface StoredFields {
8181

8282
RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException;
8383

84-
StoredFieldsSpec rowStrideStoredFieldSpec();
84+
record FieldsSpec(StoredFieldsSpec storedFieldsSpec, IgnoredFieldsSpec ignoredFieldsSpec) {
85+
public static FieldsSpec NO_REQUIREMENTS = new FieldsSpec(StoredFieldsSpec.NO_REQUIREMENTS, IgnoredFieldsSpec.NONE);
86+
87+
public FieldsSpec merge(FieldsSpec other) {
88+
return new FieldsSpec(
89+
this.storedFieldsSpec.merge(other.storedFieldsSpec),
90+
this.ignoredFieldsSpec.merge(other.ignoredFieldsSpec)
91+
);
92+
}
93+
94+
public FieldsSpec merge(StoredFieldsSpec other) {
95+
return new FieldsSpec(this.storedFieldsSpec.merge(other), this.ignoredFieldsSpec);
96+
}
97+
98+
public boolean noRequirements() {
99+
return storedFieldsSpec.noRequirements() && ignoredFieldsSpec.requiredIgnoredFields().isEmpty();
100+
}
101+
}
102+
103+
FieldsSpec rowStrideFieldSpec();
85104

86105
/**
87106
* Does this loader support loading bytes via calling {@link #ordinals}.
@@ -123,8 +142,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) {
123142
}
124143

125144
@Override
126-
public StoredFieldsSpec rowStrideStoredFieldSpec() {
127-
return StoredFieldsSpec.NO_REQUIREMENTS;
145+
public FieldsSpec rowStrideFieldSpec() {
146+
return FieldsSpec.NO_REQUIREMENTS;
128147
}
129148

130149
@Override
@@ -220,8 +239,8 @@ public String toString() {
220239
}
221240

222241
@Override
223-
public StoredFieldsSpec rowStrideStoredFieldSpec() {
224-
return StoredFieldsSpec.NO_REQUIREMENTS;
242+
public FieldsSpec rowStrideFieldSpec() {
243+
return FieldsSpec.NO_REQUIREMENTS;
225244
}
226245

227246
@Override
@@ -302,8 +321,8 @@ public String toString() {
302321
}
303322

304323
@Override
305-
public StoredFieldsSpec rowStrideStoredFieldSpec() {
306-
return delegate.rowStrideStoredFieldSpec();
324+
public FieldsSpec rowStrideFieldSpec() {
325+
return delegate.rowStrideFieldSpec();
307326
}
308327

309328
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,8 +106,8 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context)
106106
}
107107

108108
@Override
109-
public final StoredFieldsSpec rowStrideStoredFieldSpec() {
110-
return StoredFieldsSpec.NEEDS_SOURCE;
109+
public final FieldsSpec rowStrideFieldSpec() {
110+
return new FieldsSpec(StoredFieldsSpec.NEEDS_SOURCE, IgnoredFieldsSpec.NONE);
111111
}
112112

113113
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context)
4848
}
4949

5050
@Override
51-
public final StoredFieldsSpec rowStrideStoredFieldSpec() {
52-
return new StoredFieldsSpec(false, false, Set.of(field));
51+
public final FieldsSpec rowStrideFieldSpec() {
52+
return new FieldsSpec(new StoredFieldsSpec(false, false, Set.of(field)), IgnoredFieldsSpec.NONE);
5353
}
5454

5555
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import java.util.Optional;
2424
import java.util.Set;
2525
import java.util.Stack;
26-
import java.util.stream.Collectors;
2726

2827
/**
2928
* Block loader for fields that use fallback synthetic source implementation.
@@ -65,15 +64,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep
6564
}
6665

6766
@Override
68-
public StoredFieldsSpec rowStrideStoredFieldSpec() {
69-
Set<String> ignoredFieldNames;
70-
if (ignoredSourceFormat == IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) {
71-
ignoredFieldNames = fieldPaths.stream().map(IgnoredSourceFieldMapper::ignoredFieldName).collect(Collectors.toSet());
72-
} else {
73-
ignoredFieldNames = Set.of(IgnoredSourceFieldMapper.NAME);
74-
}
75-
76-
return new StoredFieldsSpec(false, false, ignoredFieldNames);
67+
public FieldsSpec rowStrideFieldSpec() {
68+
return new FieldsSpec(StoredFieldsSpec.NO_REQUIREMENTS, new IgnoredFieldsSpec(Set.of(fieldName), ignoredSourceFormat));
7769
}
7870

7971
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
import org.elasticsearch.ElasticsearchException;
13+
import org.elasticsearch.search.fetch.StoredFieldsSpec;
14+
15+
import java.util.HashSet;
16+
import java.util.Set;
17+
import java.util.stream.Collectors;
18+
19+
/**
20+
* Defines which fields need to be loaded from _ignored_source during a fetch
21+
*/
22+
public record IgnoredFieldsSpec(Set<String> requiredIgnoredFields, IgnoredSourceFieldMapper.IgnoredSourceFormat format) {
23+
public static IgnoredFieldsSpec NONE = new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE);
24+
25+
public IgnoredFieldsSpec merge(IgnoredFieldsSpec other) {
26+
if (this.format == IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE) {
27+
return other;
28+
}
29+
if (other.format == IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE) {
30+
return this;
31+
}
32+
if (other.requiredIgnoredFields.isEmpty()) {
33+
return this;
34+
}
35+
36+
if (this.format != other.format) {
37+
throw new ElasticsearchException(
38+
"failed to merge IgnoredFieldsSpec with differing formats " + this.format.name() + "," + other.format.name()
39+
);
40+
}
41+
42+
Set<String> mergedFields = new HashSet<>(requiredIgnoredFields);
43+
mergedFields.addAll(other.requiredIgnoredFields);
44+
return new IgnoredFieldsSpec(mergedFields, format);
45+
}
46+
47+
public StoredFieldsSpec requiredStoredFields() {
48+
return new StoredFieldsSpec(
49+
false,
50+
false,
51+
requiredIgnoredFields.stream().flatMap(field -> format.requiredStoredFields(field).stream()).collect(Collectors.toSet())
52+
);
53+
54+
}
55+
}

‎server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import java.util.List;
4242
import java.util.Map;
4343
import java.util.Set;
44+
import java.util.stream.Collectors;
4445
import java.util.stream.Stream;
4546

4647
/**
@@ -277,6 +278,11 @@ public Map<String, List<IgnoredSourceFieldMapper.NameValue>> loadSingleIgnoredFi
277278
public void writeIgnoredFields(Collection<NameValue> ignoredFieldValues) {
278279
assert false : "cannot write " + ignoredFieldValues.size() + " values with format NO_IGNORED_SOURCE";
279280
}
281+
282+
@Override
283+
public Set<String> requiredStoredFields(String fieldName) {
284+
return Set.of();
285+
}
280286
},
281287
SINGLE_IGNORED_SOURCE {
282288
@Override
@@ -327,6 +333,11 @@ public void writeIgnoredFields(Collection<NameValue> ignoredFieldValues) {
327333
nameValue.doc().add(new StoredField(NAME, encode(nameValue)));
328334
}
329335
}
336+
337+
@Override
338+
public Set<String> requiredStoredFields(String fieldName) {
339+
return Set.of(IgnoredSourceFieldMapper.NAME);
340+
}
330341
},
331342
PER_FIELD_IGNORED_SOURCE {
332343
@Override
@@ -403,6 +414,14 @@ public void writeIgnoredFields(Collection<NameValue> ignoredFieldValues) {
403414
}
404415
}
405416
}
417+
418+
@Override
419+
public Set<String> requiredStoredFields(String fieldName) {
420+
return FallbackSyntheticSourceBlockLoader.splitIntoFieldPaths(fieldName)
421+
.stream()
422+
.map(IgnoredSourceFieldMapper::ignoredFieldName)
423+
.collect(Collectors.toSet());
424+
}
406425
};
407426

408427
public abstract Map<String, List<IgnoredSourceFieldMapper.NameValue>> loadAllIgnoredFields(
@@ -416,6 +435,11 @@ public abstract Map<String, List<IgnoredSourceFieldMapper.NameValue>> loadSingle
416435
);
417436

418437
public abstract void writeIgnoredFields(Collection<NameValue> ignoredFieldValues);
438+
439+
/**
440+
* Get the set of stored fields needed to retrieve the value for fieldName
441+
*/
442+
public abstract Set<String> requiredStoredFields(String fieldName);
419443
}
420444

421445
public IgnoredSourceFormat ignoredSourceFormat() {

‎server/src/main/java/org/elasticsearch/index/mapper/SourceFieldBlockLoader.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.elasticsearch.search.fetch.StoredFieldsSpec;
1515

1616
import java.io.IOException;
17-
import java.util.Set;
1817

1918
/**
2019
* Load {@code _source} into blocks.
@@ -36,8 +35,8 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep
3635
}
3736

3837
@Override
39-
public StoredFieldsSpec rowStrideStoredFieldSpec() {
40-
return new StoredFieldsSpec(true, false, Set.of());
38+
public FieldsSpec rowStrideFieldSpec() {
39+
return new FieldsSpec(StoredFieldsSpec.NEEDS_SOURCE, IgnoredFieldsSpec.NONE);
4140
}
4241

4342
@Override

‎server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,12 @@ private void loadBlock(LeafReaderContext ctx, Consumer<TestBlock> test) throws I
5454
BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup);
5555
assertThat(loader.columnAtATimeReader(ctx), nullValue());
5656
BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx);
57-
assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.NEEDS_SOURCE));
57+
assertThat(loader.rowStrideFieldSpec(), equalTo(new BlockLoader.FieldsSpec(StoredFieldsSpec.NEEDS_SOURCE, IgnoredFieldsSpec.NONE)));
5858
BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader(
59-
StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null),
60-
loader.rowStrideStoredFieldSpec().requiresSource() ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) : null
59+
StoredFieldLoader.fromSpec(loader.rowStrideFieldSpec(), false).getLoader(ctx, null),
60+
loader.rowStrideFieldSpec().storedFieldsSpec().requiresSource()
61+
? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null)
62+
: null
6163
);
6264
BlockLoader.Builder builder = loader.builder(TestBlock.factory(), 1);
6365
storedFields.advanceTo(0);

0 commit comments

Comments
 (0)