Skip to content

Commit d9c930d

Browse files
martijnvgmartijnvg
authored
Include ignored source as part of loading field values in ValueSourceReaderOperator via BlockSourceReader. (elastic#114903) (elastic#115064)
Currently, in compute engine when loading source if source mode is synthetic, the synthetic source loader is already used. But the ignored_source field isn't always marked as a required source field, causing the source to potentially miss a lot of fields. This change includes _ignored_source field as a required stored field and allowing keyword fields without doc values or stored fields to be used in case of synthetic source. Relying on synthetic source to get the values (because a field doesn't have stored fields / doc values) is slow. In case of synthetic source we already keep ignored field/values in a special place, named ignored source. Long term in case of synthetic source we should only load ignored source in case a field has no doc values or stored field. Like is being explored in elastic#114886 Thereby avoiding synthesizing the complete _source in order to get only one field.
1 parent 8270382 commit d9c930d

File tree

18 files changed

+613
-58
lines changed

18 files changed

+613
-58
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
364364
SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
365365
// MatchOnlyText never has norms, so we have to use the field names field
366366
BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name());
367-
return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup);
367+
var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
368+
return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup, sourceMode);
368369
}
369370

370371
@Override

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
319319
BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
320320
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
321321
: BlockSourceReader.lookupMatchingAll();
322-
return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup);
322+
var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
323+
return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup, sourceMode);
323324
}
324325

325326
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
189189
protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) {
190190
ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB);
191191
// TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
192-
return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll());
192+
var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
193+
return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), sourceMode);
193194
}
194195

195196
protected abstract Object nullValueAsSource(T nullValue);

‎server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,22 @@
2222
import java.io.IOException;
2323
import java.util.ArrayList;
2424
import java.util.List;
25+
import java.util.Set;
2526

2627
/**
2728
* Loads values from {@code _source}. This whole process is very slow and cast-tastic,
2829
* so it doesn't really try to avoid megamorphic invocations. It's just going to be
2930
* slow.
3031
*/
3132
public abstract class BlockSourceReader implements BlockLoader.RowStrideReader {
33+
34+
// _ignored_source is needed when source mode is synthetic.
35+
static final StoredFieldsSpec NEEDS_SOURCE_AND_IGNORED_SOURCE = new StoredFieldsSpec(
36+
true,
37+
false,
38+
Set.of(IgnoredSourceFieldMapper.NAME)
39+
);
40+
3241
private final ValueFetcher fetcher;
3342
private final List<Object> ignoredValues = new ArrayList<>();
3443
private final DocIdSetIterator iter;
@@ -91,10 +100,12 @@ public interface LeafIteratorLookup {
91100
private abstract static class SourceBlockLoader implements BlockLoader {
92101
protected final ValueFetcher fetcher;
93102
private final LeafIteratorLookup lookup;
103+
private final SourceFieldMapper.Mode sourceMode;
94104

95-
private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
105+
private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
96106
this.fetcher = fetcher;
97107
this.lookup = lookup;
108+
this.sourceMode = sourceMode;
98109
}
99110

100111
@Override
@@ -104,7 +115,7 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context)
104115

105116
@Override
106117
public final StoredFieldsSpec rowStrideStoredFieldSpec() {
107-
return StoredFieldsSpec.NEEDS_SOURCE;
118+
return sourceMode == SourceFieldMapper.Mode.SYNTHETIC ? NEEDS_SOURCE_AND_IGNORED_SOURCE : StoredFieldsSpec.NEEDS_SOURCE;
108119
}
109120

110121
@Override
@@ -140,8 +151,8 @@ public final String toString() {
140151
* Load {@code boolean}s from {@code _source}.
141152
*/
142153
public static class BooleansBlockLoader extends SourceBlockLoader {
143-
public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
144-
super(fetcher, lookup);
154+
public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
155+
super(fetcher, lookup, sourceMode);
145156
}
146157

147158
@Override
@@ -180,8 +191,8 @@ public String toString() {
180191
* Load {@link BytesRef}s from {@code _source}.
181192
*/
182193
public static class BytesRefsBlockLoader extends SourceBlockLoader {
183-
public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
184-
super(fetcher, lookup);
194+
public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
195+
super(fetcher, lookup, sourceMode);
185196
}
186197

187198
@Override
@@ -191,7 +202,7 @@ public final Builder builder(BlockFactory factory, int expectedCount) {
191202

192203
@Override
193204
protected RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) throws IOException {
194-
return new BytesRefs(fetcher, iter);
205+
return new BytesRefs(fetcher, iter, null);
195206
}
196207

197208
@Override
@@ -201,8 +212,8 @@ protected String name() {
201212
}
202213

203214
public static class GeometriesBlockLoader extends SourceBlockLoader {
204-
public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
205-
super(fetcher, lookup);
215+
public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
216+
super(fetcher, lookup, sourceMode);
206217
}
207218

208219
@Override
@@ -212,7 +223,7 @@ public final Builder builder(BlockFactory factory, int expectedCount) {
212223

213224
@Override
214225
protected RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) {
215-
return new Geometries(fetcher, iter);
226+
return new Geometries(fetcher, iter, null);
216227
}
217228

218229
@Override
@@ -224,7 +235,7 @@ protected String name() {
224235
private static class BytesRefs extends BlockSourceReader {
225236
private final BytesRef scratch = new BytesRef();
226237

227-
BytesRefs(ValueFetcher fetcher, DocIdSetIterator iter) {
238+
BytesRefs(ValueFetcher fetcher, DocIdSetIterator iter, SourceFieldMapper.Mode sourceMode) {
228239
super(fetcher, iter);
229240
}
230241

@@ -241,7 +252,7 @@ public String toString() {
241252

242253
private static class Geometries extends BlockSourceReader {
243254

244-
Geometries(ValueFetcher fetcher, DocIdSetIterator iter) {
255+
Geometries(ValueFetcher fetcher, DocIdSetIterator iter, SourceFieldMapper.Mode sourceMode) {
245256
super(fetcher, iter);
246257
}
247258

@@ -264,8 +275,8 @@ public String toString() {
264275
* Load {@code double}s from {@code _source}.
265276
*/
266277
public static class DoublesBlockLoader extends SourceBlockLoader {
267-
public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
268-
super(fetcher, lookup);
278+
public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
279+
super(fetcher, lookup, sourceMode);
269280
}
270281

271282
@Override
@@ -304,8 +315,8 @@ public String toString() {
304315
* Load {@code int}s from {@code _source}.
305316
*/
306317
public static class IntsBlockLoader extends SourceBlockLoader {
307-
public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
308-
super(fetcher, lookup);
318+
public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
319+
super(fetcher, lookup, sourceMode);
309320
}
310321

311322
@Override
@@ -344,8 +355,8 @@ public String toString() {
344355
* Load {@code long}s from {@code _source}.
345356
*/
346357
public static class LongsBlockLoader extends SourceBlockLoader {
347-
public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
348-
super(fetcher, lookup);
358+
public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
359+
super(fetcher, lookup, sourceMode);
349360
}
350361

351362
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
314314
BlockSourceReader.LeafIteratorLookup lookup = isIndexed() || isStored()
315315
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
316316
: BlockSourceReader.lookupMatchingAll();
317-
return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup);
317+
return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup, blContext.indexSettings().getIndexMappingSourceMode());
318318
}
319319

320320
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -809,7 +809,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
809809
BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
810810
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
811811
: BlockSourceReader.lookupMatchingAll();
812-
return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
812+
var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
813+
return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, sourceMode);
813814
}
814815

815816
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -632,18 +632,12 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
632632
if (hasDocValues()) {
633633
return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name());
634634
}
635-
if (isSyntheticSource) {
636-
if (false == isStored()) {
637-
throw new IllegalStateException(
638-
"keyword field ["
639-
+ name()
640-
+ "] is only supported in synthetic _source index if it creates doc values or stored fields"
641-
);
642-
}
635+
if (isStored()) {
643636
return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name());
644637
}
645638
SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()));
646-
return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext));
639+
var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
640+
return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext), sourceMode);
647641
}
648642

649643
private BlockSourceReader.LeafIteratorLookup sourceBlockLoaderLookup(BlockLoaderContext blContext) {

‎server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java‎

Lines changed: 49 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -461,8 +461,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
461461
}
462462

463463
@Override
464-
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
465-
return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
464+
BlockLoader blockLoaderFromSource(
465+
SourceValueFetcher sourceValueFetcher,
466+
BlockSourceReader.LeafIteratorLookup lookup,
467+
SourceFieldMapper.Mode sourceMode
468+
) {
469+
return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode);
466470
}
467471
},
468472
FLOAT("float", NumericType.FLOAT) {
@@ -645,8 +649,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
645649
}
646650

647651
@Override
648-
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
649-
return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
652+
BlockLoader blockLoaderFromSource(
653+
SourceValueFetcher sourceValueFetcher,
654+
BlockSourceReader.LeafIteratorLookup lookup,
655+
SourceFieldMapper.Mode sourceMode
656+
) {
657+
return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode);
650658
}
651659
},
652660
DOUBLE("double", NumericType.DOUBLE) {
@@ -795,8 +803,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
795803
}
796804

797805
@Override
798-
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
799-
return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
806+
BlockLoader blockLoaderFromSource(
807+
SourceValueFetcher sourceValueFetcher,
808+
BlockSourceReader.LeafIteratorLookup lookup,
809+
SourceFieldMapper.Mode sourceMode
810+
) {
811+
return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode);
800812
}
801813
},
802814
BYTE("byte", NumericType.BYTE) {
@@ -908,8 +920,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
908920
}
909921

910922
@Override
911-
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
912-
return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
923+
BlockLoader blockLoaderFromSource(
924+
SourceValueFetcher sourceValueFetcher,
925+
BlockSourceReader.LeafIteratorLookup lookup,
926+
SourceFieldMapper.Mode sourceMode
927+
) {
928+
return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode);
913929
}
914930

915931
private boolean isOutOfRange(Object value) {
@@ -1021,8 +1037,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
10211037
}
10221038

10231039
@Override
1024-
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
1025-
return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
1040+
BlockLoader blockLoaderFromSource(
1041+
SourceValueFetcher sourceValueFetcher,
1042+
BlockSourceReader.LeafIteratorLookup lookup,
1043+
SourceFieldMapper.Mode sourceMode
1044+
) {
1045+
return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode);
10261046
}
10271047

10281048
private boolean isOutOfRange(Object value) {
@@ -1208,8 +1228,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
12081228
}
12091229

12101230
@Override
1211-
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
1212-
return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
1231+
BlockLoader blockLoaderFromSource(
1232+
SourceValueFetcher sourceValueFetcher,
1233+
BlockSourceReader.LeafIteratorLookup lookup,
1234+
SourceFieldMapper.Mode sourceMode
1235+
) {
1236+
return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode);
12131237
}
12141238
},
12151239
LONG("long", NumericType.LONG) {
@@ -1355,8 +1379,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
13551379
}
13561380

13571381
@Override
1358-
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
1359-
return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup);
1382+
BlockLoader blockLoaderFromSource(
1383+
SourceValueFetcher sourceValueFetcher,
1384+
BlockSourceReader.LeafIteratorLookup lookup,
1385+
SourceFieldMapper.Mode sourceMode
1386+
) {
1387+
return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup, sourceMode);
13601388
}
13611389

13621390
private boolean isOutOfRange(Object value) {
@@ -1634,7 +1662,11 @@ protected void writeValue(XContentBuilder b, long value) throws IOException {
16341662

16351663
abstract BlockLoader blockLoaderFromDocValues(String fieldName);
16361664

1637-
abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup);
1665+
abstract BlockLoader blockLoaderFromSource(
1666+
SourceValueFetcher sourceValueFetcher,
1667+
BlockSourceReader.LeafIteratorLookup lookup,
1668+
SourceFieldMapper.Mode sourceMode
1669+
);
16381670
}
16391671

16401672
public static class NumberFieldType extends SimpleMappedFieldType {
@@ -1773,7 +1805,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
17731805
BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
17741806
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
17751807
: BlockSourceReader.lookupMatchingAll();
1776-
return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
1808+
var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
1809+
return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup, sourceMode);
17771810
}
17781811

17791812
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1012,17 +1012,20 @@ protected String delegatingTo() {
10121012
if (isStored()) {
10131013
return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(name());
10141014
}
1015-
if (isSyntheticSource) {
1015+
if (isSyntheticSource && syntheticSourceDelegate == null) {
10161016
/*
10171017
* When we're in synthetic source mode we don't currently
10181018
* support text fields that are not stored and are not children
10191019
* of perfect keyword fields. We'd have to load from the parent
1020-
* field and then convert the result to a string.
1020+
* field and then convert the result to a string. In this case,
1021+
* even if we would synthesize the source, the current field
1022+
* would be missing.
10211023
*/
10221024
return null;
10231025
}
10241026
SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
1025-
return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext));
1027+
var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
1028+
return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), sourceMode);
10261029
}
10271030

10281031
/**

‎server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public void testEmptyArray() throws IOException {
5151
private void loadBlock(LeafReaderContext ctx, Consumer<TestBlock> test) throws IOException {
5252
ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field"));
5353
BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field");
54-
BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup);
54+
BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup, null);
5555
assertThat(loader.columnAtATimeReader(ctx), nullValue());
5656
BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx);
5757
assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.NEEDS_SOURCE));

0 commit comments

Comments
 (0)