Skip to content

Commit d480743

Browse files
lktslkts
authored
[8.x] Use FallbackSyntheticSourceBlockLoader for boolean and date fields (#124050) (#124147)
1 parent 2c937ec commit d480743

File tree

25 files changed

+719
-24
lines changed

25 files changed

+719
-24
lines changed

‎docs/changelog/124050.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 124050
2+
summary: Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -342,7 +342,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
342342
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
343343
var nullValueAdjusted = nullValue != null ? adjustSourceValue(nullValue, scalingFactor) : null;
344344

345-
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<>(nullValue) {
345+
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<Double>(nullValue) {
346346
@Override
347347
public void convertValue(Object value, List<Double> accumulator) {
348348
if (coerce && value.equals("")) {

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import java.util.Collection;
5252
import java.util.Collections;
5353
import java.util.HashSet;
54+
import java.util.List;
5455
import java.util.Map;
5556
import java.util.Objects;
5657
import java.util.Set;
@@ -162,7 +163,8 @@ public BooleanFieldMapper build(MapperBuilderContext context) {
162163
nullValue.getValue(),
163164
scriptValues(),
164165
meta.getValue(),
165-
dimension.getValue()
166+
dimension.getValue(),
167+
context.isSourceSynthetic()
166168
);
167169
hasScript = script.get() != null;
168170
onScriptError = onScriptErrorParam.getValue();
@@ -194,6 +196,7 @@ public static final class BooleanFieldType extends TermBasedFieldType {
194196
private final Boolean nullValue;
195197
private final FieldValues<Boolean> scriptValues;
196198
private final boolean isDimension;
199+
private final boolean isSyntheticSource;
197200

198201
public BooleanFieldType(
199202
String name,
@@ -203,12 +206,14 @@ public BooleanFieldType(
203206
Boolean nullValue,
204207
FieldValues<Boolean> scriptValues,
205208
Map<String, String> meta,
206-
boolean isDimension
209+
boolean isDimension,
210+
boolean isSyntheticSource
207211
) {
208212
super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
209213
this.nullValue = nullValue;
210214
this.scriptValues = scriptValues;
211215
this.isDimension = isDimension;
216+
this.isSyntheticSource = isSyntheticSource;
212217
}
213218

214219
public BooleanFieldType(String name) {
@@ -220,7 +225,7 @@ public BooleanFieldType(String name, boolean isIndexed) {
220225
}
221226

222227
public BooleanFieldType(String name, boolean isIndexed, boolean hasDocValues) {
223-
this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false);
228+
this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false, false);
224229
}
225230

226231
@Override
@@ -257,12 +262,16 @@ protected Boolean parseSourceValue(Object value) {
257262
return (Boolean) value;
258263
} else {
259264
String textValue = value.toString();
260-
return Booleans.parseBoolean(textValue.toCharArray(), 0, textValue.length(), false);
265+
return parseBoolean(textValue);
261266
}
262267
}
263268
};
264269
}
265270

271+
private boolean parseBoolean(String text) {
272+
return Booleans.parseBoolean(text.toCharArray(), 0, text.length(), false);
273+
}
274+
266275
@Override
267276
public BytesRef indexedValueForSearch(Object value) {
268277
if (value == null) {
@@ -310,13 +319,62 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
310319
if (hasDocValues()) {
311320
return new BlockDocValuesReader.BooleansBlockLoader(name());
312321
}
322+
323+
if (isSyntheticSource) {
324+
return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) {
325+
@Override
326+
public Builder builder(BlockFactory factory, int expectedCount) {
327+
return factory.booleans(expectedCount);
328+
}
329+
};
330+
}
331+
313332
ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()));
314333
BlockSourceReader.LeafIteratorLookup lookup = isIndexed() || isStored()
315334
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
316335
: BlockSourceReader.lookupMatchingAll();
317336
return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup);
318337
}
319338

339+
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
340+
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<Boolean>(nullValue) {
341+
@Override
342+
public void convertValue(Object value, List<Boolean> accumulator) {
343+
try {
344+
if (value instanceof Boolean b) {
345+
accumulator.add(b);
346+
} else {
347+
String stringValue = value.toString();
348+
// Matches logic in parser invoked by `parseCreateField`
349+
accumulator.add(parseBoolean(stringValue));
350+
}
351+
} catch (Exception e) {
352+
// Malformed value, skip it
353+
}
354+
}
355+
356+
@Override
357+
protected void parseNonNullValue(XContentParser parser, List<Boolean> accumulator) throws IOException {
358+
// Aligned with implementation of `parseCreateField(XContentParser)`
359+
try {
360+
var value = parser.booleanValue();
361+
accumulator.add(value);
362+
} catch (Exception e) {
363+
// Malformed value, skip it
364+
}
365+
}
366+
367+
@Override
368+
public void writeToBlock(List<Boolean> values, BlockLoader.Builder blockBuilder) {
369+
var longBuilder = (BlockLoader.BooleanBuilder) blockBuilder;
370+
371+
for (var value : values) {
372+
longBuilder.appendBoolean(value);
373+
}
374+
}
375+
};
376+
}
377+
320378
@Override
321379
public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
322380
FielddataOperation operation = fieldDataContext.fielddataOperation();

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java‎

Lines changed: 102 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
import org.elasticsearch.search.lookup.SearchLookup;
5757
import org.elasticsearch.search.runtime.LongScriptFieldDistanceFeatureQuery;
5858
import org.elasticsearch.xcontent.XContentBuilder;
59+
import org.elasticsearch.xcontent.XContentParser;
5960

6061
import java.io.IOException;
6162
import java.text.NumberFormat;
@@ -65,6 +66,7 @@
6566
import java.time.ZoneOffset;
6667
import java.time.ZonedDateTime;
6768
import java.util.Collections;
69+
import java.util.List;
6870
import java.util.Locale;
6971
import java.util.Map;
7072
import java.util.Objects;
@@ -370,6 +372,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
370372
index.getValue(),
371373
store.getValue(),
372374
docValues.getValue(),
375+
context.isSourceSynthetic(),
373376
buildFormatter(),
374377
resolution,
375378
nullValue.getValue(),
@@ -430,6 +433,7 @@ public static final class DateFieldType extends MappedFieldType {
430433
private final String nullValue;
431434
private final FieldValues<Long> scriptValues;
432435
private final boolean pointsMetadataAvailable;
436+
private final boolean isSyntheticSource;
433437

434438
public DateFieldType(
435439
String name,
@@ -442,6 +446,34 @@ public DateFieldType(
442446
String nullValue,
443447
FieldValues<Long> scriptValues,
444448
Map<String, String> meta
449+
) {
450+
this(
451+
name,
452+
isIndexed,
453+
pointsMetadataAvailable,
454+
isStored,
455+
hasDocValues,
456+
false,
457+
dateTimeFormatter,
458+
resolution,
459+
nullValue,
460+
scriptValues,
461+
meta
462+
);
463+
}
464+
465+
public DateFieldType(
466+
String name,
467+
boolean isIndexed,
468+
boolean pointsMetadataAvailable,
469+
boolean isStored,
470+
boolean hasDocValues,
471+
boolean isSyntheticSource,
472+
DateFormatter dateTimeFormatter,
473+
Resolution resolution,
474+
String nullValue,
475+
FieldValues<Long> scriptValues,
476+
Map<String, String> meta
445477
) {
446478
super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
447479
this.dateTimeFormatter = dateTimeFormatter;
@@ -450,6 +482,7 @@ public DateFieldType(
450482
this.nullValue = nullValue;
451483
this.scriptValues = scriptValues;
452484
this.pointsMetadataAvailable = pointsMetadataAvailable;
485+
this.isSyntheticSource = isSyntheticSource;
453486
}
454487

455488
public DateFieldType(
@@ -463,11 +496,23 @@ public DateFieldType(
463496
FieldValues<Long> scriptValues,
464497
Map<String, String> meta
465498
) {
466-
this(name, isIndexed, isIndexed, isStored, hasDocValues, dateTimeFormatter, resolution, nullValue, scriptValues, meta);
499+
this(name, isIndexed, isIndexed, isStored, hasDocValues, false, dateTimeFormatter, resolution, nullValue, scriptValues, meta);
467500
}
468501

469502
public DateFieldType(String name) {
470-
this(name, true, true, false, true, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, null, null, Collections.emptyMap());
503+
this(
504+
name,
505+
true,
506+
true,
507+
false,
508+
true,
509+
false,
510+
DEFAULT_DATE_TIME_FORMATTER,
511+
Resolution.MILLISECONDS,
512+
null,
513+
null,
514+
Collections.emptyMap()
515+
);
471516
}
472517

473518
public DateFieldType(String name, boolean isIndexed) {
@@ -477,6 +522,7 @@ public DateFieldType(String name, boolean isIndexed) {
477522
isIndexed,
478523
false,
479524
true,
525+
false,
480526
DEFAULT_DATE_TIME_FORMATTER,
481527
Resolution.MILLISECONDS,
482528
null,
@@ -486,15 +532,15 @@ public DateFieldType(String name, boolean isIndexed) {
486532
}
487533

488534
public DateFieldType(String name, DateFormatter dateFormatter) {
489-
this(name, true, true, false, true, dateFormatter, Resolution.MILLISECONDS, null, null, Collections.emptyMap());
535+
this(name, true, true, false, true, false, dateFormatter, Resolution.MILLISECONDS, null, null, Collections.emptyMap());
490536
}
491537

492538
public DateFieldType(String name, Resolution resolution) {
493-
this(name, true, true, false, true, DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap());
539+
this(name, true, true, false, true, false, DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap());
494540
}
495541

496542
public DateFieldType(String name, Resolution resolution, DateFormatter dateFormatter) {
497-
this(name, true, true, false, true, dateFormatter, resolution, null, null, Collections.emptyMap());
543+
this(name, true, true, false, true, false, dateFormatter, resolution, null, null, Collections.emptyMap());
498544
}
499545

500546
@Override
@@ -806,12 +852,63 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
806852
if (hasDocValues()) {
807853
return new BlockDocValuesReader.LongsBlockLoader(name());
808854
}
855+
856+
if (isSyntheticSource) {
857+
return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) {
858+
@Override
859+
public Builder builder(BlockFactory factory, int expectedCount) {
860+
return factory.longs(expectedCount);
861+
}
862+
};
863+
}
864+
809865
BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
810866
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
811867
: BlockSourceReader.lookupMatchingAll();
812868
return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
813869
}
814870

871+
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
872+
Function<String, Long> dateParser = this::parse;
873+
874+
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<Long>(nullValue) {
875+
@Override
876+
public void convertValue(Object value, List<Long> accumulator) {
877+
try {
878+
String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString();
879+
accumulator.add(dateParser.apply(date));
880+
} catch (Exception e) {
881+
// Malformed value, skip it
882+
}
883+
}
884+
885+
@Override
886+
protected void parseNonNullValue(XContentParser parser, List<Long> accumulator) throws IOException {
887+
// Aligned with implementation of `parseCreateField(XContentParser)`
888+
try {
889+
String dateAsString = parser.textOrNull();
890+
891+
if (dateAsString == null) {
892+
accumulator.add(dateParser.apply(nullValue));
893+
} else {
894+
accumulator.add(dateParser.apply(dateAsString));
895+
}
896+
} catch (Exception e) {
897+
// Malformed value, skip it
898+
}
899+
}
900+
901+
@Override
902+
public void writeToBlock(List<Long> values, BlockLoader.Builder blockBuilder) {
903+
var longBuilder = (BlockLoader.LongBuilder) blockBuilder;
904+
905+
for (var value : values) {
906+
longBuilder.appendLong(value);
907+
}
908+
}
909+
};
910+
}
911+
815912
@Override
816913
public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
817914
FielddataOperation operation = fieldDataContext.fielddataOperation();

‎server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -275,9 +275,9 @@ public interface Reader<T> {
275275
}
276276

277277
public abstract static class ReaderWithNullValueSupport<T> implements Reader<T> {
278-
private final T nullValue;
278+
private final Object nullValue;
279279

280-
public ReaderWithNullValueSupport(T nullValue) {
280+
public ReaderWithNullValueSupport(Object nullValue) {
281281
this.nullValue = nullValue;
282282
}
283283

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -669,7 +669,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
669669

670670
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
671671
var nullValueBytes = nullValue != null ? new BytesRef(nullValue) : null;
672-
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<>(nullValueBytes) {
672+
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<BytesRef>(nullValueBytes) {
673673
@Override
674674
public void convertValue(Object value, List<BytesRef> accumulator) {
675675
String stringValue = ((BytesRef) value).utf8ToString();

‎server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ public void testRequireDocValuesOnDoubles() {
363363
public void testRequireDocValuesOnBools() {
364364
doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field"));
365365
doTestRequireDocValues(
366-
new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap(), false)
366+
new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap(), false, false)
367367
);
368368
}
369369

‎server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ public void testFetchSourceValue() throws IOException {
8181
true,
8282
null,
8383
Collections.emptyMap(),
84+
false,
8485
false
8586
);
8687
assertEquals(List.of(true), fetchSourceValue(nullFieldType, null));

0 commit comments

Comments
 (0)