Skip to content

Commit 6dacdb0

Browse files
lktsgeorgewallace
lkts
authored and
georgewallace
committed
Use FallbackSyntheticSourceBlockLoader for boolean and date fields (elastic#124050)
1 parent 6c110ce commit 6dacdb0

File tree

26 files changed

+684
-23
lines changed

26 files changed

+684
-23
lines changed

‎docs/changelog/124050.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 124050
2+
summary: Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -342,7 +342,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
342342
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
343343
var nullValueAdjusted = nullValue != null ? adjustSourceValue(nullValue, scalingFactor) : null;
344344

345-
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<>(nullValue) {
345+
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<Double>(nullValue) {
346346
@Override
347347
public void convertValue(Object value, List<Double> accumulator) {
348348
if (coerce && value.equals("")) {

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import java.util.Collection;
5151
import java.util.Collections;
5252
import java.util.HashSet;
53+
import java.util.List;
5354
import java.util.Map;
5455
import java.util.Objects;
5556
import java.util.Set;
@@ -159,7 +160,8 @@ public BooleanFieldMapper build(MapperBuilderContext context) {
159160
nullValue.getValue(),
160161
scriptValues(),
161162
meta.getValue(),
162-
dimension.getValue()
163+
dimension.getValue(),
164+
context.isSourceSynthetic()
163165
);
164166
hasScript = script.get() != null;
165167
onScriptError = onScriptErrorParam.getValue();
@@ -188,6 +190,7 @@ public static final class BooleanFieldType extends TermBasedFieldType {
188190
private final Boolean nullValue;
189191
private final FieldValues<Boolean> scriptValues;
190192
private final boolean isDimension;
193+
private final boolean isSyntheticSource;
191194

192195
public BooleanFieldType(
193196
String name,
@@ -197,12 +200,14 @@ public BooleanFieldType(
197200
Boolean nullValue,
198201
FieldValues<Boolean> scriptValues,
199202
Map<String, String> meta,
200-
boolean isDimension
203+
boolean isDimension,
204+
boolean isSyntheticSource
201205
) {
202206
super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
203207
this.nullValue = nullValue;
204208
this.scriptValues = scriptValues;
205209
this.isDimension = isDimension;
210+
this.isSyntheticSource = isSyntheticSource;
206211
}
207212

208213
public BooleanFieldType(String name) {
@@ -214,7 +219,7 @@ public BooleanFieldType(String name, boolean isIndexed) {
214219
}
215220

216221
public BooleanFieldType(String name, boolean isIndexed, boolean hasDocValues) {
217-
this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false);
222+
this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false, false);
218223
}
219224

220225
@Override
@@ -251,12 +256,16 @@ protected Boolean parseSourceValue(Object value) {
251256
return (Boolean) value;
252257
} else {
253258
String textValue = value.toString();
254-
return Booleans.parseBoolean(textValue.toCharArray(), 0, textValue.length(), false);
259+
return parseBoolean(textValue);
255260
}
256261
}
257262
};
258263
}
259264

265+
private boolean parseBoolean(String text) {
266+
return Booleans.parseBoolean(text.toCharArray(), 0, text.length(), false);
267+
}
268+
260269
@Override
261270
public BytesRef indexedValueForSearch(Object value) {
262271
if (value == null) {
@@ -304,13 +313,62 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
304313
if (hasDocValues()) {
305314
return new BlockDocValuesReader.BooleansBlockLoader(name());
306315
}
316+
317+
if (isSyntheticSource) {
318+
return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) {
319+
@Override
320+
public Builder builder(BlockFactory factory, int expectedCount) {
321+
return factory.booleans(expectedCount);
322+
}
323+
};
324+
}
325+
307326
ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()));
308327
BlockSourceReader.LeafIteratorLookup lookup = isIndexed() || isStored()
309328
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
310329
: BlockSourceReader.lookupMatchingAll();
311330
return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup);
312331
}
313332

333+
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
334+
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<Boolean>(nullValue) {
335+
@Override
336+
public void convertValue(Object value, List<Boolean> accumulator) {
337+
try {
338+
if (value instanceof Boolean b) {
339+
accumulator.add(b);
340+
} else {
341+
String stringValue = value.toString();
342+
// Matches logic in parser invoked by `parseCreateField`
343+
accumulator.add(parseBoolean(stringValue));
344+
}
345+
} catch (Exception e) {
346+
// Malformed value, skip it
347+
}
348+
}
349+
350+
@Override
351+
protected void parseNonNullValue(XContentParser parser, List<Boolean> accumulator) throws IOException {
352+
// Aligned with implementation of `parseCreateField(XContentParser)`
353+
try {
354+
var value = parser.booleanValue();
355+
accumulator.add(value);
356+
} catch (Exception e) {
357+
// Malformed value, skip it
358+
}
359+
}
360+
361+
@Override
362+
public void writeToBlock(List<Boolean> values, BlockLoader.Builder blockBuilder) {
363+
var longBuilder = (BlockLoader.BooleanBuilder) blockBuilder;
364+
365+
for (var value : values) {
366+
longBuilder.appendBoolean(value);
367+
}
368+
}
369+
};
370+
}
371+
314372
@Override
315373
public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
316374
FielddataOperation operation = fieldDataContext.fielddataOperation();

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java‎

Lines changed: 76 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import org.elasticsearch.search.lookup.SearchLookup;
6262
import org.elasticsearch.search.runtime.LongScriptFieldDistanceFeatureQuery;
6363
import org.elasticsearch.xcontent.XContentBuilder;
64+
import org.elasticsearch.xcontent.XContentParser;
6465

6566
import java.io.IOException;
6667
import java.text.NumberFormat;
@@ -417,6 +418,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
417418
store.getValue(),
418419
docValues.getValue(),
419420
hasDocValuesSkipper,
421+
context.isSourceSynthetic(),
420422
buildFormatter(),
421423
resolution,
422424
nullValue.getValue(),
@@ -485,6 +487,7 @@ public static final class DateFieldType extends MappedFieldType {
485487
private final FieldValues<Long> scriptValues;
486488
private final boolean pointsMetadataAvailable;
487489
private final boolean hasDocValuesSkipper;
490+
private final boolean isSyntheticSource;
488491

489492
public DateFieldType(
490493
String name,
@@ -505,6 +508,7 @@ public DateFieldType(
505508
isStored,
506509
hasDocValues,
507510
false,
511+
false,
508512
dateTimeFormatter,
509513
resolution,
510514
nullValue,
@@ -520,6 +524,7 @@ public DateFieldType(
520524
boolean isStored,
521525
boolean hasDocValues,
522526
boolean hasDocValuesSkipper,
527+
boolean isSyntheticSource,
523528
DateFormatter dateTimeFormatter,
524529
Resolution resolution,
525530
String nullValue,
@@ -534,6 +539,7 @@ public DateFieldType(
534539
this.scriptValues = scriptValues;
535540
this.pointsMetadataAvailable = pointsMetadataAvailable;
536541
this.hasDocValuesSkipper = hasDocValuesSkipper;
542+
this.isSyntheticSource = isSyntheticSource;
537543
}
538544

539545
public DateFieldType(
@@ -547,7 +553,20 @@ public DateFieldType(
547553
FieldValues<Long> scriptValues,
548554
Map<String, String> meta
549555
) {
550-
this(name, isIndexed, isIndexed, isStored, hasDocValues, false, dateTimeFormatter, resolution, nullValue, scriptValues, meta);
556+
this(
557+
name,
558+
isIndexed,
559+
isIndexed,
560+
isStored,
561+
hasDocValues,
562+
false,
563+
false,
564+
dateTimeFormatter,
565+
resolution,
566+
nullValue,
567+
scriptValues,
568+
meta
569+
);
551570
}
552571

553572
public DateFieldType(String name) {
@@ -558,6 +577,7 @@ public DateFieldType(String name) {
558577
false,
559578
true,
560579
false,
580+
false,
561581
DEFAULT_DATE_TIME_FORMATTER,
562582
Resolution.MILLISECONDS,
563583
null,
@@ -574,6 +594,7 @@ public DateFieldType(String name, boolean isIndexed) {
574594
false,
575595
true,
576596
false,
597+
false,
577598
DEFAULT_DATE_TIME_FORMATTER,
578599
Resolution.MILLISECONDS,
579600
null,
@@ -583,15 +604,15 @@ public DateFieldType(String name, boolean isIndexed) {
583604
}
584605

585606
public DateFieldType(String name, DateFormatter dateFormatter) {
586-
this(name, true, true, false, true, false, dateFormatter, Resolution.MILLISECONDS, null, null, Collections.emptyMap());
607+
this(name, true, true, false, true, false, false, dateFormatter, Resolution.MILLISECONDS, null, null, Collections.emptyMap());
587608
}
588609

589610
public DateFieldType(String name, Resolution resolution) {
590-
this(name, true, true, false, true, false, DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap());
611+
this(name, true, true, false, true, false, false, DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap());
591612
}
592613

593614
public DateFieldType(String name, Resolution resolution, DateFormatter dateFormatter) {
594-
this(name, true, true, false, true, false, dateFormatter, resolution, null, null, Collections.emptyMap());
615+
this(name, true, true, false, true, false, false, dateFormatter, resolution, null, null, Collections.emptyMap());
595616
}
596617

597618
@Override
@@ -923,12 +944,63 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
923944
if (hasDocValues()) {
924945
return new BlockDocValuesReader.LongsBlockLoader(name());
925946
}
947+
948+
if (isSyntheticSource) {
949+
return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) {
950+
@Override
951+
public Builder builder(BlockFactory factory, int expectedCount) {
952+
return factory.longs(expectedCount);
953+
}
954+
};
955+
}
956+
926957
BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
927958
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
928959
: BlockSourceReader.lookupMatchingAll();
929960
return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
930961
}
931962

963+
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
964+
Function<String, Long> dateParser = this::parse;
965+
966+
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<Long>(nullValue) {
967+
@Override
968+
public void convertValue(Object value, List<Long> accumulator) {
969+
try {
970+
String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString();
971+
accumulator.add(dateParser.apply(date));
972+
} catch (Exception e) {
973+
// Malformed value, skip it
974+
}
975+
}
976+
977+
@Override
978+
protected void parseNonNullValue(XContentParser parser, List<Long> accumulator) throws IOException {
979+
// Aligned with implementation of `parseCreateField(XContentParser)`
980+
try {
981+
String dateAsString = parser.textOrNull();
982+
983+
if (dateAsString == null) {
984+
accumulator.add(dateParser.apply(nullValue));
985+
} else {
986+
accumulator.add(dateParser.apply(dateAsString));
987+
}
988+
} catch (Exception e) {
989+
// Malformed value, skip it
990+
}
991+
}
992+
993+
@Override
994+
public void writeToBlock(List<Long> values, BlockLoader.Builder blockBuilder) {
995+
var longBuilder = (BlockLoader.LongBuilder) blockBuilder;
996+
997+
for (var value : values) {
998+
longBuilder.appendLong(value);
999+
}
1000+
}
1001+
};
1002+
}
1003+
9321004
@Override
9331005
public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
9341006
FielddataOperation operation = fieldDataContext.fielddataOperation();

‎server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -275,9 +275,9 @@ public interface Reader<T> {
275275
}
276276

277277
public abstract static class ReaderWithNullValueSupport<T> implements Reader<T> {
278-
private final T nullValue;
278+
private final Object nullValue;
279279

280-
public ReaderWithNullValueSupport(T nullValue) {
280+
public ReaderWithNullValueSupport(Object nullValue) {
281281
this.nullValue = nullValue;
282282
}
283283

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -770,7 +770,7 @@ public Builder builder(BlockFactory factory, int expectedCount) {
770770

771771
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
772772
var nullValueBytes = nullValue != null ? new BytesRef(nullValue) : null;
773-
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<>(nullValueBytes) {
773+
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<BytesRef>(nullValueBytes) {
774774
@Override
775775
public void convertValue(Object value, List<BytesRef> accumulator) {
776776
String stringValue = ((BytesRef) value).utf8ToString();

‎server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ public void testRequireDocValuesOnDoubles() {
363363
public void testRequireDocValuesOnBools() {
364364
doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field"));
365365
doTestRequireDocValues(
366-
new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap(), false)
366+
new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap(), false, false)
367367
);
368368
}
369369

‎server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ public void testFetchSourceValue() throws IOException {
8181
true,
8282
null,
8383
Collections.emptyMap(),
84+
false,
8485
false
8586
);
8687
assertEquals(List.of(true), fetchSourceValue(nullFieldType, null));

‎server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ public void testIsFieldWithinQueryDateMillisDocValueSkipper() throws IOException
9494
false,
9595
true,
9696
true,
97+
false,
9798
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
9899
Resolution.MILLISECONDS,
99100
null,
@@ -111,6 +112,7 @@ public void testIsFieldWithinQueryDateNanosDocValueSkipper() throws IOException
111112
false,
112113
true,
113114
true,
115+
false,
114116
DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
115117
Resolution.NANOSECONDS,
116118
null,

0 commit comments

Comments
 (0)