diff --git a/docs/changelog/124050.yaml b/docs/changelog/124050.yaml new file mode 100644 index 0000000000000..352678dd4bb5a --- /dev/null +++ b/docs/changelog/124050.yaml @@ -0,0 +1,5 @@ +pr: 124050 +summary: Use `FallbackSyntheticSourceBlockLoader` for boolean and date fields +area: Mapping +type: enhancement +issues: [] diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 17729d7c57dde..d68f417f91b42 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -342,7 +342,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { var nullValueAdjusted = nullValue != null ? adjustSourceValue(nullValue, scalingFactor) : null; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<>(nullValue) { + return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValue) { @Override public void convertValue(Object value, List accumulator) { if (coerce && value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 8100b83463c93..d7f4a9d4241a3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -50,6 +50,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -159,7 +160,8 @@ public BooleanFieldMapper build(MapperBuilderContext context) { nullValue.getValue(), scriptValues(), meta.getValue(), - dimension.getValue() + dimension.getValue(), + context.isSourceSynthetic() ); hasScript = script.get() != null; onScriptError = onScriptErrorParam.getValue(); @@ -188,6 +190,7 @@ public static final class BooleanFieldType extends TermBasedFieldType { private final Boolean nullValue; private final FieldValues scriptValues; private final boolean isDimension; + private final boolean isSyntheticSource; public BooleanFieldType( String name, @@ -197,12 +200,14 @@ public BooleanFieldType( Boolean nullValue, FieldValues scriptValues, Map meta, - boolean isDimension + boolean isDimension, + boolean isSyntheticSource ) { super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.nullValue = nullValue; this.scriptValues = scriptValues; this.isDimension = isDimension; + this.isSyntheticSource = isSyntheticSource; } public BooleanFieldType(String name) { @@ -214,7 +219,7 @@ public BooleanFieldType(String name, boolean isIndexed) { } public BooleanFieldType(String name, boolean isIndexed, boolean hasDocValues) { - this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false); + this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false, false); } @Override @@ -251,12 +256,16 @@ protected Boolean parseSourceValue(Object value) { return (Boolean) value; } else { String textValue = value.toString(); - return Booleans.parseBoolean(textValue.toCharArray(), 0, textValue.length(), false); + return parseBoolean(textValue); } } }; } + private boolean parseBoolean(String text) { + return Booleans.parseBoolean(text.toCharArray(), 0, text.length(), false); + } + @Override public BytesRef indexedValueForSearch(Object value) { if (value == null) { @@ -304,6 +313,16 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { return new BlockDocValuesReader.BooleansBlockLoader(name()); } + + if (isSyntheticSource) { + return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.booleans(expectedCount); + } + }; + } + ValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); BlockSourceReader.LeafIteratorLookup lookup = isIndexed() || isStored() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) @@ -311,6 +330,45 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup); } + private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { + return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValue) { + @Override + public void convertValue(Object value, List accumulator) { + try { + if (value instanceof Boolean b) { + accumulator.add(b); + } else { + String stringValue = value.toString(); + // Matches logic in parser invoked by `parseCreateField` + accumulator.add(parseBoolean(stringValue)); + } + } catch (Exception e) { + // Malformed value, skip it + } + } + + @Override + protected void parseNonNullValue(XContentParser parser, List accumulator) throws IOException { + // Aligned with implementation of `parseCreateField(XContentParser)` + try { + var value = parser.booleanValue(); + accumulator.add(value); + } catch (Exception e) { + // Malformed value, skip it + } + } + + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + var longBuilder = (BlockLoader.BooleanBuilder) blockBuilder; + + for (var value : values) { + longBuilder.appendBoolean(value); + } + } + }; + } + @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { FielddataOperation operation = fieldDataContext.fielddataOperation(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 0bf53682d2d17..72eb8841d5bae 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -61,6 +61,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.runtime.LongScriptFieldDistanceFeatureQuery; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.text.NumberFormat; @@ -417,6 +418,7 @@ public DateFieldMapper build(MapperBuilderContext context) { store.getValue(), docValues.getValue(), hasDocValuesSkipper, + context.isSourceSynthetic(), buildFormatter(), resolution, nullValue.getValue(), @@ -485,6 +487,7 @@ public static final class DateFieldType extends MappedFieldType { private final FieldValues scriptValues; private final boolean pointsMetadataAvailable; private final boolean hasDocValuesSkipper; + private final boolean isSyntheticSource; public DateFieldType( String name, @@ -505,6 +508,7 @@ public DateFieldType( isStored, hasDocValues, false, + false, dateTimeFormatter, resolution, nullValue, @@ -520,6 +524,7 @@ public DateFieldType( boolean isStored, boolean hasDocValues, boolean hasDocValuesSkipper, + boolean isSyntheticSource, DateFormatter dateTimeFormatter, Resolution resolution, String nullValue, @@ -534,6 +539,7 @@ public DateFieldType( this.scriptValues = scriptValues; this.pointsMetadataAvailable = pointsMetadataAvailable; this.hasDocValuesSkipper = hasDocValuesSkipper; + this.isSyntheticSource = isSyntheticSource; } public DateFieldType( @@ -547,7 +553,20 @@ public DateFieldType( FieldValues scriptValues, Map meta ) { - this(name, isIndexed, isIndexed, isStored, hasDocValues, false, dateTimeFormatter, resolution, nullValue, scriptValues, meta); + this( + name, + isIndexed, + isIndexed, + isStored, + hasDocValues, + false, + false, + dateTimeFormatter, + resolution, + nullValue, + scriptValues, + meta + ); } public DateFieldType(String name) { @@ -558,6 +577,7 @@ public DateFieldType(String name) { false, true, false, + false, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, null, @@ -574,6 +594,7 @@ public DateFieldType(String name, boolean isIndexed) { false, true, false, + false, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, null, @@ -583,15 +604,15 @@ public DateFieldType(String name, boolean isIndexed) { } public DateFieldType(String name, DateFormatter dateFormatter) { - this(name, true, true, false, true, false, dateFormatter, Resolution.MILLISECONDS, null, null, Collections.emptyMap()); + this(name, true, true, false, true, false, false, dateFormatter, Resolution.MILLISECONDS, null, null, Collections.emptyMap()); } public DateFieldType(String name, Resolution resolution) { - this(name, true, true, false, true, false, DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap()); + this(name, true, true, false, true, false, false, DEFAULT_DATE_TIME_FORMATTER, resolution, null, null, Collections.emptyMap()); } public DateFieldType(String name, Resolution resolution, DateFormatter dateFormatter) { - this(name, true, true, false, true, false, dateFormatter, resolution, null, null, Collections.emptyMap()); + this(name, true, true, false, true, false, false, dateFormatter, resolution, null, null, Collections.emptyMap()); } @Override @@ -923,12 +944,63 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { return new BlockDocValuesReader.LongsBlockLoader(name()); } + + if (isSyntheticSource) { + return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + }; + } + BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup); } + private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { + Function dateParser = this::parse; + + return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValue) { + @Override + public void convertValue(Object value, List accumulator) { + try { + String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString(); + accumulator.add(dateParser.apply(date)); + } catch (Exception e) { + // Malformed value, skip it + } + } + + @Override + protected void parseNonNullValue(XContentParser parser, List accumulator) throws IOException { + // Aligned with implementation of `parseCreateField(XContentParser)` + try { + String dateAsString = parser.textOrNull(); + + if (dateAsString == null) { + accumulator.add(dateParser.apply(nullValue)); + } else { + accumulator.add(dateParser.apply(dateAsString)); + } + } catch (Exception e) { + // Malformed value, skip it + } + } + + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + var longBuilder = (BlockLoader.LongBuilder) blockBuilder; + + for (var value : values) { + longBuilder.appendLong(value); + } + } + }; + } + @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { FielddataOperation operation = fieldDataContext.fielddataOperation(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java index 68b2e31a4b011..8474b754e951d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java @@ -275,9 +275,9 @@ public interface Reader { } public abstract static class ReaderWithNullValueSupport implements Reader { - private final T nullValue; + private final Object nullValue; - public ReaderWithNullValueSupport(T nullValue) { + public ReaderWithNullValueSupport(Object nullValue) { this.nullValue = nullValue; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 127024f17a222..340af979c0593 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -770,7 +770,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBlockLoaderReader() { var nullValueBytes = nullValue != null ? new BytesRef(nullValue) : null; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<>(nullValueBytes) { + return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValueBytes) { @Override public void convertValue(Object value, List accumulator) { String stringValue = ((BytesRef) value).utf8ToString(); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 172545ab459c2..07e836403e7ef 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -363,7 +363,7 @@ public void testRequireDocValuesOnDoubles() { public void testRequireDocValuesOnBools() { doTestRequireDocValues(new BooleanFieldMapper.BooleanFieldType("field")); doTestRequireDocValues( - new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap(), false) + new BooleanFieldMapper.BooleanFieldType("field", true, false, false, null, null, Collections.emptyMap(), false, false) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java index 19a38e19fbf6e..38f73cd9f681d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java @@ -81,6 +81,7 @@ public void testFetchSourceValue() throws IOException { true, null, Collections.emptyMap(), + false, false ); assertEquals(List.of(true), fetchSourceValue(nullFieldType, null)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index ad258086affc7..16c2c5ca5ddb8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -94,6 +94,7 @@ public void testIsFieldWithinQueryDateMillisDocValueSkipper() throws IOException false, true, true, + false, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, null, @@ -111,6 +112,7 @@ public void testIsFieldWithinQueryDateNanosDocValueSkipper() throws IOException false, true, true, + false, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, Resolution.NANOSECONDS, null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..906208548b963 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/BooleanFieldBlockLoaderTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.logsdb.datageneration.FieldType; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class BooleanFieldBlockLoaderTests extends BlockLoaderTestCase { + public BooleanFieldBlockLoaderTests() { + super(FieldType.BOOLEAN); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value, boolean syntheticSource) { + var nullValue = switch (fieldMapping.get("null_value")) { + case Boolean b -> b; + case String s -> Boolean.parseBoolean(s); + case null -> null; + default -> throw new IllegalStateException("Unexpected null_value format"); + }; + + if (value instanceof List == false) { + return convert(value, nullValue); + } + + if ((boolean) fieldMapping.getOrDefault("doc_values", false)) { + // Sorted + var resultList = ((List) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).sorted().toList(); + return maybeFoldList(resultList); + } + + // parsing from source, not sorted + var resultList = ((List) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).toList(); + return maybeFoldList(resultList); + } + + @SuppressWarnings("unchecked") + private Object convert(Object value, Object nullValue) { + if (value == null) { + return nullValue; + } + if (value instanceof String s) { + if (s.isEmpty()) { + // This is a documented behavior. + return false; + } + if (value.equals("true")) { + return true; + } + if (value.equals("false")) { + return false; + } + } + if (value instanceof Boolean b) { + return b; + } + + // Malformed values are excluded + return null; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..a969a79aa044f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.logsdb.datageneration.FieldType; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +public class DateFieldBlockLoaderTests extends BlockLoaderTestCase { + public DateFieldBlockLoaderTests() { + super(FieldType.DATE); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value, boolean syntheticSource) { + var format = (String) fieldMapping.get("format"); + var nullValue = fieldMapping.get("null_value") != null ? format(fieldMapping.get("null_value"), format) : null; + + if (value instanceof List == false) { + return convert(value, nullValue, format); + } + + if ((boolean) fieldMapping.getOrDefault("doc_values", false)) { + // Sorted + var resultList = ((List) value).stream() + .map(v -> convert(v, nullValue, format)) + .filter(Objects::nonNull) + .sorted() + .toList(); + return maybeFoldList(resultList); + } + + // parsing from source, not sorted + var resultList = ((List) value).stream().map(v -> convert(v, nullValue, format)).filter(Objects::nonNull).toList(); + return maybeFoldList(resultList); + } + + private Long convert(Object value, Long nullValue, String format) { + if (value == null) { + return nullValue; + } + + return format(value, format); + } + + private Long format(Object value, String format) { + if (format == null) { + return switch (value) { + case Integer i -> i.longValue(); + case Long l -> l; + case String s -> { + try { + yield Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(s)).toEpochMilli(); + } catch (Exception e) { + // malformed + yield null; + } + } + case null -> null; + default -> throw new IllegalStateException("Unexpected value: " + value); + }; + } + + try { + return Instant.from( + DateTimeFormatter.ofPattern(format, Locale.ROOT).withZone(ZoneId.from(ZoneOffset.UTC)).parse((String) value) + ).toEpochMilli(); + } catch (Exception e) { + // malformed + return null; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java index 909cccf9e7d54..b7f7bd9de5ef7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java @@ -46,7 +46,6 @@ protected Object expected(Map fieldMapping, Object value, boolea if ((boolean) fieldMapping.getOrDefault("doc_values", false)) { // Sorted and no duplicates - var resultList = convertValues.andThen(Stream::distinct) .andThen(Stream::sorted) .andThen(Stream::toList) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java index ab6fd109ed375..a8b1e68061041 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java @@ -71,6 +71,16 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle( this.documentGenerator = new DocumentGenerator(specification); } + @Override + public void testFieldHasValue() { + assumeTrue("random test inherited from MapperServiceTestCase", false); + } + + @Override + public void testFieldHasValueWithEmptyFieldInfos() { + assumeTrue("random test inherited from MapperServiceTestCase", false); + } + public void testBlockLoader() throws IOException { var template = new Template(Map.of(fieldName, new Template.Leaf(fieldName, fieldType))); var syntheticSource = randomBoolean(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java index e5f68362005d5..43c2663ac71e8 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldBlockLoaderTestCase.java @@ -30,7 +30,7 @@ protected Object expected(Map fieldMapping, Object value, boolea } if ((boolean) fieldMapping.getOrDefault("doc_values", false)) { - // Sorted and no duplicates + // Sorted var resultList = ((List) value).stream() .map(v -> convert(v, nullValue, fieldMapping)) .filter(Objects::nonNull) diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java index 4bf65fcf6ecf6..851812268e9ba 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java @@ -10,8 +10,10 @@ package org.elasticsearch.logsdb.datageneration; import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.fields.leaf.BooleanFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.ByteFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.CountedKeywordFieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.fields.leaf.DateFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.DoubleFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.FloatFieldDataGenerator; import org.elasticsearch.logsdb.datageneration.fields.leaf.HalfFloatFieldDataGenerator; @@ -36,7 +38,9 @@ public enum FieldType { FLOAT("float"), HALF_FLOAT("half_float"), SCALED_FLOAT("scaled_float"), - COUNTED_KEYWORD("counted_keyword"); + COUNTED_KEYWORD("counted_keyword"), + BOOLEAN("boolean"), + DATE("date"); private final String name; @@ -57,6 +61,8 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) { case HALF_FLOAT -> new HalfFloatFieldDataGenerator(fieldName, dataSource); case SCALED_FLOAT -> new ScaledFloatFieldDataGenerator(fieldName, dataSource); case COUNTED_KEYWORD -> new CountedKeywordFieldDataGenerator(fieldName, dataSource); + case BOOLEAN -> new BooleanFieldDataGenerator(dataSource); + case DATE -> new DateFieldDataGenerator(dataSource); }; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java index beef9fb4dd799..2a17f9311faa9 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java @@ -46,6 +46,14 @@ default DataSourceResponse.StringGenerator handle(DataSourceRequest.StringGenera return null; } + default DataSourceResponse.BooleanGenerator handle(DataSourceRequest.BooleanGenerator request) { + return null; + } + + default DataSourceResponse.InstantGenerator handle(DataSourceRequest.InstantGenerator request) { + return null; + } + default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) { return null; } @@ -62,6 +70,10 @@ default DataSourceResponse.MalformedWrapper handle(DataSourceRequest.MalformedWr return null; } + default DataSourceResponse.TransformWrapper handle(DataSourceRequest.TransformWrapper request) { + return null; + } + default DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) { return null; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 0e6e796ff6d54..2a1ca7297d7db 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -15,6 +15,7 @@ import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; public interface DataSourceRequest { @@ -74,6 +75,18 @@ public DataSourceResponse.StringGenerator accept(DataSourceHandler handler) { } } + record BooleanGenerator() implements DataSourceRequest { + public DataSourceResponse.BooleanGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + + record InstantGenerator() implements DataSourceRequest { + public DataSourceResponse.InstantGenerator accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + record NullWrapper() implements DataSourceRequest { public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) { return handler.handle(this); @@ -98,6 +111,14 @@ public DataSourceResponse.MalformedWrapper accept(DataSourceHandler handler) { } } + record TransformWrapper(double transformedProportion, Function transformation) + implements + DataSourceRequest { + public DataSourceResponse.TransformWrapper accept(DataSourceHandler handler) { + return handler.handle(this); + } + } + record ChildFieldGenerator(DataGeneratorSpecification specification) implements DataSourceRequest { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java index e9f1adb98d248..e7a64471e024c 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.logsdb.datageneration.FieldType; +import java.time.Instant; import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -35,6 +36,10 @@ record HalfFloatGenerator(Supplier generator) implements DataSourceRespon record StringGenerator(Supplier generator) implements DataSourceResponse {} + record BooleanGenerator(Supplier generator) implements DataSourceResponse {} + + record InstantGenerator(Supplier generator) implements DataSourceResponse {} + record NullWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} record ArrayWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} @@ -43,6 +48,8 @@ record RepeatingWrapper(Function, Supplier> wrapper) im record MalformedWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} + record TransformWrapper(Function, Supplier> wrapper) implements DataSourceResponse {} + interface ChildFieldGenerator extends DataSourceResponse { int generateChildFieldCount(); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index e7b9140b07daf..93faf795ff565 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -14,7 +14,12 @@ import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.test.ESTestCase; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -35,6 +40,8 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> numberMapping(map, request.fieldType()); case SCALED_FLOAT -> scaledFloatMapping(map); case COUNTED_KEYWORD -> plain(Map.of("index", ESTestCase.randomBoolean())); + case BOOLEAN -> booleanMapping(map); + case DATE -> dateMapping(map); }); } @@ -113,6 +120,52 @@ private Supplier> scaledFloatMapping(Map inj }; } + private Supplier> booleanMapping(Map injected) { + return () -> { + if (ESTestCase.randomDouble() <= 0.2) { + injected.put("null_value", ESTestCase.randomFrom(true, false, "true", "false")); + } + + if (ESTestCase.randomBoolean()) { + injected.put("ignore_malformed", ESTestCase.randomBoolean()); + } + + return injected; + }; + } + + // just a custom format, specific format does not matter + private static final String FORMAT = "yyyy_MM_dd_HH_mm_ss_n"; + + private Supplier> dateMapping(Map injected) { + return () -> { + String format = null; + if (ESTestCase.randomBoolean()) { + format = FORMAT; + injected.put("format", format); + } + + if (ESTestCase.randomDouble() <= 0.2) { + var instant = ESTestCase.randomInstantBetween(Instant.parse("2300-01-01T00:00:00Z"), Instant.parse("2350-01-01T00:00:00Z")); + + if (format == null) { + injected.put("null_value", instant.toEpochMilli()); + } else { + injected.put( + "null_value", + DateTimeFormatter.ofPattern(format, Locale.ROOT).withZone(ZoneId.from(ZoneOffset.UTC)).format(instant) + ); + } + } + + if (ESTestCase.randomBoolean()) { + injected.put("ignore_malformed", ESTestCase.randomBoolean()); + } + + return injected; + }; + } + @Override public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { if (request.isNested()) { diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java index 5d94a841dc0ed..80f7e3c24e3a9 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultPrimitiveTypesHandler.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.ESTestCase; import java.math.BigInteger; +import java.time.Instant; public class DefaultPrimitiveTypesHandler implements DataSourceHandler { @Override @@ -59,4 +60,16 @@ public DataSourceResponse.HalfFloatGenerator handle(DataSourceRequest.HalfFloatG public DataSourceResponse.StringGenerator handle(DataSourceRequest.StringGenerator request) { return new DataSourceResponse.StringGenerator(() -> ESTestCase.randomAlphaOfLengthBetween(0, 50)); } + + @Override + public DataSourceResponse.BooleanGenerator handle(DataSourceRequest.BooleanGenerator request) { + return new DataSourceResponse.BooleanGenerator(ESTestCase::randomBoolean); + } + + private static final Instant MAX_INSTANT = Instant.parse("2200-01-01T00:00:00Z"); + + @Override + public DataSourceResponse.InstantGenerator handle(DataSourceRequest.InstantGenerator request) { + return new DataSourceResponse.InstantGenerator(() -> ESTestCase.randomInstantBetween(Instant.ofEpochMilli(1), MAX_INSTANT)); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java index ac686e0201327..ae4ad196c47d0 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultWrappersHandler.java @@ -37,9 +37,14 @@ public DataSourceResponse.MalformedWrapper handle(DataSourceRequest.MalformedWra return new DataSourceResponse.MalformedWrapper(injectMalformed(request.malformedValues())); } + @Override + public DataSourceResponse.TransformWrapper handle(DataSourceRequest.TransformWrapper request) { + return new DataSourceResponse.TransformWrapper(transform(request.transformedProportion(), request.transformation())); + } + private static Function, Supplier> injectNulls() { // Inject some nulls but majority of data should be non-null (as it likely is in reality). - return (values) -> () -> ESTestCase.randomDouble() <= 0.05 ? null : values.get(); + return transform(0.05, ignored -> null); } private static Function, Supplier> wrapInArray() { @@ -69,6 +74,13 @@ private static Function, Supplier> repeatValues() { } private static Function, Supplier> injectMalformed(Supplier malformedValues) { - return (values) -> () -> ESTestCase.randomDouble() <= 0.1 ? malformedValues.get() : values.get(); + return transform(0.1, ignored -> malformedValues.get()); + } + + private static Function, Supplier> transform( + double transformedProportion, + Function transformation + ) { + return (values) -> () -> ESTestCase.randomDouble() <= transformedProportion ? transformation.apply(values.get()) : values.get(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/BooleanFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/BooleanFieldDataGenerator.java new file mode 100644 index 0000000000000..482ed1b1321e5 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/BooleanFieldDataGenerator.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration.fields.leaf; + +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; + +import java.util.Map; +import java.util.function.Supplier; + +public class BooleanFieldDataGenerator implements FieldDataGenerator { + private final DataSource dataSource; + private final Supplier booleans; + private final Supplier booleansWithStrings; + private final Supplier booleansWithStringsAndMalformed; + + public BooleanFieldDataGenerator(DataSource dataSource) { + this.dataSource = dataSource; + + var booleans = dataSource.get(new DataSourceRequest.BooleanGenerator()).generator(); + this.booleans = booleans::get; + + // produces "true" and "false" strings + var toStringTransform = dataSource.get(new DataSourceRequest.TransformWrapper(0.5, Object::toString)).wrapper(); + this.booleansWithStrings = toStringTransform.apply(this.booleans::get); + + var strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + this.booleansWithStringsAndMalformed = Wrappers.defaultsWithMalformed(booleansWithStrings, strings::get, dataSource); + } + + @Override + public Object generateValue(Map fieldMapping) { + if (fieldMapping == null) { + // dynamically mapped, use booleans only to avoid mapping the field as string + return Wrappers.defaults(booleans, dataSource).get(); + } + + if ((Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + return booleansWithStringsAndMalformed.get(); + } + + return Wrappers.defaults(booleansWithStrings, dataSource).get(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DateFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DateFieldDataGenerator.java new file mode 100644 index 0000000000000..ae267ca7570fc --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/leaf/DateFieldDataGenerator.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.logsdb.datageneration.fields.leaf; + +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; +import org.elasticsearch.logsdb.datageneration.datasource.DataSource; +import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Locale; +import java.util.Map; +import java.util.function.Supplier; + +public class DateFieldDataGenerator implements FieldDataGenerator { + private final DataSource dataSource; + private final Supplier instants; + private final Supplier strings; + + public DateFieldDataGenerator(DataSource dataSource) { + this.dataSource = dataSource; + this.instants = () -> dataSource.get(new DataSourceRequest.InstantGenerator()).generator().get(); + this.strings = dataSource.get(new DataSourceRequest.StringGenerator()).generator(); + } + + @Override + public Object generateValue(Map fieldMapping) { + Supplier supplier = () -> instants.get().toEpochMilli(); + + if (fieldMapping != null && fieldMapping.get("format") != null) { + String format = (String) fieldMapping.get("format"); + supplier = () -> DateTimeFormatter.ofPattern(format, Locale.ROOT).withZone(ZoneId.from(ZoneOffset.UTC)).format(instants.get()); + } + + if (fieldMapping != null && (Boolean) fieldMapping.getOrDefault("ignore_malformed", false)) { + supplier = Wrappers.defaultsWithMalformed(supplier, strings::get, dataSource); + } else { + supplier = Wrappers.defaults(supplier, dataSource); + } + + return supplier.get(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java index e2acea4ad91de..f424a7ecf45ed 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/FieldSpecificMatcher.java @@ -11,15 +11,22 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.logsdb.datageneration.matchers.MatchResult; import org.elasticsearch.xcontent.XContentBuilder; import java.math.BigInteger; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TreeMap; +import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.logsdb.datageneration.matchers.Messages.formatErrorMessage; @@ -334,23 +341,138 @@ Object convert(Object value, Object nullValue) { } } - // TODO basic implementation only right now - class DateMatcher extends GenericMappingAwareMatcher { - DateMatcher( + class BooleanMatcher extends GenericMappingAwareMatcher { + BooleanMatcher( XContentBuilder actualMappings, Settings.Builder actualSettings, XContentBuilder expectedMappings, Settings.Builder expectedSettings ) { - super("date", actualMappings, actualSettings, expectedMappings, expectedSettings); + super("boolean", actualMappings, actualSettings, expectedMappings, expectedSettings); } @Override Object convert(Object value, Object nullValue) { + Boolean nullValueBool = null; + if (nullValue != null) { + nullValueBool = nullValue instanceof Boolean b ? b : Boolean.parseBoolean((String) nullValue); + } + + if (value == null) { + return nullValueBool; + } + if (value instanceof String s && s.isEmpty()) { + // This a documented behavior. + return false; + } + if (value instanceof String s) { + try { + return Boolean.parseBoolean(s); + } catch (Exception e) { + // malformed + return value; + } + } + return value; } } + class DateMatcher implements FieldSpecificMatcher { + private final XContentBuilder actualMappings; + private final Settings.Builder actualSettings; + private final XContentBuilder expectedMappings; + private final Settings.Builder expectedSettings; + + DateMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + this.actualMappings = actualMappings; + this.actualSettings = actualSettings; + this.expectedMappings = expectedMappings; + this.expectedSettings = expectedSettings; + } + + @Override + public MatchResult match( + List actual, + List expected, + Map actualMapping, + Map expectedMapping + ) { + var format = (String) getMappingParameter("format", actualMapping, expectedMapping); + var nullValue = getNullValue(actualMapping, expectedMapping); + + Function convert = v -> convert(v, nullValue); + if (format != null) { + var formatter = DateTimeFormatter.ofPattern(format, Locale.ROOT).withZone(ZoneId.from(ZoneOffset.UTC)); + convert = v -> convert(v, nullValue, formatter); + } + + var actualNormalized = normalize(actual, convert); + var expectedNormalized = normalize(expected, convert); + + return actualNormalized.equals(expectedNormalized) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Values of type [date] don't match after normalization, normalized " + + prettyPrintCollections(actualNormalized, expectedNormalized) + ) + ); + } + + private Set normalize(List values, Function convert) { + if (values == null) { + return Set.of(); + } + + return values.stream().map(convert).filter(Objects::nonNull).collect(Collectors.toSet()); + } + + Object convert(Object value, Object nullValue) { + if (value == null) { + return nullValue == null ? null : Instant.ofEpochMilli((Long) nullValue); + } + if (value instanceof Integer i) { + return Instant.ofEpochMilli(i); + } + if (value instanceof Long l) { + return Instant.ofEpochMilli(l); + } + + assert value instanceof String; + try { + // values from synthetic source will be formatted with default formatter + return Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse((String) value)); + } catch (Exception e) { + // malformed + return value; + } + } + + Object convert(Object value, Object nullValue, DateTimeFormatter dateTimeFormatter) { + if (value == null) { + return nullValue == null ? null : Instant.from(dateTimeFormatter.parse((String) nullValue)).toEpochMilli(); + } + + assert value instanceof String; + try { + return Instant.from(dateTimeFormatter.parse((String) value)).toEpochMilli(); + } catch (Exception e) { + // malformed + return value; + } + } + } + /** * Generic matcher that supports common matching logic like null values. */ diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java index 7390f846b017a..8350ef3ab7a72 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/matchers/source/SourceMatcher.java @@ -94,6 +94,7 @@ public SourceMatcher( "counted_keyword", new FieldSpecificMatcher.CountedKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) ); + put("boolean", new FieldSpecificMatcher.BooleanMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); } }; this.dynamicFieldMatcher = new DynamicFieldMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 04ea6a73e228e..d67f8c388479e 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -364,7 +364,7 @@ private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBloc var nullValueEncoded = nullValueFormatted != null ? (Number) unsignedToSortableSignedLong(parseUnsignedLong(nullValueFormatted)) : null; - return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<>(nullValueFormatted) { + return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport(nullValueFormatted) { @Override public void convertValue(Object value, List accumulator) { if (value.equals("")) {