diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java index 9ed5e1accef59..fba3c752bb239 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java @@ -261,7 +261,8 @@ private static BlockLoader numericBlockLoader(WhereAndBaseName w, NumberFieldMap null, false, null, - null + null, + false ).blockLoader(null); } diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 4085e74d35db6..e61171aeff027 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -83,7 +83,7 @@ public class ScriptScoreBenchmark { private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList()); private final Map fieldTypes = Map.ofEntries( - Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null)) + Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null, false)) ); private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None(); private final CircuitBreakerService breakerService = new NoneCircuitBreakerService(); diff --git a/docs/changelog/122280.yaml b/docs/changelog/122280.yaml new file mode 100644 index 0000000000000..93a7e4e1aaf57 --- /dev/null +++ b/docs/changelog/122280.yaml @@ -0,0 +1,5 @@ +pr: 122280 +summary: Use `FallbackSyntheticSourceBlockLoader` for number fields +area: Mapping +type: enhancement +issues: [] diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index fd0098851c5f8..37ad8d128fad4 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -86,7 +86,8 @@ public TokenCountFieldMapper build(MapperBuilderContext context) { store.getValue(), hasDocValues.getValue(), nullValue.getValue(), - meta.getValue() + meta.getValue(), + context.isSourceSynthetic() ); return new TokenCountFieldMapper(leafName(), ft, builderParams(this, context), this); } @@ -100,7 +101,8 @@ static class TokenCountFieldType extends NumberFieldMapper.NumberFieldType { boolean isStored, boolean hasDocValues, Number nullValue, - Map meta + Map meta, + boolean isSyntheticSource ) { super( name, @@ -114,7 +116,8 @@ static class TokenCountFieldType extends NumberFieldMapper.NumberFieldType { null, false, null, - null + null, + isSyntheticSource ); } diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index 68b93db39646f..11dbf34f6c791 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -50,7 +50,7 @@ public SizeFieldMapper build() { private static class SizeFieldType extends NumberFieldType { SizeFieldType() { - super(NAME, NumberType.INTEGER, true, true, true, false, null, Collections.emptyMap(), null, false, null, null); + super(NAME, NumberType.INTEGER, true, true, true, false, null, Collections.emptyMap(), null, false, null, null, false); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 6b4e87a70ab9e..27339430efd5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -270,7 +270,7 @@ public NumberFieldMapper build(MapperBuilderContext context) { dimension.setValue(true); } - MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this); + MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this, context.isSourceSynthetic()); hasScript = script.get() != null; onScriptError = onScriptErrorParam.getValue(); return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this); @@ -464,6 +464,11 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); } + + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } }, FLOAT("float", NumericType.FLOAT) { @Override @@ -648,6 +653,11 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); } + + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } }, DOUBLE("double", NumericType.DOUBLE) { @Override @@ -798,6 +808,11 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); } + + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } }, BYTE("byte", NumericType.BYTE) { @Override @@ -912,6 +927,11 @@ BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSo return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); } + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } + private boolean isOutOfRange(Object value) { double doubleValue = objectToDouble(value); return doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE; @@ -1025,6 +1045,11 @@ BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSo return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); } + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } + private boolean isOutOfRange(Object value) { double doubleValue = objectToDouble(value); return doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE; @@ -1211,6 +1236,11 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); } + + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce); + } }, LONG("long", NumericType.LONG) { @Override @@ -1359,6 +1389,26 @@ BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSo return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup); } + @Override + BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) { + var reader = new NumberFallbackSyntheticSourceReader(this, nullValue, coerce) { + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + var builder = (BlockLoader.LongBuilder) blockBuilder; + for (var value : values) { + builder.appendLong(value.longValue()); + } + } + }; + + return new FallbackSyntheticSourceBlockLoader(reader, fieldName) { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + }; + } + private boolean isOutOfRange(Object value) { if (value instanceof Long) { return false; @@ -1635,6 +1685,106 @@ protected void writeValue(XContentBuilder b, long value) throws IOException { abstract BlockLoader blockLoaderFromDocValues(String fieldName); abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup); + + abstract BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce); + + // All values that fit into integer are returned as integers + private static BlockLoader integerBlockLoaderFromFallbackSyntheticSource( + NumberType type, + String fieldName, + Number nullValue, + boolean coerce + ) { + var reader = new NumberFallbackSyntheticSourceReader(type, nullValue, coerce) { + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + var builder = (BlockLoader.IntBuilder) blockBuilder; + for (var value : values) { + builder.appendInt(value.intValue()); + } + } + }; + + return new FallbackSyntheticSourceBlockLoader(reader, fieldName) { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.ints(expectedCount); + } + }; + } + + // All floating point values are returned as doubles + private static BlockLoader floatingPointBlockLoaderFromFallbackSyntheticSource( + NumberType type, + String fieldName, + Number nullValue, + boolean coerce + ) { + var reader = new NumberFallbackSyntheticSourceReader(type, nullValue, coerce) { + @Override + public void writeToBlock(List values, BlockLoader.Builder blockBuilder) { + var builder = (BlockLoader.DoubleBuilder) blockBuilder; + for (var value : values) { + builder.appendDouble(value.doubleValue()); + } + } + }; + + return new FallbackSyntheticSourceBlockLoader(reader, fieldName) { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.doubles(expectedCount); + } + }; + } + + abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport< + Number> { + private final NumberType type; + private final Number nullValue; + private final boolean coerce; + + NumberFallbackSyntheticSourceReader(NumberType type, Number nullValue, boolean coerce) { + super(nullValue); + this.type = type; + this.nullValue = nullValue; + this.coerce = coerce; + } + + @Override + public void convertValue(Object value, List accumulator) { + if (coerce && value.equals("")) { + if (nullValue != null) { + accumulator.add(nullValue); + } + } + + try { + var converted = type.parse(value, coerce); + accumulator.add(converted); + } catch (Exception e) { + // Malformed value, skip it + } + } + + @Override + public void parseNonNullValue(XContentParser parser, List accumulator) throws IOException { + // Aligned with implementation of `value(XContentParser)` + if (coerce && parser.currentToken() == Token.VALUE_STRING && parser.textLength() == 0) { + if (nullValue != null) { + accumulator.add(nullValue); + } + } + + try { + Number rawValue = type.parse(parser, coerce); + // Transform number to correct type (e.g. reduce precision) + accumulator.add(type.parse(rawValue, coerce)); + } catch (Exception e) { + // Malformed value, skip it + } + } + }; } public static class NumberFieldType extends SimpleMappedFieldType { @@ -1646,6 +1796,7 @@ public static class NumberFieldType extends SimpleMappedFieldType { private final boolean isDimension; private final MetricType metricType; private final IndexMode indexMode; + private final boolean isSyntheticSource; public NumberFieldType( String name, @@ -1659,7 +1810,8 @@ public NumberFieldType( FieldValues script, boolean isDimension, MetricType metricType, - IndexMode indexMode + IndexMode indexMode, + boolean isSyntheticSource ) { super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); this.type = Objects.requireNonNull(type); @@ -1669,9 +1821,10 @@ public NumberFieldType( this.isDimension = isDimension; this.metricType = metricType; this.indexMode = indexMode; + this.isSyntheticSource = isSyntheticSource; } - NumberFieldType(String name, Builder builder) { + NumberFieldType(String name, Builder builder, boolean isSyntheticSource) { this( name, builder.type, @@ -1684,7 +1837,8 @@ public NumberFieldType( builder.scriptValues(), builder.dimension.getValue(), builder.metric.getValue(), - builder.indexMode + builder.indexMode, + isSyntheticSource ); } @@ -1693,7 +1847,7 @@ public NumberFieldType(String name, NumberType type) { } public NumberFieldType(String name, NumberType type, boolean isIndexed) { - this(name, type, isIndexed, false, true, true, null, Collections.emptyMap(), null, false, null, null); + this(name, type, isIndexed, false, true, true, null, Collections.emptyMap(), null, false, null, null, false); } @Override @@ -1770,6 +1924,11 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { return type.blockLoaderFromDocValues(name()); } + + if (isSyntheticSource) { + return type.blockLoaderFromFallbackSyntheticSource(name(), nullValue, coerce); + } + BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); @@ -1885,7 +2044,7 @@ public MetricType getMetricType() { private final MetricType metricType; private boolean allowMultipleValues; private final IndexVersion indexCreatedVersion; - private final boolean storeMalformedFields; + private final boolean isSyntheticSource; private final IndexMode indexMode; @@ -1893,7 +2052,7 @@ private NumberFieldMapper( String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams, - boolean storeMalformedFields, + boolean isSyntheticSource, Builder builder ) { super(simpleName, mappedFieldType, builderParams); @@ -1913,7 +2072,7 @@ private NumberFieldMapper( this.metricType = builder.metric.getValue(); this.allowMultipleValues = builder.allowMultipleValues; this.indexCreatedVersion = builder.indexCreatedVersion; - this.storeMalformedFields = storeMalformedFields; + this.isSyntheticSource = isSyntheticSource; this.indexMode = builder.indexMode; } @@ -1948,7 +2107,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } catch (IllegalArgumentException e) { if (ignoreMalformed.value() && context.parser().currentToken().isValue()) { context.addIgnoredField(mappedFieldType.name()); - if (storeMalformedFields) { + if (isSyntheticSource) { // Save a copy of the field so synthetic source can load it context.doc().add(IgnoreMalformedStoredValues.storedField(fullPath(), context.parser())); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 36c25b352a792..172545ab459c2 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -333,7 +333,8 @@ public void testRequireDocValuesOnLongs() { null, false, null, - null + null, + false ) ); } @@ -353,7 +354,8 @@ public void testRequireDocValuesOnDoubles() { null, false, null, - null + null, + false ) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index a8b935c79ccc0..164e0232bf409 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -140,7 +140,8 @@ private static MappedFieldType unsearchable() { null, false, null, - null + null, + false ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..28d7cbcfb42db --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class ByteFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public ByteFieldBlockLoaderTests() { + super(FieldType.BYTE); + } + + @Override + protected Integer convert(Number value) { + // All values that fit into int are represented as ints + return value.intValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..e0b62b21ad87a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DoubleFieldBlockLoaderTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class DoubleFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public DoubleFieldBlockLoaderTests() { + super(FieldType.DOUBLE); + } + + @Override + protected Double convert(Number value) { + return value.doubleValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..63439a97d7c9d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/FloatFieldBlockLoaderTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class FloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public FloatFieldBlockLoaderTests() { + super(FieldType.FLOAT); + } + + @Override + protected Double convert(Number value) { + // All float values are represented as double + return value.doubleValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..1e8cedb734af3 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/HalfFloatFieldBlockLoaderTests.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class HalfFloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public HalfFloatFieldBlockLoaderTests() { + super(FieldType.HALF_FLOAT); + } + + @Override + protected Double convert(Number value) { + // All float values are represented as double + return (double) HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(value.floatValue())); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..5d7b9d78442cb --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/IntegerFieldBlockLoaderTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class IntegerFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public IntegerFieldBlockLoaderTests() { + super(FieldType.INTEGER); + } + + @Override + protected Integer convert(Number value) { + return value.intValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java index 4d5eb2ea641ae..909cccf9e7d54 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java @@ -59,18 +59,6 @@ protected Object expected(Map fieldMapping, Object value, boolea return maybeFoldList(resultList); } - private Object maybeFoldList(List list) { - if (list.isEmpty()) { - return null; - } - - if (list.size() == 1) { - return list.get(0); - } - - return list; - } - private BytesRef convert(String value, String nullValue, int ignoreAbove) { if (value == null) { if (nullValue != null) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..ff953294fb618 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/LongFieldBlockLoaderTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class LongFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public LongFieldBlockLoaderTests() { + super(FieldType.LONG); + } + + @Override + protected Long convert(Number value) { + return value.longValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/NumberFieldBlockLoaderTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/NumberFieldBlockLoaderTestCase.java new file mode 100644 index 0000000000000..e523d011c3ab1 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/NumberFieldBlockLoaderTestCase.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.index.mapper.BlockLoaderTestCase; +import org.elasticsearch.logsdb.datageneration.FieldType; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public abstract class NumberFieldBlockLoaderTestCase extends BlockLoaderTestCase { + public NumberFieldBlockLoaderTestCase(FieldType fieldType) { + super(fieldType); + } + + @Override + @SuppressWarnings("unchecked") + protected Object expected(Map fieldMapping, Object value, boolean syntheticSource) { + var nullValue = fieldMapping.get("null_value") != null ? convert((Number) fieldMapping.get("null_value")) : null; + + if (value instanceof List == false) { + return convert(value, nullValue); + } + + if ((boolean) fieldMapping.getOrDefault("doc_values", false)) { + // Sorted and no duplicates + var resultList = ((List) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).sorted().toList(); + return maybeFoldList(resultList); + } + + // parsing from source + var resultList = ((List) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).toList(); + return maybeFoldList(resultList); + } + + @SuppressWarnings("unchecked") + private T convert(Object value, T nullValue) { + if (value == null) { + return nullValue; + } + // String coercion is true by default + if (value instanceof String s && s.isEmpty()) { + return nullValue; + } + if (value instanceof Number n) { + return convert(n); + } + + // Malformed values are excluded + return null; + } + + protected abstract T convert(Number value); +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java new file mode 100644 index 0000000000000..a40bc1c404f45 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/ShortFieldBlockLoaderTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.logsdb.datageneration.FieldType; + +public class ShortFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase { + public ShortFieldBlockLoaderTests() { + super(FieldType.SHORT); + } + + @Override + protected Integer convert(Number value) { + // All values that fit into int are represented as ints + return value.intValue(); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java index 2d0622dbb6322..230495db7327b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java @@ -92,7 +92,8 @@ private ValuesSourceConfig getVSConfig( null, false, null, - null + null, + false ); return ValuesSourceConfig.resolveFieldOnly(ft, context); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java index ba186695bcdae..e7d19c0f56dbc 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregatorTests.java @@ -1494,7 +1494,8 @@ public void testDocValuesFieldExistsForNumber() throws IOException { null, false, null, - null + null, + false ); docValuesFieldExistsTestCase(new ExistsQueryBuilder("f"), ft, true, i -> { final LuceneDocument document = new LuceneDocument(); @@ -1517,7 +1518,8 @@ public void testDocValuesFieldExistsForNumberWithoutData() throws IOException { null, false, null, - null + null, + false ) ); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index 32831f46c7a1b..ab92ea8593445 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -311,7 +311,8 @@ public void testUnboundedRanges() throws IOException { null, false, null, - null + null, + false ) ) ); @@ -426,7 +427,8 @@ public void testNotFitIntoDouble() throws IOException { null, false, null, - null + null, + false ); long start = 2L << 54; // Double stores 53 bits of mantissa, so we aggregate a bunch of bigger values @@ -707,7 +709,8 @@ private void testCase( null, false, null, - null + null, + false ); RangeAggregationBuilder aggregationBuilder = new RangeAggregationBuilder("test_range_agg"); aggregationBuilder.field(NUMBER_FIELD_NAME); diff --git a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java index 1261c8300902f..336fae2cfec2f 100644 --- a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java @@ -154,7 +154,8 @@ public void testBuild() throws IOException { null, false, null, - null + null, + false ); when(searchExecutionContext.getFieldType("field")).thenReturn(numberFieldType); IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> builder.build(searchExecutionContext)); @@ -172,7 +173,8 @@ public void testBuild() throws IOException { null, false, null, - null + null, + false ); when(searchExecutionContext.getFieldType("field")).thenReturn(numberFieldType); builder.setInnerHits(new InnerHitBuilder().setName("field")); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java index db8a38c63c64f..a7595cf52297b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java @@ -145,6 +145,18 @@ private void processLevel(Map level, String field, ArrayList list) { + if (list.isEmpty()) { + return null; + } + + if (list.size() == 1) { + return list.get(0); + } + + return list; + } + private Object setupAndInvokeBlockLoader(MapperService mapperService, XContentBuilder document, String fieldName) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 71b77c604690a..2e6e7898f3f2f 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -197,7 +197,8 @@ private MappedFieldType counterField(String name) { null, false, TimeSeriesParams.MetricType.COUNTER, - IndexMode.TIME_SERIES + IndexMode.TIME_SERIES, + false ); }