Skip to content

Commit 147d071

Browse files
lktslkts
committed
Use FallbackSyntheticSourceBlockLoader for number fields
1 parent d93f9c4 commit 147d071

File tree

20 files changed

+440
-34
lines changed

20 files changed

+440
-34
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,8 @@ private static BlockLoader numericBlockLoader(WhereAndBaseName w, NumberFieldMap
261261
null,
262262
false,
263263
null,
264-
null
264+
null,
265+
false
265266
).blockLoader(null);
266267
}
267268

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/FallbackSyntheticSourceBlockLoaderBenchmark.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
package org.elasticsearch.benchmark.index.mapper;
2+
3+
public class FallbackSyntheticSourceBlockLoaderBenchmark {
4+
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public class ScriptScoreBenchmark {
8383
private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
8484

8585
private final Map<String, MappedFieldType> fieldTypes = Map.ofEntries(
86-
Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null))
86+
Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null, false))
8787
);
8888
private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None();
8989
private final CircuitBreakerService breakerService = new NoneCircuitBreakerService();

‎server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java‎

Lines changed: 171 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ public NumberFieldMapper build(MapperBuilderContext context) {
269269
dimension.setValue(true);
270270
}
271271

272-
MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this);
272+
MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this, context.isSourceSynthetic());
273273
hasScript = script.get() != null;
274274
onScriptError = onScriptErrorParam.getValue();
275275
return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this);
@@ -463,6 +463,11 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
463463
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
464464
return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
465465
}
466+
467+
@Override
468+
BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
469+
return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
470+
}
466471
},
467472
FLOAT("float", NumericType.FLOAT) {
468473
@Override
@@ -647,6 +652,11 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
647652
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
648653
return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
649654
}
655+
656+
@Override
657+
BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
658+
return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
659+
}
650660
},
651661
DOUBLE("double", NumericType.DOUBLE) {
652662
@Override
@@ -797,6 +807,11 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
797807
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
798808
return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
799809
}
810+
811+
@Override
812+
BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
813+
return floatingPointBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
814+
}
800815
},
801816
BYTE("byte", NumericType.BYTE) {
802817
@Override
@@ -911,6 +926,11 @@ BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSo
911926
return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
912927
}
913928

929+
@Override
930+
BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
931+
return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
932+
}
933+
914934
private boolean isOutOfRange(Object value) {
915935
double doubleValue = objectToDouble(value);
916936
return doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE;
@@ -1024,6 +1044,11 @@ BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSo
10241044
return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
10251045
}
10261046

1047+
@Override
1048+
BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
1049+
return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
1050+
}
1051+
10271052
private boolean isOutOfRange(Object value) {
10281053
double doubleValue = objectToDouble(value);
10291054
return doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE;
@@ -1210,6 +1235,11 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
12101235
BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
12111236
return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
12121237
}
1238+
1239+
@Override
1240+
BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
1241+
return integerBlockLoaderFromFallbackSyntheticSource(this, fieldName, nullValue, coerce);
1242+
}
12131243
},
12141244
LONG("long", NumericType.LONG) {
12151245
@Override
@@ -1358,6 +1388,26 @@ BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSo
13581388
return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup);
13591389
}
13601390

1391+
@Override
1392+
BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce) {
1393+
var reader = new NumberFallbackSyntheticSourceReader(this, nullValue, coerce) {
1394+
@Override
1395+
public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {
1396+
var builder = (BlockLoader.LongBuilder) blockBuilder;
1397+
for (var value : values) {
1398+
builder.appendLong(value.longValue());
1399+
}
1400+
}
1401+
};
1402+
1403+
return new FallbackSyntheticSourceBlockLoader(reader, fieldName) {
1404+
@Override
1405+
public Builder builder(BlockFactory factory, int expectedCount) {
1406+
return factory.longs(expectedCount);
1407+
}
1408+
};
1409+
}
1410+
13611411
private boolean isOutOfRange(Object value) {
13621412
if (value instanceof Long) {
13631413
return false;
@@ -1626,6 +1676,109 @@ protected void writeValue(XContentBuilder b, long value) throws IOException {
16261676
abstract BlockLoader blockLoaderFromDocValues(String fieldName);
16271677

16281678
abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup);
1679+
1680+
abstract BlockLoader blockLoaderFromFallbackSyntheticSource(String fieldName, Number nullValue, boolean coerce);
1681+
1682+
// All values that fit into integer are returned as integers
1683+
private static BlockLoader integerBlockLoaderFromFallbackSyntheticSource(
1684+
NumberType type,
1685+
String fieldName,
1686+
Number nullValue,
1687+
boolean coerce
1688+
) {
1689+
var reader = new NumberFallbackSyntheticSourceReader(type, nullValue, coerce) {
1690+
@Override
1691+
public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {
1692+
var builder = (BlockLoader.IntBuilder) blockBuilder;
1693+
for (var value : values) {
1694+
builder.appendInt(value.intValue());
1695+
}
1696+
}
1697+
};
1698+
1699+
return new FallbackSyntheticSourceBlockLoader(reader, fieldName) {
1700+
@Override
1701+
public Builder builder(BlockFactory factory, int expectedCount) {
1702+
return factory.ints(expectedCount);
1703+
}
1704+
};
1705+
}
1706+
1707+
// All floating point values are returned as doubles
1708+
private static BlockLoader floatingPointBlockLoaderFromFallbackSyntheticSource(
1709+
NumberType type,
1710+
String fieldName,
1711+
Number nullValue,
1712+
boolean coerce
1713+
) {
1714+
var reader = new NumberFallbackSyntheticSourceReader(type, nullValue, coerce) {
1715+
@Override
1716+
public void writeToBlock(List<Number> values, BlockLoader.Builder blockBuilder) {
1717+
var builder = (BlockLoader.DoubleBuilder) blockBuilder;
1718+
for (var value : values) {
1719+
builder.appendDouble(value.doubleValue());
1720+
}
1721+
}
1722+
};
1723+
1724+
return new FallbackSyntheticSourceBlockLoader(reader, fieldName) {
1725+
@Override
1726+
public Builder builder(BlockFactory factory, int expectedCount) {
1727+
return factory.doubles(expectedCount);
1728+
}
1729+
};
1730+
}
1731+
1732+
abstract static class NumberFallbackSyntheticSourceReader extends FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<
1733+
Number> {
1734+
private final NumberType type;
1735+
private final Number nullValue;
1736+
private final boolean coerce;
1737+
1738+
NumberFallbackSyntheticSourceReader(NumberType type, Number nullValue, boolean coerce) {
1739+
super(nullValue);
1740+
this.type = type;
1741+
this.nullValue = nullValue;
1742+
this.coerce = coerce;
1743+
}
1744+
1745+
@Override
1746+
public void convertValue(Object value, List<Number> accumulator) {
1747+
if (coerce && value.equals("")) {
1748+
if (nullValue != null) {
1749+
accumulator.add(nullValue);
1750+
}
1751+
}
1752+
1753+
try {
1754+
var converted = type.parse(value, coerce);
1755+
accumulator.add(converted);
1756+
} catch (Exception e) {
1757+
// Malformed value, skip it
1758+
}
1759+
}
1760+
1761+
@Override
1762+
public void parseNonNullValue(XContentParser parser, List<Number> accumulator) throws IOException {
1763+
// Aligned with implementation of `value(XContentParser)`
1764+
if (coerce && parser.currentToken() == Token.VALUE_STRING && parser.textLength() == 0) {
1765+
if (nullValue != null) {
1766+
accumulator.add(nullValue);
1767+
}
1768+
}
1769+
1770+
try {
1771+
Number rawValue = type.parse(parser, coerce);
1772+
// Transform number to correct type (e.g. reduce precision)
1773+
accumulator.add(type.parse(rawValue, coerce));
1774+
} catch (Exception e) {
1775+
// Malformed value, skip it./gradlew ":server:test" --tests
1776+
// "org.elasticsearch.index.mapper.blockloader.HalfFloatFieldBlockLoaderTests.testBlockLoader"
1777+
// -Dtests.seed=F720BB3C1E5FC1C0:E76110A0744DB0CB -Dtests.locale=or-IN -Dtests.timezone=America/Miquelon
1778+
// -Druntime.java=23
1779+
}
1780+
}
1781+
};
16291782
}
16301783

16311784
public static class NumberFieldType extends SimpleMappedFieldType {
@@ -1637,6 +1790,7 @@ public static class NumberFieldType extends SimpleMappedFieldType {
16371790
private final boolean isDimension;
16381791
private final MetricType metricType;
16391792
private final IndexMode indexMode;
1793+
private final boolean isSyntheticSource;
16401794

16411795
public NumberFieldType(
16421796
String name,
@@ -1650,7 +1804,8 @@ public NumberFieldType(
16501804
FieldValues<Number> script,
16511805
boolean isDimension,
16521806
MetricType metricType,
1653-
IndexMode indexMode
1807+
IndexMode indexMode,
1808+
boolean isSyntheticSource
16541809
) {
16551810
super(name, isIndexed, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
16561811
this.type = Objects.requireNonNull(type);
@@ -1660,9 +1815,10 @@ public NumberFieldType(
16601815
this.isDimension = isDimension;
16611816
this.metricType = metricType;
16621817
this.indexMode = indexMode;
1818+
this.isSyntheticSource = isSyntheticSource;
16631819
}
16641820

1665-
NumberFieldType(String name, Builder builder) {
1821+
NumberFieldType(String name, Builder builder, boolean isSyntheticSource) {
16661822
this(
16671823
name,
16681824
builder.type,
@@ -1675,7 +1831,8 @@ public NumberFieldType(
16751831
builder.scriptValues(),
16761832
builder.dimension.getValue(),
16771833
builder.metric.getValue(),
1678-
builder.indexMode
1834+
builder.indexMode,
1835+
isSyntheticSource
16791836
);
16801837
}
16811838

@@ -1684,7 +1841,7 @@ public NumberFieldType(String name, NumberType type) {
16841841
}
16851842

16861843
public NumberFieldType(String name, NumberType type, boolean isIndexed) {
1687-
this(name, type, isIndexed, false, true, true, null, Collections.emptyMap(), null, false, null, null);
1844+
this(name, type, isIndexed, false, true, true, null, Collections.emptyMap(), null, false, null, null, false);
16881845
}
16891846

16901847
@Override
@@ -1761,6 +1918,11 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
17611918
if (hasDocValues()) {
17621919
return type.blockLoaderFromDocValues(name());
17631920
}
1921+
1922+
if (isSyntheticSource) {
1923+
return type.blockLoaderFromFallbackSyntheticSource(name(), nullValue, coerce);
1924+
}
1925+
17641926
BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
17651927
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
17661928
: BlockSourceReader.lookupMatchingAll();
@@ -1876,15 +2038,15 @@ public MetricType getMetricType() {
18762038
private final MetricType metricType;
18772039
private boolean allowMultipleValues;
18782040
private final IndexVersion indexCreatedVersion;
1879-
private final boolean storeMalformedFields;
2041+
private final boolean isSyntheticSource;
18802042

18812043
private final IndexMode indexMode;
18822044

18832045
private NumberFieldMapper(
18842046
String simpleName,
18852047
MappedFieldType mappedFieldType,
18862048
BuilderParams builderParams,
1887-
boolean storeMalformedFields,
2049+
boolean isSyntheticSource,
18882050
Builder builder
18892051
) {
18902052
super(simpleName, mappedFieldType, builderParams);
@@ -1904,7 +2066,7 @@ private NumberFieldMapper(
19042066
this.metricType = builder.metric.getValue();
19052067
this.allowMultipleValues = builder.allowMultipleValues;
19062068
this.indexCreatedVersion = builder.indexCreatedVersion;
1907-
this.storeMalformedFields = storeMalformedFields;
2069+
this.isSyntheticSource = isSyntheticSource;
19082070
this.indexMode = builder.indexMode;
19092071
}
19102072

@@ -1939,7 +2101,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
19392101
} catch (IllegalArgumentException e) {
19402102
if (ignoreMalformed.value() && context.parser().currentToken().isValue()) {
19412103
context.addIgnoredField(mappedFieldType.name());
1942-
if (storeMalformedFields) {
2104+
if (isSyntheticSource) {
19432105
// Save a copy of the field so synthetic source can load it
19442106
context.doc().add(IgnoreMalformedStoredValues.storedField(fullPath(), context.parser()));
19452107
}

‎server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,8 @@ public void testRequireDocValuesOnLongs() {
333333
null,
334334
false,
335335
null,
336-
null
336+
null,
337+
false
337338
)
338339
);
339340
}
@@ -353,7 +354,8 @@ public void testRequireDocValuesOnDoubles() {
353354
null,
354355
false,
355356
null,
356-
null
357+
null,
358+
false
357359
)
358360
);
359361
}

‎server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,8 @@ private static MappedFieldType unsearchable() {
140140
null,
141141
false,
142142
null,
143-
null
143+
null,
144+
false
144145
);
145146
}
146147

‎server/src/test/java/org/elasticsearch/index/mapper/blockloader/ByteFieldBlockLoaderTests.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper.blockloader;
11+
12+
import org.elasticsearch.logsdb.datageneration.FieldType;
13+
14+
public class ByteFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Integer> {
15+
public ByteFieldBlockLoaderTests() {
16+
super(FieldType.BYTE);
17+
}
18+
19+
@Override
20+
protected Integer convert(Number value) {
21+
// All values that fit into int are represented as ints
22+
return value.intValue();
23+
}
24+
}

0 commit comments

Comments
 (0)