Skip to content

Commit 8121cef

Browse files
jordan-powersjordan-powers
authored
Store arrays offsets for scaled float fields natively with synthetic source (elastic#125793) (elastic#125891)
This patch builds on the work in elastic#113757, elastic#122999, elastic#124594, elastic#125529, and elastic#125709 to natively store array offsets for scaled float fields instead of falling back to ignored source when synthetic_source_keep: arrays. (cherry picked from commit 71e74bd) # Conflicts: # server/src/main/java/org/elasticsearch/index/IndexVersions.java
1 parent bd2e4e9 commit 8121cef

File tree

7 files changed

+297
-153
lines changed

7 files changed

+297
-153
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java‎

Lines changed: 111 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import org.elasticsearch.common.settings.Settings;
2020
import org.elasticsearch.common.xcontent.support.XContentMapValues;
2121
import org.elasticsearch.index.IndexMode;
22+
import org.elasticsearch.index.IndexVersion;
23+
import org.elasticsearch.index.IndexVersions;
2224
import org.elasticsearch.index.fielddata.FieldData;
2325
import org.elasticsearch.index.fielddata.FieldDataContext;
2426
import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -32,6 +34,7 @@
3234
import org.elasticsearch.index.mapper.BlockDocValuesReader;
3335
import org.elasticsearch.index.mapper.BlockLoader;
3436
import org.elasticsearch.index.mapper.BlockSourceReader;
37+
import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
3538
import org.elasticsearch.index.mapper.DocumentParserContext;
3639
import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
3740
import org.elasticsearch.index.mapper.FieldMapper;
@@ -40,6 +43,8 @@
4043
import org.elasticsearch.index.mapper.NumberFieldMapper;
4144
import org.elasticsearch.index.mapper.SimpleMappedFieldType;
4245
import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader;
46+
import org.elasticsearch.index.mapper.SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer;
47+
import org.elasticsearch.index.mapper.SourceLoader;
4348
import org.elasticsearch.index.mapper.SourceValueFetcher;
4449
import org.elasticsearch.index.mapper.TextSearchInfo;
4550
import org.elasticsearch.index.mapper.TimeSeriesParams;
@@ -67,6 +72,8 @@
6772
import java.util.Objects;
6873
import java.util.Set;
6974

75+
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
76+
7077
/** A {@link FieldMapper} for scaled floats. Values are internally multiplied
7178
* by a scaling factor and rounded to the closest long. */
7279
public class ScaledFloatFieldMapper extends FieldMapper {
@@ -125,12 +132,34 @@ public static class Builder extends FieldMapper.Builder {
125132
private final Parameter<TimeSeriesParams.MetricType> metric;
126133

127134
private final IndexMode indexMode;
135+
private final IndexVersion indexCreatedVersion;
136+
private final SourceKeepMode indexSourceKeepMode;
128137

129-
public Builder(String name, Settings settings, IndexMode indexMode) {
130-
this(name, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexMode);
138+
public Builder(
139+
String name,
140+
Settings settings,
141+
IndexMode indexMode,
142+
IndexVersion indexCreatedVersion,
143+
SourceKeepMode indexSourceKeepMode
144+
) {
145+
this(
146+
name,
147+
IGNORE_MALFORMED_SETTING.get(settings),
148+
COERCE_SETTING.get(settings),
149+
indexMode,
150+
indexCreatedVersion,
151+
indexSourceKeepMode
152+
);
131153
}
132154

133-
public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDefault, IndexMode indexMode) {
155+
public Builder(
156+
String name,
157+
boolean ignoreMalformedByDefault,
158+
boolean coerceByDefault,
159+
IndexMode indexMode,
160+
IndexVersion indexCreatedVersion,
161+
SourceKeepMode indexSourceKeepMode
162+
) {
134163
super(name);
135164
this.ignoreMalformed = Parameter.explicitBoolParam(
136165
"ignore_malformed",
@@ -159,6 +188,8 @@ public Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDe
159188
);
160189
}
161190
});
191+
this.indexCreatedVersion = indexCreatedVersion;
192+
this.indexSourceKeepMode = indexSourceKeepMode;
162193
}
163194

164195
Builder scalingFactor(double scalingFactor) {
@@ -200,11 +231,35 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) {
200231
coerce.getValue().value(),
201232
context.isSourceSynthetic()
202233
);
203-
return new ScaledFloatFieldMapper(leafName(), type, builderParams(this, context), context.isSourceSynthetic(), this);
234+
String offsetsFieldName = getOffsetsFieldName(
235+
context,
236+
indexSourceKeepMode,
237+
hasDocValues.getValue(),
238+
stored.getValue(),
239+
this,
240+
indexCreatedVersion,
241+
IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY
242+
);
243+
return new ScaledFloatFieldMapper(
244+
leafName(),
245+
type,
246+
builderParams(this, context),
247+
context.isSourceSynthetic(),
248+
this,
249+
offsetsFieldName
250+
);
204251
}
205252
}
206253

207-
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings(), c.getIndexSettings().getMode()));
254+
public static final TypeParser PARSER = new TypeParser(
255+
(n, c) -> new Builder(
256+
n,
257+
c.getSettings(),
258+
c.getIndexSettings().getMode(),
259+
c.indexVersionCreated(),
260+
c.getIndexSettings().sourceKeepMode()
261+
)
262+
);
208263

209264
public static final class ScaledFloatFieldType extends SimpleMappedFieldType {
210265

@@ -532,12 +587,17 @@ public String toString() {
532587
private final TimeSeriesParams.MetricType metricType;
533588
private final IndexMode indexMode;
534589

590+
private final IndexVersion indexCreatedVersion;
591+
private final String offsetsFieldName;
592+
private final SourceKeepMode indexSourceKeepMode;
593+
535594
private ScaledFloatFieldMapper(
536595
String simpleName,
537596
ScaledFloatFieldType mappedFieldType,
538597
BuilderParams builderParams,
539598
boolean isSourceSynthetic,
540-
Builder builder
599+
Builder builder,
600+
String offsetsFieldName
541601
) {
542602
super(simpleName, mappedFieldType, builderParams);
543603
this.isSourceSynthetic = isSourceSynthetic;
@@ -552,6 +612,9 @@ private ScaledFloatFieldMapper(
552612
this.coerceByDefault = builder.coerce.getDefaultValue().value();
553613
this.metricType = builder.metric.getValue();
554614
this.indexMode = builder.indexMode;
615+
this.indexCreatedVersion = builder.indexCreatedVersion;
616+
this.offsetsFieldName = offsetsFieldName;
617+
this.indexSourceKeepMode = builder.indexSourceKeepMode;
555618
}
556619

557620
boolean coerce() {
@@ -563,6 +626,11 @@ public boolean ignoreMalformed() {
563626
return ignoreMalformed.value();
564627
}
565628

629+
@Override
630+
public String getOffsetFieldName() {
631+
return offsetsFieldName;
632+
}
633+
566634
@Override
567635
public ScaledFloatFieldType fieldType() {
568636
return (ScaledFloatFieldType) super.fieldType();
@@ -575,7 +643,9 @@ protected String contentType() {
575643

576644
@Override
577645
public FieldMapper.Builder getMergeBuilder() {
578-
return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode).metric(metricType).init(this);
646+
return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault, indexMode, indexCreatedVersion, indexSourceKeepMode)
647+
.metric(metricType)
648+
.init(this);
579649
}
580650

581651
@Override
@@ -605,11 +675,16 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
605675
value = numericValue;
606676
}
607677

678+
boolean shouldStoreOffsets = offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField();
679+
608680
if (value == null) {
609681
value = nullValue;
610682
}
611683

612684
if (value == null) {
685+
if (shouldStoreOffsets) {
686+
context.getOffSetContext().recordNull(offsetsFieldName);
687+
}
613688
return;
614689
}
615690

@@ -635,6 +710,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
635710

636711
NumberFieldMapper.NumberType.LONG.addFields(context.doc(), fieldType().name(), scaledValue, indexed, hasDocValues, stored);
637712

713+
if (shouldStoreOffsets) {
714+
context.getOffSetContext().recordOffset(offsetsFieldName, scaledValue);
715+
}
716+
638717
if (hasDocValues == false && (indexed || stored)) {
639718
context.addToFieldNames(fieldType().name());
640719
}
@@ -776,17 +855,34 @@ public int docValueCount() {
776855
}
777856
}
778857

858+
private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
859+
if (offsetsFieldName != null) {
860+
var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
861+
layers.add(
862+
new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
863+
fullPath(),
864+
offsetsFieldName,
865+
(b, value) -> b.value(decodeForSyntheticSource(value, scalingFactor))
866+
)
867+
);
868+
if (ignoreMalformed.value()) {
869+
layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
870+
}
871+
return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
872+
} else {
873+
return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
874+
@Override
875+
protected void writeValue(XContentBuilder b, long value) throws IOException {
876+
b.value(decodeForSyntheticSource(value, scalingFactor));
877+
}
878+
};
879+
}
880+
}
881+
779882
@Override
780883
protected SyntheticSourceSupport syntheticSourceSupport() {
781884
if (hasDocValues) {
782-
return new SyntheticSourceSupport.Native(
783-
() -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
784-
@Override
785-
protected void writeValue(XContentBuilder b, long value) throws IOException {
786-
b.value(decodeForSyntheticSource(value, scalingFactor));
787-
}
788-
}
789-
);
885+
return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
790886
}
791887

792888
return super.syntheticSourceSupport();

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.elasticsearch.index.mapper.DocumentMapper;
2020
import org.elasticsearch.index.mapper.DocumentParsingException;
2121
import org.elasticsearch.index.mapper.MappedFieldType;
22+
import org.elasticsearch.index.mapper.Mapper;
2223
import org.elasticsearch.index.mapper.MapperParsingException;
2324
import org.elasticsearch.index.mapper.MapperService;
2425
import org.elasticsearch.index.mapper.NumberFieldMapperTests;
@@ -365,6 +366,24 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed)
365366
return new ScaledFloatSyntheticSourceSupport(ignoreMalformed);
366367
}
367368

369+
@Override
370+
protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
371+
return new ScaledFloatSyntheticSourceSupport(ignoreMalformed) {
372+
@Override
373+
public SyntheticSourceExample example(int maxVals) {
374+
var example = super.example(maxVals);
375+
// Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays
376+
// uses the inputValue as both the input and expected.
377+
return new SyntheticSourceExample(
378+
example.expectedForSyntheticSource(),
379+
example.expectedForSyntheticSource(),
380+
example.expectedForBlockLoader(),
381+
example.mapping()
382+
);
383+
}
384+
};
385+
}
386+
368387
private static class ScaledFloatSyntheticSourceSupport implements SyntheticSourceSupport {
369388
private final boolean ignoreMalformedEnabled;
370389
private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false);

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -222,14 +222,14 @@ public void testFieldData() throws IOException {
222222
}
223223

224224
public void testFetchSourceValue() throws IOException {
225-
MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100)
225+
MappedFieldType mapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100)
226226
.build(MapperBuilderContext.root(false, false))
227227
.fieldType();
228228
assertEquals(List.of(3.14), fetchSourceValue(mapper, 3.1415926));
229229
assertEquals(List.of(3.14), fetchSourceValue(mapper, "3.1415"));
230230
assertEquals(List.of(), fetchSourceValue(mapper, ""));
231231

232-
MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null).scalingFactor(100)
232+
MappedFieldType nullValueMapper = new ScaledFloatFieldMapper.Builder("field", false, false, null, null, null).scalingFactor(100)
233233
.nullValue(2.71)
234234
.build(MapperBuilderContext.root(false, false))
235235
.fieldType();

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatOffsetDocValuesLoaderTests.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper.extras;
11+
12+
import org.elasticsearch.index.mapper.OffsetDocValuesLoaderTestCase;
13+
import org.elasticsearch.plugins.Plugin;
14+
import org.elasticsearch.xcontent.XContentBuilder;
15+
16+
import java.io.IOException;
17+
import java.util.Collection;
18+
19+
import static java.util.Collections.singletonList;
20+
21+
public class ScaledFloatOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
22+
private static final double TEST_SCALING_FACTOR = 10.0;
23+
24+
@Override
25+
protected Collection<? extends Plugin> getPlugins() {
26+
return singletonList(new MapperExtrasPlugin());
27+
}
28+
29+
@Override
30+
protected void minimalMapping(XContentBuilder b) throws IOException {
31+
b.field("type", "scaled_float").field("scaling_factor", TEST_SCALING_FACTOR);
32+
}
33+
34+
public void testOffsetArray() throws Exception {
35+
verifyOffsets("{\"field\":[1.0,10.0,100.0,0.1,10.0,1.0,0.1,100.0]}");
36+
verifyOffsets("{\"field\":[10.0,null,1.0,null,5.0,null,null,6.3,1.5]}");
37+
}
38+
39+
@Override
40+
protected String getFieldTypeName() {
41+
fail("Should not be called because minimalMapping is overridden");
42+
return null;
43+
}
44+
45+
@Override
46+
protected Double randomValue() {
47+
return randomLong() / TEST_SCALING_FACTOR;
48+
}
49+
}

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatSyntheticSourceNativeArrayIntegrationTests.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper.extras;
11+
12+
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
13+
14+
import org.elasticsearch.index.mapper.NativeArrayIntegrationTestCase;
15+
import org.elasticsearch.plugins.Plugin;
16+
import org.elasticsearch.xcontent.XContentBuilder;
17+
18+
import java.io.IOException;
19+
import java.util.Collection;
20+
21+
import static java.util.Collections.singletonList;
22+
23+
public class ScaledFloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
24+
private static final double TEST_SCALING_FACTOR = 10.0;
25+
26+
@Override
27+
protected Collection<Class<? extends Plugin>> getPlugins() {
28+
return singletonList(MapperExtrasPlugin.class);
29+
}
30+
31+
@Override
32+
protected void minimalMapping(XContentBuilder b) throws IOException {
33+
b.field("type", "scaled_float").field("scaling_factor", TEST_SCALING_FACTOR);
34+
}
35+
36+
@Override
37+
protected String getFieldTypeName() {
38+
fail("Should not be called because minimalMapping is overridden");
39+
return null;
40+
}
41+
42+
@Override
43+
protected Object getRandomValue() {
44+
return randomLong() / TEST_SCALING_FACTOR;
45+
}
46+
47+
@Override
48+
protected Object getMalformedValue() {
49+
return randomBoolean() ? RandomStrings.randomAsciiOfLength(random(), 8) : Double.POSITIVE_INFINITY;
50+
}
51+
}

0 commit comments

Comments
 (0)