Skip to content

Commit 9314b2f

Browse files
jordan-powersjordan-powers
authored
Store arrays offsets for boolean fields natively with synthetic source (elastic#125529) (elastic#125596)
This patch builds on the work in elastic#113757, elastic#122999, and elastic#124594 to natively store array offsets for boolean fields instead of falling back to ignored source when `synthetic_source_keep: arrays`. (cherry picked from commit af1f145) # Conflicts: # server/src/main/java/org/elasticsearch/index/IndexVersions.java # server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
1 parent 1747531 commit 9314b2f

File tree

9 files changed

+218
-48
lines changed

9 files changed

+218
-48
lines changed

‎docs/changelog/125529.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 125529
2+
summary: Store arrays offsets for boolean fields natively with synthetic source
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java‎

Lines changed: 87 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.core.Nullable;
2929
import org.elasticsearch.features.NodeFeature;
3030
import org.elasticsearch.index.IndexVersion;
31+
import org.elasticsearch.index.IndexVersions;
3132
import org.elasticsearch.index.analysis.NamedAnalyzer;
3233
import org.elasticsearch.index.fielddata.FieldDataContext;
3334
import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -48,6 +49,7 @@
4849

4950
import java.io.IOException;
5051
import java.time.ZoneId;
52+
import java.util.ArrayList;
5153
import java.util.Collection;
5254
import java.util.Collections;
5355
import java.util.HashSet;
@@ -56,6 +58,8 @@
5658
import java.util.Objects;
5759
import java.util.Set;
5860

61+
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
62+
5963
/**
6064
* A field mapper for boolean fields.
6165
*/
@@ -102,9 +106,17 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
102106

103107
private final IndexVersion indexCreatedVersion;
104108

109+
private final SourceKeepMode indexSourceKeepMode;
110+
105111
private final Parameter<Boolean> dimension;
106112

107-
public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion) {
113+
public Builder(
114+
String name,
115+
ScriptCompiler scriptCompiler,
116+
boolean ignoreMalformedByDefault,
117+
IndexVersion indexCreatedVersion,
118+
SourceKeepMode indexSourceKeepMode
119+
) {
108120
super(name);
109121
this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
110122
this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
@@ -129,6 +141,8 @@ public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalform
129141
);
130142
}
131143
});
144+
145+
this.indexSourceKeepMode = indexSourceKeepMode;
132146
}
133147

134148
public Builder dimension(boolean dimension) {
@@ -168,7 +182,23 @@ public BooleanFieldMapper build(MapperBuilderContext context) {
168182
);
169183
hasScript = script.get() != null;
170184
onScriptError = onScriptErrorParam.getValue();
171-
return new BooleanFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this);
185+
String offsetsFieldName = getOffsetsFieldName(
186+
context,
187+
indexSourceKeepMode,
188+
docValues.getValue(),
189+
stored.getValue(),
190+
this,
191+
indexCreatedVersion,
192+
IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY
193+
);
194+
return new BooleanFieldMapper(
195+
leafName(),
196+
ft,
197+
builderParams(this, context),
198+
context.isSourceSynthetic(),
199+
this,
200+
offsetsFieldName
201+
);
172202
}
173203

174204
private FieldValues<Boolean> scriptValues() {
@@ -187,7 +217,13 @@ private FieldValues<Boolean> scriptValues() {
187217
private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099);
188218

189219
public static final TypeParser PARSER = new TypeParser(
190-
(n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated()),
220+
(n, c) -> new Builder(
221+
n,
222+
c.scriptCompiler(),
223+
IGNORE_MALFORMED_SETTING.get(c.getSettings()),
224+
c.indexVersionCreated(),
225+
c.getIndexSettings().sourceKeepMode()
226+
),
191227
MINIMUM_COMPATIBILITY_VERSION
192228
);
193229

@@ -490,12 +526,16 @@ public Query rangeQuery(
490526

491527
private final boolean storeMalformedFields;
492528

529+
private final String offsetsFieldName;
530+
private final SourceKeepMode indexSourceKeepMode;
531+
493532
protected BooleanFieldMapper(
494533
String simpleName,
495534
MappedFieldType mappedFieldType,
496535
BuilderParams builderParams,
497536
boolean storeMalformedFields,
498-
Builder builder
537+
Builder builder,
538+
String offsetsFieldName
499539
) {
500540
super(simpleName, mappedFieldType, builderParams);
501541
this.nullValue = builder.nullValue.getValue();
@@ -509,6 +549,8 @@ protected BooleanFieldMapper(
509549
this.ignoreMalformed = builder.ignoreMalformed.getValue();
510550
this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
511551
this.storeMalformedFields = storeMalformedFields;
552+
this.offsetsFieldName = offsetsFieldName;
553+
this.indexSourceKeepMode = builder.indexSourceKeepMode;
512554
}
513555

514556
@Override
@@ -521,6 +563,11 @@ public BooleanFieldType fieldType() {
521563
return (BooleanFieldType) super.fieldType();
522564
}
523565

566+
@Override
567+
public String getOffsetFieldName() {
568+
return offsetsFieldName;
569+
}
570+
524571
@Override
525572
protected void parseCreateField(DocumentParserContext context) throws IOException {
526573
if (indexed == false && stored == false && hasDocValues == false) {
@@ -543,12 +590,20 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
543590
// Save a copy of the field so synthetic source can load it
544591
context.doc().add(IgnoreMalformedStoredValues.storedField(fullPath(), context.parser()));
545592
}
593+
return;
546594
} else {
547595
throw e;
548596
}
549597
}
550598
}
551599
indexValue(context, value);
600+
if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
601+
if (value != null) {
602+
context.getOffSetContext().recordOffset(offsetsFieldName, value);
603+
} else {
604+
context.getOffSetContext().recordNull(offsetsFieldName);
605+
}
606+
}
552607
}
553608

554609
private void indexValue(DocumentParserContext context, Boolean value) {
@@ -584,8 +639,9 @@ protected void indexScriptValues(
584639

585640
@Override
586641
public FieldMapper.Builder getMergeBuilder() {
587-
return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(fieldType().isDimension())
588-
.init(this);
642+
return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension(
643+
fieldType().isDimension()
644+
).init(this);
589645
}
590646

591647
@Override
@@ -607,17 +663,34 @@ protected String contentType() {
607663
return CONTENT_TYPE;
608664
}
609665

666+
private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
667+
if (offsetsFieldName != null) {
668+
var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
669+
layers.add(
670+
new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
671+
fullPath(),
672+
offsetsFieldName,
673+
(b, value) -> b.value(value == 1)
674+
)
675+
);
676+
if (ignoreMalformed.value()) {
677+
layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
678+
}
679+
return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
680+
} else {
681+
return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
682+
@Override
683+
protected void writeValue(XContentBuilder b, long value) throws IOException {
684+
b.value(value == 1);
685+
}
686+
};
687+
}
688+
}
689+
610690
@Override
611691
protected SyntheticSourceSupport syntheticSourceSupport() {
612692
if (hasDocValues) {
613-
return new SyntheticSourceSupport.Native(
614-
() -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
615-
@Override
616-
protected void writeValue(XContentBuilder b, long value) throws IOException {
617-
b.value(value == 1);
618-
}
619-
}
620-
);
693+
return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
621694
}
622695

623696
return super.syntheticSourceSupport();

‎server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,8 @@ public boolean newDynamicBooleanField(DocumentParserContext context, String name
387387
name,
388388
ScriptCompiler.NONE,
389389
ignoreMalformed,
390-
context.indexSettings().getIndexVersionCreated()
390+
context.indexSettings().getIndexVersionCreated(),
391+
context.indexSettings().sourceKeepMode()
391392
),
392393
context
393394
);

‎server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java‎

Lines changed: 54 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -312,43 +312,68 @@ protected boolean supportsIgnoreMalformed() {
312312
return true;
313313
}
314314

315-
@Override
316-
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
317-
return new SyntheticSourceSupport() {
318-
Boolean nullValue = usually() ? null : randomBoolean();
315+
private class BooleanSyntheticSourceSupport implements SyntheticSourceSupport {
316+
Boolean nullValue = usually() ? null : randomBoolean();
317+
private boolean ignoreMalformed;
319318

320-
@Override
321-
public SyntheticSourceExample example(int maxVals) throws IOException {
322-
if (randomBoolean()) {
323-
Tuple<Boolean, Boolean> v = generateValue();
324-
return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
325-
}
326-
List<Tuple<Boolean, Boolean>> values = randomList(1, maxVals, this::generateValue);
327-
List<Boolean> in = values.stream().map(Tuple::v1).toList();
328-
List<Boolean> outList = values.stream().map(Tuple::v2).sorted().toList();
329-
Object out = outList.size() == 1 ? outList.get(0) : outList;
330-
return new SyntheticSourceExample(in, out, this::mapping);
319+
BooleanSyntheticSourceSupport(boolean ignoreMalformed) {
320+
this.ignoreMalformed = ignoreMalformed;
321+
}
322+
323+
@Override
324+
public SyntheticSourceExample example(int maxVals) throws IOException {
325+
if (randomBoolean()) {
326+
Tuple<Boolean, Boolean> v = generateValue();
327+
return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
331328
}
329+
List<Tuple<Boolean, Boolean>> values = randomList(1, maxVals, this::generateValue);
330+
List<Boolean> in = values.stream().map(Tuple::v1).toList();
331+
List<Boolean> outList = values.stream().map(Tuple::v2).sorted().toList();
332+
Object out = outList.size() == 1 ? outList.get(0) : outList;
333+
return new SyntheticSourceExample(in, out, this::mapping);
334+
}
332335

333-
private Tuple<Boolean, Boolean> generateValue() {
334-
if (nullValue != null && randomBoolean()) {
335-
return Tuple.tuple(null, nullValue);
336-
}
337-
boolean b = randomBoolean();
338-
return Tuple.tuple(b, b);
336+
private Tuple<Boolean, Boolean> generateValue() {
337+
if (nullValue != null && randomBoolean()) {
338+
return Tuple.tuple(null, nullValue);
339339
}
340+
boolean b = randomBoolean();
341+
return Tuple.tuple(b, b);
342+
}
340343

341-
private void mapping(XContentBuilder b) throws IOException {
342-
minimalMapping(b);
343-
if (nullValue != null) {
344-
b.field("null_value", nullValue);
345-
}
346-
b.field("ignore_malformed", ignoreMalformed);
344+
private void mapping(XContentBuilder b) throws IOException {
345+
minimalMapping(b);
346+
if (nullValue != null) {
347+
b.field("null_value", nullValue);
347348
}
349+
b.field("ignore_malformed", ignoreMalformed);
350+
}
351+
352+
@Override
353+
public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
354+
return List.of();
355+
}
356+
};
348357

358+
@Override
359+
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
360+
return new BooleanSyntheticSourceSupport(ignoreMalformed);
361+
}
362+
363+
@Override
364+
protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode keepMode) {
365+
return new BooleanSyntheticSourceSupport(ignoreMalformed) {
349366
@Override
350-
public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
351-
return List.of();
367+
public SyntheticSourceExample example(int maxVals) throws IOException {
368+
var example = super.example(maxVals);
369+
// Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays
370+
// uses the inputValue as both the input and expected.
371+
return new SyntheticSourceExample(
372+
example.expectedForSyntheticSource(),
373+
example.expectedForSyntheticSource(),
374+
example.expectedForBlockLoader(),
375+
example.mapping()
376+
);
352377
}
353378
};
354379
}

‎server/src/test/java/org/elasticsearch/index/mapper/BooleanOffsetDocValuesLoaderTests.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
public class BooleanOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
13+
14+
public void testOffsetArray() throws Exception {
15+
verifyOffsets("{\"field\":[true,false,true,true,false,true]}");
16+
verifyOffsets("{\"field\":[true,null,false,false,null,null,true,false]}");
17+
verifyOffsets("{\"field\":[true,true,true,true]}");
18+
}
19+
20+
public void testOffsetNestedArray() throws Exception {
21+
verifyOffsets("{\"field\":[[\"true\",[false,[true]]],[\"true\",false,true]]}", "{\"field\":[true,false,true,true,false,true]}");
22+
verifyOffsets(
23+
"{\"field\":[true,[null,[[false,false],[null,null]],[true,false]]]}",
24+
"{\"field\":[true,null,false,false,null,null,true,false]}"
25+
);
26+
}
27+
28+
@Override
29+
protected String getFieldTypeName() {
30+
return "boolean";
31+
}
32+
33+
@Override
34+
protected Object randomValue() {
35+
return randomBoolean();
36+
}
37+
}

‎server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx)
354354
}
355355

356356
public void testDualingQueries() throws IOException {
357-
BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE, false, IndexVersion.current()).build(
357+
BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE, false, IndexVersion.current(), null).build(
358358
MapperBuilderContext.root(false, false)
359359
);
360360
try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {

‎server/src/test/java/org/elasticsearch/index/mapper/BooleanSyntheticSourceNativeArrayIntegrationTests.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
13+
14+
public class BooleanSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase {
15+
@Override
16+
protected String getFieldTypeName() {
17+
return "boolean";
18+
}
19+
20+
@Override
21+
protected Object getRandomValue() {
22+
return randomBoolean();
23+
}
24+
25+
@Override
26+
protected Object getMalformedValue() {
27+
return RandomStrings.randomAsciiOfLength(random(), 8);
28+
}
29+
}

0 commit comments

Comments
 (0)