Skip to content

Commit c205ac2

Browse files
jordan-powersomricohenn
jordan-powers
authored and
omricohenn
committed
Store arrays offsets for boolean fields natively with synthetic source (elastic#125529)
This patch builds on the work in elastic#113757, elastic#122999, and elastic#124594 to natively store array offsets for boolean fields instead of falling back to ignored source when `synthetic_source_keep: arrays`.
1 parent 4b7534b commit c205ac2

File tree

10 files changed

+219
-48
lines changed

10 files changed

+219
-48
lines changed

‎docs/changelog/125529.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 125529
2+
summary: Store arrays offsets for boolean fields natively with synthetic source
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ private static Version parseUnchecked(String version) {
153153
public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_IP = def(9_014_0_00, Version.LUCENE_10_1_0);
154154
public static final IndexVersion ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS = def(9_015_0_00, Version.LUCENE_10_1_0);
155155
public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER = def(9_016_0_00, Version.LUCENE_10_1_0);
156+
public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BOOLEAN = def(9_017_0_00, Version.LUCENE_10_1_0);
156157
/*
157158
* STOP! READ THIS FIRST! No, really,
158159
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java‎

Lines changed: 87 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.elasticsearch.core.Booleans;
2828
import org.elasticsearch.core.Nullable;
2929
import org.elasticsearch.index.IndexVersion;
30+
import org.elasticsearch.index.IndexVersions;
3031
import org.elasticsearch.index.analysis.NamedAnalyzer;
3132
import org.elasticsearch.index.fielddata.FieldDataContext;
3233
import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -47,6 +48,7 @@
4748

4849
import java.io.IOException;
4950
import java.time.ZoneId;
51+
import java.util.ArrayList;
5052
import java.util.Collection;
5153
import java.util.Collections;
5254
import java.util.HashSet;
@@ -55,6 +57,8 @@
5557
import java.util.Objects;
5658
import java.util.Set;
5759

60+
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
61+
5862
/**
5963
* A field mapper for boolean fields.
6064
*/
@@ -99,9 +103,17 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
99103

100104
private final IndexVersion indexCreatedVersion;
101105

106+
private final SourceKeepMode indexSourceKeepMode;
107+
102108
private final Parameter<Boolean> dimension;
103109

104-
public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion) {
110+
public Builder(
111+
String name,
112+
ScriptCompiler scriptCompiler,
113+
boolean ignoreMalformedByDefault,
114+
IndexVersion indexCreatedVersion,
115+
SourceKeepMode indexSourceKeepMode
116+
) {
105117
super(name);
106118
this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
107119
this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
@@ -126,6 +138,8 @@ public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalform
126138
);
127139
}
128140
});
141+
142+
this.indexSourceKeepMode = indexSourceKeepMode;
129143
}
130144

131145
public Builder dimension(boolean dimension) {
@@ -165,7 +179,23 @@ public BooleanFieldMapper build(MapperBuilderContext context) {
165179
);
166180
hasScript = script.get() != null;
167181
onScriptError = onScriptErrorParam.getValue();
168-
return new BooleanFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this);
182+
String offsetsFieldName = getOffsetsFieldName(
183+
context,
184+
indexSourceKeepMode,
185+
docValues.getValue(),
186+
stored.getValue(),
187+
this,
188+
indexCreatedVersion,
189+
IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BOOLEAN
190+
);
191+
return new BooleanFieldMapper(
192+
leafName(),
193+
ft,
194+
builderParams(this, context),
195+
context.isSourceSynthetic(),
196+
this,
197+
offsetsFieldName
198+
);
169199
}
170200

171201
private FieldValues<Boolean> scriptValues() {
@@ -182,7 +212,13 @@ private FieldValues<Boolean> scriptValues() {
182212
}
183213

184214
public static final TypeParser PARSER = createTypeParserWithLegacySupport(
185-
(n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated())
215+
(n, c) -> new Builder(
216+
n,
217+
c.scriptCompiler(),
218+
IGNORE_MALFORMED_SETTING.get(c.getSettings()),
219+
c.indexVersionCreated(),
220+
c.getIndexSettings().sourceKeepMode()
221+
)
186222
);
187223

188224
public static final class BooleanFieldType extends TermBasedFieldType {
@@ -484,12 +520,16 @@ public Query rangeQuery(
484520

485521
private final boolean storeMalformedFields;
486522

523+
private final String offsetsFieldName;
524+
private final SourceKeepMode indexSourceKeepMode;
525+
487526
protected BooleanFieldMapper(
488527
String simpleName,
489528
MappedFieldType mappedFieldType,
490529
BuilderParams builderParams,
491530
boolean storeMalformedFields,
492-
Builder builder
531+
Builder builder,
532+
String offsetsFieldName
493533
) {
494534
super(simpleName, mappedFieldType, builderParams);
495535
this.nullValue = builder.nullValue.getValue();
@@ -503,6 +543,8 @@ protected BooleanFieldMapper(
503543
this.ignoreMalformed = builder.ignoreMalformed.getValue();
504544
this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
505545
this.storeMalformedFields = storeMalformedFields;
546+
this.offsetsFieldName = offsetsFieldName;
547+
this.indexSourceKeepMode = builder.indexSourceKeepMode;
506548
}
507549

508550
@Override
@@ -515,6 +557,11 @@ public BooleanFieldType fieldType() {
515557
return (BooleanFieldType) super.fieldType();
516558
}
517559

560+
@Override
561+
public String getOffsetFieldName() {
562+
return offsetsFieldName;
563+
}
564+
518565
@Override
519566
protected void parseCreateField(DocumentParserContext context) throws IOException {
520567
if (indexed == false && stored == false && hasDocValues == false) {
@@ -537,12 +584,20 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
537584
// Save a copy of the field so synthetic source can load it
538585
context.doc().add(IgnoreMalformedStoredValues.storedField(fullPath(), context.parser()));
539586
}
587+
return;
540588
} else {
541589
throw e;
542590
}
543591
}
544592
}
545593
indexValue(context, value);
594+
if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
595+
if (value != null) {
596+
context.getOffSetContext().recordOffset(offsetsFieldName, value);
597+
} else {
598+
context.getOffSetContext().recordNull(offsetsFieldName);
599+
}
600+
}
546601
}
547602

548603
private void indexValue(DocumentParserContext context, Boolean value) {
@@ -578,8 +633,9 @@ protected void indexScriptValues(
578633

579634
@Override
580635
public FieldMapper.Builder getMergeBuilder() {
581-
return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(fieldType().isDimension())
582-
.init(this);
636+
return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension(
637+
fieldType().isDimension()
638+
).init(this);
583639
}
584640

585641
@Override
@@ -601,17 +657,34 @@ protected String contentType() {
601657
return CONTENT_TYPE;
602658
}
603659

660+
private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
661+
if (offsetsFieldName != null) {
662+
var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
663+
layers.add(
664+
new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
665+
fullPath(),
666+
offsetsFieldName,
667+
(b, value) -> b.value(value == 1)
668+
)
669+
);
670+
if (ignoreMalformed.value()) {
671+
layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
672+
}
673+
return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
674+
} else {
675+
return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
676+
@Override
677+
protected void writeValue(XContentBuilder b, long value) throws IOException {
678+
b.value(value == 1);
679+
}
680+
};
681+
}
682+
}
683+
604684
@Override
605685
protected SyntheticSourceSupport syntheticSourceSupport() {
606686
if (hasDocValues) {
607-
return new SyntheticSourceSupport.Native(
608-
() -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
609-
@Override
610-
protected void writeValue(XContentBuilder b, long value) throws IOException {
611-
b.value(value == 1);
612-
}
613-
}
614-
);
687+
return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
615688
}
616689

617690
return super.syntheticSourceSupport();

‎server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -388,7 +388,8 @@ public boolean newDynamicBooleanField(DocumentParserContext context, String name
388388
name,
389389
ScriptCompiler.NONE,
390390
ignoreMalformed,
391-
context.indexSettings().getIndexVersionCreated()
391+
context.indexSettings().getIndexVersionCreated(),
392+
context.indexSettings().sourceKeepMode()
392393
),
393394
context
394395
);

‎server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java‎

Lines changed: 54 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -312,43 +312,68 @@ protected boolean supportsIgnoreMalformed() {
312312
return true;
313313
}
314314

315-
@Override
316-
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
317-
return new SyntheticSourceSupport() {
318-
Boolean nullValue = usually() ? null : randomBoolean();
315+
private class BooleanSyntheticSourceSupport implements SyntheticSourceSupport {
316+
Boolean nullValue = usually() ? null : randomBoolean();
317+
private boolean ignoreMalformed;
319318

320-
@Override
321-
public SyntheticSourceExample example(int maxVals) throws IOException {
322-
if (randomBoolean()) {
323-
Tuple<Boolean, Boolean> v = generateValue();
324-
return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
325-
}
326-
List<Tuple<Boolean, Boolean>> values = randomList(1, maxVals, this::generateValue);
327-
List<Boolean> in = values.stream().map(Tuple::v1).toList();
328-
List<Boolean> outList = values.stream().map(Tuple::v2).sorted().toList();
329-
Object out = outList.size() == 1 ? outList.get(0) : outList;
330-
return new SyntheticSourceExample(in, out, this::mapping);
319+
BooleanSyntheticSourceSupport(boolean ignoreMalformed) {
320+
this.ignoreMalformed = ignoreMalformed;
321+
}
322+
323+
@Override
324+
public SyntheticSourceExample example(int maxVals) throws IOException {
325+
if (randomBoolean()) {
326+
Tuple<Boolean, Boolean> v = generateValue();
327+
return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
331328
}
329+
List<Tuple<Boolean, Boolean>> values = randomList(1, maxVals, this::generateValue);
330+
List<Boolean> in = values.stream().map(Tuple::v1).toList();
331+
List<Boolean> outList = values.stream().map(Tuple::v2).sorted().toList();
332+
Object out = outList.size() == 1 ? outList.get(0) : outList;
333+
return new SyntheticSourceExample(in, out, this::mapping);
334+
}
332335

333-
private Tuple<Boolean, Boolean> generateValue() {
334-
if (nullValue != null && randomBoolean()) {
335-
return Tuple.tuple(null, nullValue);
336-
}
337-
boolean b = randomBoolean();
338-
return Tuple.tuple(b, b);
336+
private Tuple<Boolean, Boolean> generateValue() {
337+
if (nullValue != null && randomBoolean()) {
338+
return Tuple.tuple(null, nullValue);
339339
}
340+
boolean b = randomBoolean();
341+
return Tuple.tuple(b, b);
342+
}
340343

341-
private void mapping(XContentBuilder b) throws IOException {
342-
minimalMapping(b);
343-
if (nullValue != null) {
344-
b.field("null_value", nullValue);
345-
}
346-
b.field("ignore_malformed", ignoreMalformed);
344+
private void mapping(XContentBuilder b) throws IOException {
345+
minimalMapping(b);
346+
if (nullValue != null) {
347+
b.field("null_value", nullValue);
347348
}
349+
b.field("ignore_malformed", ignoreMalformed);
350+
}
351+
352+
@Override
353+
public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
354+
return List.of();
355+
}
356+
};
348357

358+
@Override
359+
protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
360+
return new BooleanSyntheticSourceSupport(ignoreMalformed);
361+
}
362+
363+
@Override
364+
protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode keepMode) {
365+
return new BooleanSyntheticSourceSupport(ignoreMalformed) {
349366
@Override
350-
public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
351-
return List.of();
367+
public SyntheticSourceExample example(int maxVals) throws IOException {
368+
var example = super.example(maxVals);
369+
// Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays
370+
// uses the inputValue as both the input and expected.
371+
return new SyntheticSourceExample(
372+
example.expectedForSyntheticSource(),
373+
example.expectedForSyntheticSource(),
374+
example.expectedForBlockLoader(),
375+
example.mapping()
376+
);
352377
}
353378
};
354379
}

‎server/src/test/java/org/elasticsearch/index/mapper/BooleanOffsetDocValuesLoaderTests.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
public class BooleanOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
13+
14+
public void testOffsetArray() throws Exception {
15+
verifyOffsets("{\"field\":[true,false,true,true,false,true]}");
16+
verifyOffsets("{\"field\":[true,null,false,false,null,null,true,false]}");
17+
verifyOffsets("{\"field\":[true,true,true,true]}");
18+
}
19+
20+
public void testOffsetNestedArray() throws Exception {
21+
verifyOffsets("{\"field\":[[\"true\",[false,[true]]],[\"true\",false,true]]}", "{\"field\":[true,false,true,true,false,true]}");
22+
verifyOffsets(
23+
"{\"field\":[true,[null,[[false,false],[null,null]],[true,false]]]}",
24+
"{\"field\":[true,null,false,false,null,null,true,false]}"
25+
);
26+
}
27+
28+
@Override
29+
protected String getFieldTypeName() {
30+
return "boolean";
31+
}
32+
33+
@Override
34+
protected Object randomValue() {
35+
return randomBoolean();
36+
}
37+
}

‎server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,7 @@ protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx)
362362
}
363363

364364
public void testDualingQueries() throws IOException {
365-
BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE, false, IndexVersion.current()).build(
365+
BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE, false, IndexVersion.current(), null).build(
366366
MapperBuilderContext.root(false, false)
367367
);
368368
try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {

0 commit comments

Comments
 (0)