diff --git a/docs/changelog/125529.yaml b/docs/changelog/125529.yaml new file mode 100644 index 0000000000000..b90327afa16d3 --- /dev/null +++ b/docs/changelog/125529.yaml @@ -0,0 +1,5 @@ +pr: 125529 +summary: Store arrays offsets for boolean fields natively with synthetic source +area: Mapping +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 4a3690fe452c0..a584c3e4ac818 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -153,6 +153,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_IP = def(9_014_0_00, Version.LUCENE_10_1_0); public static final IndexVersion ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS = def(9_015_0_00, Version.LUCENE_10_1_0); public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER = def(9_016_0_00, Version.LUCENE_10_1_0); + public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BOOLEAN = def(9_017_0_00, Version.LUCENE_10_1_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 975c9d84740d4..d0bf0dc348ad5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -27,6 +27,7 @@ import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -47,6 +48,7 @@ import java.io.IOException; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -55,6 +57,8 @@ import java.util.Objects; import java.util.Set; +import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; + /** * A field mapper for boolean fields. */ @@ -99,9 +103,17 @@ public static final class Builder extends FieldMapper.DimensionBuilder { private final IndexVersion indexCreatedVersion; + private final SourceKeepMode indexSourceKeepMode; + private final Parameter dimension; - public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion) { + public Builder( + String name, + ScriptCompiler scriptCompiler, + boolean ignoreMalformedByDefault, + IndexVersion indexCreatedVersion, + SourceKeepMode indexSourceKeepMode + ) { super(name); this.scriptCompiler = Objects.requireNonNull(scriptCompiler); this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion); @@ -126,6 +138,8 @@ public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalform ); } }); + + this.indexSourceKeepMode = indexSourceKeepMode; } public Builder dimension(boolean dimension) { @@ -165,7 +179,23 @@ public BooleanFieldMapper build(MapperBuilderContext context) { ); hasScript = script.get() != null; onScriptError = onScriptErrorParam.getValue(); - return new BooleanFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this); + String offsetsFieldName = getOffsetsFieldName( + context, + indexSourceKeepMode, + docValues.getValue(), + stored.getValue(), + this, + indexCreatedVersion, + IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BOOLEAN + ); + return new BooleanFieldMapper( + leafName(), + ft, + builderParams(this, context), + context.isSourceSynthetic(), + this, + offsetsFieldName + ); } private FieldValues scriptValues() { @@ -182,7 +212,13 @@ private FieldValues scriptValues() { } public static final TypeParser PARSER = createTypeParserWithLegacySupport( - (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated()) + (n, c) -> new Builder( + n, + c.scriptCompiler(), + IGNORE_MALFORMED_SETTING.get(c.getSettings()), + c.indexVersionCreated(), + c.getIndexSettings().sourceKeepMode() + ) ); public static final class BooleanFieldType extends TermBasedFieldType { @@ -484,12 +520,16 @@ public Query rangeQuery( private final boolean storeMalformedFields; + private final String offsetsFieldName; + private final SourceKeepMode indexSourceKeepMode; + protected BooleanFieldMapper( String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams, boolean storeMalformedFields, - Builder builder + Builder builder, + String offsetsFieldName ) { super(simpleName, mappedFieldType, builderParams); this.nullValue = builder.nullValue.getValue(); @@ -503,6 +543,8 @@ protected BooleanFieldMapper( this.ignoreMalformed = builder.ignoreMalformed.getValue(); this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value(); this.storeMalformedFields = storeMalformedFields; + this.offsetsFieldName = offsetsFieldName; + this.indexSourceKeepMode = builder.indexSourceKeepMode; } @Override @@ -515,6 +557,11 @@ public BooleanFieldType fieldType() { return (BooleanFieldType) super.fieldType(); } + @Override + public String getOffsetFieldName() { + return offsetsFieldName; + } + @Override protected void parseCreateField(DocumentParserContext context) throws IOException { if (indexed == false && stored == false && hasDocValues == false) { @@ -537,12 +584,20 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio // Save a copy of the field so synthetic source can load it context.doc().add(IgnoreMalformedStoredValues.storedField(fullPath(), context.parser())); } + return; } else { throw e; } } } indexValue(context, value); + if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) { + if (value != null) { + context.getOffSetContext().recordOffset(offsetsFieldName, value); + } else { + context.getOffSetContext().recordNull(offsetsFieldName); + } + } } private void indexValue(DocumentParserContext context, Boolean value) { @@ -578,8 +633,9 @@ protected void indexScriptValues( @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(fieldType().isDimension()) - .init(this); + return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension( + fieldType().isDimension() + ).init(this); } @Override @@ -601,17 +657,34 @@ protected String contentType() { return CONTENT_TYPE; } + private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() { + if (offsetsFieldName != null) { + var layers = new ArrayList(); + layers.add( + new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer( + fullPath(), + offsetsFieldName, + (b, value) -> b.value(value == 1) + ) + ); + if (ignoreMalformed.value()) { + layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath())); + } + return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers); + } else { + return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(value == 1); + } + }; + } + } + @Override protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - return new SyntheticSourceSupport.Native( - () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(value == 1); - } - } - ); + return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader); } return super.syntheticSourceSupport(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index fb9e24a86b001..fc0b0d864547b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -388,7 +388,8 @@ public boolean newDynamicBooleanField(DocumentParserContext context, String name name, ScriptCompiler.NONE, ignoreMalformed, - context.indexSettings().getIndexVersionCreated() + context.indexSettings().getIndexVersionCreated(), + context.indexSettings().sourceKeepMode() ), context ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 83553503c3c5e..80c862c02aa7c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -312,43 +312,68 @@ protected boolean supportsIgnoreMalformed() { return true; } - @Override - protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { - return new SyntheticSourceSupport() { - Boolean nullValue = usually() ? null : randomBoolean(); + private class BooleanSyntheticSourceSupport implements SyntheticSourceSupport { + Boolean nullValue = usually() ? null : randomBoolean(); + private boolean ignoreMalformed; - @Override - public SyntheticSourceExample example(int maxVals) throws IOException { - if (randomBoolean()) { - Tuple v = generateValue(); - return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping); - } - List> values = randomList(1, maxVals, this::generateValue); - List in = values.stream().map(Tuple::v1).toList(); - List outList = values.stream().map(Tuple::v2).sorted().toList(); - Object out = outList.size() == 1 ? outList.get(0) : outList; - return new SyntheticSourceExample(in, out, this::mapping); + BooleanSyntheticSourceSupport(boolean ignoreMalformed) { + this.ignoreMalformed = ignoreMalformed; + } + + @Override + public SyntheticSourceExample example(int maxVals) throws IOException { + if (randomBoolean()) { + Tuple v = generateValue(); + return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping); } + List> values = randomList(1, maxVals, this::generateValue); + List in = values.stream().map(Tuple::v1).toList(); + List outList = values.stream().map(Tuple::v2).sorted().toList(); + Object out = outList.size() == 1 ? outList.get(0) : outList; + return new SyntheticSourceExample(in, out, this::mapping); + } - private Tuple generateValue() { - if (nullValue != null && randomBoolean()) { - return Tuple.tuple(null, nullValue); - } - boolean b = randomBoolean(); - return Tuple.tuple(b, b); + private Tuple generateValue() { + if (nullValue != null && randomBoolean()) { + return Tuple.tuple(null, nullValue); } + boolean b = randomBoolean(); + return Tuple.tuple(b, b); + } - private void mapping(XContentBuilder b) throws IOException { - minimalMapping(b); - if (nullValue != null) { - b.field("null_value", nullValue); - } - b.field("ignore_malformed", ignoreMalformed); + private void mapping(XContentBuilder b) throws IOException { + minimalMapping(b); + if (nullValue != null) { + b.field("null_value", nullValue); } + b.field("ignore_malformed", ignoreMalformed); + } + + @Override + public List invalidExample() throws IOException { + return List.of(); + } + }; + @Override + protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { + return new BooleanSyntheticSourceSupport(ignoreMalformed); + } + + @Override + protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode keepMode) { + return new BooleanSyntheticSourceSupport(ignoreMalformed) { @Override - public List invalidExample() throws IOException { - return List.of(); + public SyntheticSourceExample example(int maxVals) throws IOException { + var example = super.example(maxVals); + // Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays + // uses the inputValue as both the input and expected. + return new SyntheticSourceExample( + example.expectedForSyntheticSource(), + example.expectedForSyntheticSource(), + example.expectedForBlockLoader(), + example.mapping() + ); } }; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanOffsetDocValuesLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanOffsetDocValuesLoaderTests.java new file mode 100644 index 0000000000000..0c918a0c0cd7a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanOffsetDocValuesLoaderTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +public class BooleanOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase { + + public void testOffsetArray() throws Exception { + verifyOffsets("{\"field\":[true,false,true,true,false,true]}"); + verifyOffsets("{\"field\":[true,null,false,false,null,null,true,false]}"); + verifyOffsets("{\"field\":[true,true,true,true]}"); + } + + public void testOffsetNestedArray() throws Exception { + verifyOffsets("{\"field\":[[\"true\",[false,[true]]],[\"true\",false,true]]}", "{\"field\":[true,false,true,true,false,true]}"); + verifyOffsets( + "{\"field\":[true,[null,[[false,false],[null,null]],[true,false]]]}", + "{\"field\":[true,null,false,false,null,null,true,false]}" + ); + } + + @Override + protected String getFieldTypeName() { + return "boolean"; + } + + @Override + protected Object randomValue() { + return randomBoolean(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java index 4aa983a78b07b..ce9a9bc0688f3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java @@ -362,7 +362,7 @@ protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx) } public void testDualingQueries() throws IOException { - BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE, false, IndexVersion.current()).build( + BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE, false, IndexVersion.current(), null).build( MapperBuilderContext.root(false, false) ); try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanSyntheticSourceNativeArrayIntegrationTests.java new file mode 100644 index 0000000000000..10ed529d70553 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanSyntheticSourceNativeArrayIntegrationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import com.carrotsearch.randomizedtesting.generators.RandomStrings; + +public class BooleanSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + @Override + protected String getFieldTypeName() { + return "boolean"; + } + + @Override + protected Object getRandomValue() { + return randomBoolean(); + } + + @Override + protected Object getMalformedValue() { + return RandomStrings.randomAsciiOfLength(random(), 8); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index e385177b87147..132d24f2389bf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -160,7 +160,7 @@ public void testFieldAliasWithDifferentNestedScopes() { } private static FieldMapper createFieldMapper(String parent, String name) { - return new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current()).build( + return new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current(), null).build( new MapperBuilderContext( parent, false, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsSerializationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsSerializationTests.java index 656a39a028a28..14781f62a23ca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsSerializationTests.java @@ -38,10 +38,10 @@ public void testSorting() { sortedNames.sort(Comparator.naturalOrder()); for (String name : names) { - builder.add(new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current())); + builder.add(new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, false, IndexVersion.current(), null)); } - Mapper.Builder root = new BooleanFieldMapper.Builder("root", ScriptCompiler.NONE, false, IndexVersion.current()); + Mapper.Builder root = new BooleanFieldMapper.Builder("root", ScriptCompiler.NONE, false, IndexVersion.current(), null); FieldMapper.MultiFields multiFields = builder.build(root, MapperBuilderContext.root(false, false)); String serialized = Strings.toString(multiFields);