Store arrays offsets for boolean fields natively with synthetic source

docs/changelog/125529.yaml

@@ -0,0 +1,5 @@
+pr: 125529
+summary: Store arrays offsets for boolean fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -153,6 +153,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_IP = def(9_014_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS = def(9_015_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER = def(9_016_0_00, Version.LUCENE_10_1_0);
+    public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BOOLEAN = def(9_017_0_00, Version.LUCENE_10_1_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.core.Booleans;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -47,6 +48,7 @@
 
 import java.io.IOException;
 import java.time.ZoneId;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -55,6 +57,8 @@
 import java.util.Objects;
 import java.util.Set;
 
+import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
+
 /**
  * A field mapper for boolean fields.
  */
@@ -99,9 +103,17 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
 
         private final IndexVersion indexCreatedVersion;
 
+        private final SourceKeepMode indexSourceKeepMode;
+
         private final Parameter<Boolean> dimension;
 
-        public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion) {
+        public Builder(
+            String name,
+            ScriptCompiler scriptCompiler,
+            boolean ignoreMalformedByDefault,
+            IndexVersion indexCreatedVersion,
+            SourceKeepMode indexSourceKeepMode
+        ) {
             super(name);
             this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
             this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
@@ -126,6 +138,8 @@ public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalform
                     );
                 }
             });
+
+            this.indexSourceKeepMode = indexSourceKeepMode;
         }
 
         public Builder dimension(boolean dimension) {
@@ -165,7 +179,23 @@ public BooleanFieldMapper build(MapperBuilderContext context) {
             );
             hasScript = script.get() != null;
             onScriptError = onScriptErrorParam.getValue();
-            return new BooleanFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this);
+            String offsetsFieldName = getOffsetsFieldName(
+                context,
+                indexSourceKeepMode,
+                docValues.getValue(),
+                stored.getValue(),
+                this,
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BOOLEAN
+            );
+            return new BooleanFieldMapper(
+                leafName(),
+                ft,
+                builderParams(this, context),
+                context.isSourceSynthetic(),
+                this,
+                offsetsFieldName
+            );
         }
 
         private FieldValues<Boolean> scriptValues() {
@@ -182,7 +212,13 @@ private FieldValues<Boolean> scriptValues() {
     }
 
     public static final TypeParser PARSER = createTypeParserWithLegacySupport(
-        (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated())
+        (n, c) -> new Builder(
+            n,
+            c.scriptCompiler(),
+            IGNORE_MALFORMED_SETTING.get(c.getSettings()),
+            c.indexVersionCreated(),
+            c.getIndexSettings().sourceKeepMode()
+        )
     );
 
     public static final class BooleanFieldType extends TermBasedFieldType {
@@ -484,12 +520,16 @@ public Query rangeQuery(
 
     private final boolean storeMalformedFields;
 
+    private final String offsetsFieldName;
+    private final SourceKeepMode indexSourceKeepMode;
+
     protected BooleanFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
         BuilderParams builderParams,
         boolean storeMalformedFields,
-        Builder builder
+        Builder builder,
+        String offsetsFieldName
     ) {
         super(simpleName, mappedFieldType, builderParams);
         this.nullValue = builder.nullValue.getValue();
@@ -503,6 +543,8 @@ protected BooleanFieldMapper(
         this.ignoreMalformed = builder.ignoreMalformed.getValue();
         this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
         this.storeMalformedFields = storeMalformedFields;
+        this.offsetsFieldName = offsetsFieldName;
+        this.indexSourceKeepMode = builder.indexSourceKeepMode;
     }
 
     @Override
@@ -515,6 +557,11 @@ public BooleanFieldType fieldType() {
         return (BooleanFieldType) super.fieldType();
     }
 
+    @Override
+    public String getOffsetFieldName() {
+        return offsetsFieldName;
+    }
+
     @Override
     protected void parseCreateField(DocumentParserContext context) throws IOException {
         if (indexed == false && stored == false && hasDocValues == false) {
@@ -537,12 +584,20 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
                         // Save a copy of the field so synthetic source can load it
                         context.doc().add(IgnoreMalformedStoredValues.storedField(fullPath(), context.parser()));
                     }
+                    return;
                 } else {
                     throw e;
                 }
             }
         }
         indexValue(context, value);
+        if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
+            if (value != null) {
+                context.getOffSetContext().recordOffset(offsetsFieldName, value);
+            } else {
+                context.getOffSetContext().recordNull(offsetsFieldName);
+            }
+        }
     }
 
     private void indexValue(DocumentParserContext context, Boolean value) {
@@ -578,8 +633,9 @@ protected void indexScriptValues(
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(fieldType().isDimension())
-            .init(this);
+        return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension(
+            fieldType().isDimension()
+        ).init(this);
     }
 
     @Override
@@ -601,17 +657,34 @@ protected String contentType() {
         return CONTENT_TYPE;
     }
 
+    private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
+        if (offsetsFieldName != null) {
+            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+            layers.add(
+                new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
+                    fullPath(),
+                    offsetsFieldName,
+                    (b, value) -> b.value(value == 1)
+                )
+            );
+            if (ignoreMalformed.value()) {
+                layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
+            }
+            return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
+        } else {
+            return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
+                @Override
+                protected void writeValue(XContentBuilder b, long value) throws IOException {
+                    b.value(value == 1);
+                }
+            };
+        }
+    }
+
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
         if (hasDocValues) {
-            return new SyntheticSourceSupport.Native(
-                () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) {
-                    @Override
-                    protected void writeValue(XContentBuilder b, long value) throws IOException {
-                        b.value(value == 1);
-                    }
-                }
-            );
+            return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
         }
 
         return super.syntheticSourceSupport();

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -388,7 +388,8 @@ public boolean newDynamicBooleanField(DocumentParserContext context, String name
                     name,
                     ScriptCompiler.NONE,
                     ignoreMalformed,
-                    context.indexSettings().getIndexVersionCreated()
+                    context.indexSettings().getIndexVersionCreated(),
+                    context.indexSettings().sourceKeepMode()
                 ),
                 context
             );

server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java

@@ -312,43 +312,68 @@ protected boolean supportsIgnoreMalformed() {
         return true;
     }
 
-    @Override
-    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
-        return new SyntheticSourceSupport() {
-            Boolean nullValue = usually() ? null : randomBoolean();
+    private class BooleanSyntheticSourceSupport implements SyntheticSourceSupport {
+        Boolean nullValue = usually() ? null : randomBoolean();
+        private boolean ignoreMalformed;
 
-            @Override
-            public SyntheticSourceExample example(int maxVals) throws IOException {
-                if (randomBoolean()) {
-                    Tuple<Boolean, Boolean> v = generateValue();
-                    return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
-                }
-                List<Tuple<Boolean, Boolean>> values = randomList(1, maxVals, this::generateValue);
-                List<Boolean> in = values.stream().map(Tuple::v1).toList();
-                List<Boolean> outList = values.stream().map(Tuple::v2).sorted().toList();
-                Object out = outList.size() == 1 ? outList.get(0) : outList;
-                return new SyntheticSourceExample(in, out, this::mapping);
+        BooleanSyntheticSourceSupport(boolean ignoreMalformed) {
+            this.ignoreMalformed = ignoreMalformed;
+        }
+
+        @Override
+        public SyntheticSourceExample example(int maxVals) throws IOException {
+            if (randomBoolean()) {
+                Tuple<Boolean, Boolean> v = generateValue();
+                return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
             }
+            List<Tuple<Boolean, Boolean>> values = randomList(1, maxVals, this::generateValue);
+            List<Boolean> in = values.stream().map(Tuple::v1).toList();
+            List<Boolean> outList = values.stream().map(Tuple::v2).sorted().toList();
+            Object out = outList.size() == 1 ? outList.get(0) : outList;
+            return new SyntheticSourceExample(in, out, this::mapping);
+        }
 
-            private Tuple<Boolean, Boolean> generateValue() {
-                if (nullValue != null && randomBoolean()) {
-                    return Tuple.tuple(null, nullValue);
-                }
-                boolean b = randomBoolean();
-                return Tuple.tuple(b, b);
+        private Tuple<Boolean, Boolean> generateValue() {
+            if (nullValue != null && randomBoolean()) {
+                return Tuple.tuple(null, nullValue);
             }
+            boolean b = randomBoolean();
+            return Tuple.tuple(b, b);
+        }
 
-            private void mapping(XContentBuilder b) throws IOException {
-                minimalMapping(b);
-                if (nullValue != null) {
-                    b.field("null_value", nullValue);
-                }
-                b.field("ignore_malformed", ignoreMalformed);
+        private void mapping(XContentBuilder b) throws IOException {
+            minimalMapping(b);
+            if (nullValue != null) {
+                b.field("null_value", nullValue);
             }
+            b.field("ignore_malformed", ignoreMalformed);
+        }
+
+        @Override
+        public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
+            return List.of();
+        }
+    };
 
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) {
+        return new BooleanSyntheticSourceSupport(ignoreMalformed);
+    }
+
+    @Override
+    protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode keepMode) {
+        return new BooleanSyntheticSourceSupport(ignoreMalformed) {
             @Override
-            public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
-                return List.of();
+            public SyntheticSourceExample example(int maxVals) throws IOException {
+                var example = super.example(maxVals);
+                // Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays
+                // uses the inputValue as both the input and expected.
+                return new SyntheticSourceExample(
+                    example.expectedForSyntheticSource(),
+                    example.expectedForSyntheticSource(),
+                    example.expectedForBlockLoader(),
+                    example.mapping()
+                );
             }
         };
     }

server/src/test/java/org/elasticsearch/index/mapper/BooleanOffsetDocValuesLoaderTests.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper;
+
+public class BooleanOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
+
+    public void testOffsetArray() throws Exception {
+        verifyOffsets("{\"field\":[true,false,true,true,false,true]}");
+        verifyOffsets("{\"field\":[true,null,false,false,null,null,true,false]}");
+        verifyOffsets("{\"field\":[true,true,true,true]}");
+    }
+
+    public void testOffsetNestedArray() throws Exception {
+        verifyOffsets("{\"field\":[[\"true\",[false,[true]]],[\"true\",false,true]]}", "{\"field\":[true,false,true,true,false,true]}");
+        verifyOffsets(
+            "{\"field\":[true,[null,[[false,false],[null,null]],[true,false]]]}",
+            "{\"field\":[true,null,false,false,null,null,true,false]}"
+        );
+    }
+
+    @Override
+    protected String getFieldTypeName() {
+        return "boolean";
+    }
+
+    @Override
+    protected Object randomValue() {
+        return randomBoolean();
+    }
+}

server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java

@@ -362,7 +362,7 @@ protected Query randomTermsQuery(MappedFieldType ft, SearchExecutionContext ctx)
     }
 
     public void testDualingQueries() throws IOException {
-        BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE, false, IndexVersion.current()).build(
+        BooleanFieldMapper ootb = new BooleanFieldMapper.Builder("foo", ScriptCompiler.NONE, false, IndexVersion.current(), null).build(
             MapperBuilderContext.root(false, false)
         );
         try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {