Natively store synthetic source array offsets for numeric fields (elastic#124594)

This patch builds on the work in elastic#122999 and elastic#113757 to natively store
array offsets for numeric fields instead of falling back to ignored source
when `source_keep_mode: arrays`.

Author: jordan-powers

docs/changelog/124594.yaml

@@ -0,0 +1,5 @@
+pr: 124594
+summary: Store arrays offsets for numeric fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -152,6 +152,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD = def(9_013_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_IP = def(9_014_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS = def(9_015_0_00, Version.LUCENE_10_1_0);
+    public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER = def(9_016_0_00, Version.LUCENE_10_1_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -353,7 +353,8 @@ public boolean newDynamicLongField(DocumentParserContext context, String name) t
                     ScriptCompiler.NONE,
                     context.indexSettings().getSettings(),
                     context.indexSettings().getIndexVersionCreated(),
-                    context.indexSettings().getMode()
+                    context.indexSettings().getMode(),
+                    context.indexSettings().sourceKeepMode()
                 ),
                 context
             );
@@ -371,7 +372,8 @@ public boolean newDynamicDoubleField(DocumentParserContext context, String name)
                     ScriptCompiler.NONE,
                     context.indexSettings().getSettings(),
                     context.indexSettings().getIndexVersionCreated(),
-                    context.indexSettings().getMode()
+                    context.indexSettings().getMode(),
+                    context.indexSettings().sourceKeepMode()
                 ),
                 context
             );

server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java

@@ -97,6 +97,7 @@ static String getOffsetsFieldName(
             && sourceKeepMode == Mapper.SourceKeepMode.ARRAYS
             && hasDocValues
             && isStored == false
+            && context.isInNestedContext() == false
             && fieldMapperBuilder.copyTo.copyToFields().isEmpty()
             && fieldMapperBuilder.multiFieldsBuilder.hasMultiFields() == false
             && indexVersionSupportStoringArraysNatively(indexCreatedVersion, minSupportedVersionMain)) {

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -37,6 +37,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
@@ -70,6 +71,7 @@
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.time.ZoneId;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -80,6 +82,8 @@
 import java.util.function.BiFunction;
 import java.util.function.Function;
 
+import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
+
 /** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */
 public class NumberFieldMapper extends FieldMapper {
 
@@ -127,20 +131,31 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
         private final IndexVersion indexCreatedVersion;
 
         private final IndexMode indexMode;
+        private final SourceKeepMode indexSourceKeepMode;
 
         public Builder(
             String name,
             NumberType type,
             ScriptCompiler compiler,
             Settings settings,
             IndexVersion indexCreatedVersion,
-            IndexMode mode
+            IndexMode mode,
+            SourceKeepMode indexSourceKeepMode
         ) {
-            this(name, type, compiler, IGNORE_MALFORMED_SETTING.get(settings), COERCE_SETTING.get(settings), indexCreatedVersion, mode);
+            this(
+                name,
+                type,
+                compiler,
+                IGNORE_MALFORMED_SETTING.get(settings),
+                COERCE_SETTING.get(settings),
+                indexCreatedVersion,
+                mode,
+                indexSourceKeepMode
+            );
         }
 
         public static Builder docValuesOnly(String name, NumberType type, IndexVersion indexCreatedVersion) {
-            Builder builder = new Builder(name, type, ScriptCompiler.NONE, false, false, indexCreatedVersion, null);
+            Builder builder = new Builder(name, type, ScriptCompiler.NONE, false, false, indexCreatedVersion, null, null);
             builder.indexed.setValue(false);
             builder.dimension.setValue(false);
             return builder;
@@ -153,7 +168,8 @@ public Builder(
             boolean ignoreMalformedByDefault,
             boolean coerceByDefault,
             IndexVersion indexCreatedVersion,
-            IndexMode mode
+            IndexMode mode,
+            SourceKeepMode indexSourceKeepMode
         ) {
             super(name);
             this.type = type;
@@ -209,6 +225,8 @@ public Builder(
 
             this.script.precludesParameters(ignoreMalformed, coerce, nullValue);
             addScriptValidation(script, indexed, hasDocValues);
+
+            this.indexSourceKeepMode = indexSourceKeepMode;
         }
 
         Builder nullValue(Number number) {
@@ -272,7 +290,16 @@ public NumberFieldMapper build(MapperBuilderContext context) {
             MappedFieldType ft = new NumberFieldType(context.buildFullName(leafName()), this, context.isSourceSynthetic());
             hasScript = script.get() != null;
             onScriptError = onScriptErrorParam.getValue();
-            return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this);
+            String offsetsFieldName = getOffsetsFieldName(
+                context,
+                indexSourceKeepMode,
+                hasDocValues.getValue(),
+                stored.getValue(),
+                this,
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER
+            );
+            return new NumberFieldMapper(leafName(), ft, builderParams(this, context), context.isSourceSynthetic(), this, offsetsFieldName);
         }
     }
 
@@ -445,13 +472,8 @@ private static void validateFiniteValue(float value) {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
-                    @Override
-                    protected void writeValue(XContentBuilder b, long value) throws IOException {
-                        b.value(HalfFloatPoint.sortableShortToHalfFloat((short) value));
-                    }
-                };
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(HalfFloatPoint.sortableShortToHalfFloat((short) value));
             }
 
             @Override
@@ -634,13 +656,8 @@ private static void validateFiniteValue(float value) {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
-                    @Override
-                    protected void writeValue(XContentBuilder b, long value) throws IOException {
-                        b.value(NumericUtils.sortableIntToFloat((int) value));
-                    }
-                };
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(NumericUtils.sortableIntToFloat((int) value));
             }
 
             @Override
@@ -789,13 +806,8 @@ private static void validateParsed(double value) {
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
-                    @Override
-                    protected void writeValue(XContentBuilder b, long value) throws IOException {
-                        b.value(NumericUtils.sortableLongToDouble(value));
-                    }
-                };
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(NumericUtils.sortableLongToDouble(value));
             }
 
             @Override
@@ -838,12 +850,12 @@ public Number parsePoint(byte[] value) {
             }
 
             @Override
-            public Short parse(XContentParser parser, boolean coerce) throws IOException {
+            public Byte parse(XContentParser parser, boolean coerce) throws IOException {
                 int value = parser.intValue(coerce);
                 if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
                     throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte");
                 }
-                return (short) value;
+                return (byte) value;
             }
 
             @Override
@@ -912,8 +924,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(value);
             }
 
             @Override
@@ -1030,8 +1042,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(value);
             }
 
             @Override
@@ -1222,8 +1234,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(value);
             }
 
             @Override
@@ -1374,8 +1386,8 @@ public IndexFieldData.Builder getValueFetcherFieldDataBuilder(
             }
 
             @Override
-            SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
-                return syntheticLongFieldLoader(fieldName, fieldSimpleName, ignoreMalformed);
+            public void writeValue(XContentBuilder b, long value) throws IOException {
+                b.value(value);
             }
 
             @Override
@@ -1427,7 +1439,15 @@ private boolean isOutOfRange(Object value) {
             this.name = name;
             this.numericType = numericType;
             this.parser = createTypeParserWithLegacySupport(
-                (n, c) -> new Builder(n, this, c.scriptCompiler(), c.getSettings(), c.indexVersionCreated(), c.getIndexSettings().getMode())
+                (n, c) -> new Builder(
+                    n,
+                    this,
+                    c.scriptCompiler(),
+                    c.getSettings(),
+                    c.indexVersionCreated(),
+                    c.getIndexSettings().getMode(),
+                    c.getIndexSettings().sourceKeepMode()
+                )
             );
         }
 
@@ -1658,17 +1678,13 @@ public double reduceToStoredPrecision(double value) {
             return ((Number) value).doubleValue();
         }
 
-        abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed);
+        abstract void writeValue(XContentBuilder builder, long longValue) throws IOException;
 
-        private static SourceLoader.SyntheticFieldLoader syntheticLongFieldLoader(
-            String fieldName,
-            String fieldSimpleName,
-            boolean ignoreMalformed
-        ) {
+        SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName, boolean ignoreMalformed) {
             return new SortedNumericDocValuesSyntheticFieldLoader(fieldName, fieldSimpleName, ignoreMalformed) {
                 @Override
-                protected void writeValue(XContentBuilder b, long value) throws IOException {
-                    b.value(value);
+                public void writeValue(XContentBuilder b, long value) throws IOException {
+                    NumberType.this.writeValue(b, value);
                 }
             };
         }
@@ -2036,15 +2052,18 @@ public MetricType getMetricType() {
     private boolean allowMultipleValues;
     private final IndexVersion indexCreatedVersion;
     private final boolean isSyntheticSource;
+    private final String offsetsFieldName;
 
     private final IndexMode indexMode;
+    private final SourceKeepMode indexSourceKeepMode;
 
     private NumberFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
         BuilderParams builderParams,
         boolean isSyntheticSource,
-        Builder builder
+        Builder builder,
+        String offsetsFieldName
     ) {
         super(simpleName, mappedFieldType, builderParams);
         this.type = builder.type;
@@ -2065,6 +2084,8 @@ private NumberFieldMapper(
         this.indexCreatedVersion = builder.indexCreatedVersion;
         this.isSyntheticSource = isSyntheticSource;
         this.indexMode = builder.indexMode;
+        this.offsetsFieldName = offsetsFieldName;
+        this.indexSourceKeepMode = builder.indexSourceKeepMode;
     }
 
     boolean coerce() {
@@ -2081,6 +2102,11 @@ public NumberFieldType fieldType() {
         return (NumberFieldType) super.fieldType();
     }
 
+    @Override
+    public String getOffsetFieldName() {
+        return offsetsFieldName;
+    }
+
     public NumberType type() {
         return type;
     }
@@ -2109,7 +2135,17 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         }
         if (value != null) {
             indexValue(context, value);
+        } else {
+            value = fieldType().nullValue;
+        }
+        if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
+            if (value != null) {
+                context.getOffSetContext().recordOffset(offsetsFieldName, (Comparable<?>) value);
+            } else {
+                context.getOffSetContext().recordNull(offsetsFieldName);
+            }
         }
+
     }
 
     /**
@@ -2170,11 +2206,16 @@ protected void indexScriptValues(
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), type, scriptCompiler, ignoreMalformedByDefault, coerceByDefault, indexCreatedVersion, indexMode)
-            .dimension(dimension)
-            .metric(metricType)
-            .allowMultipleValues(allowMultipleValues)
-            .init(this);
+        return new Builder(
+            leafName(),
+            type,
+            scriptCompiler,
+            ignoreMalformedByDefault,
+            coerceByDefault,
+            indexCreatedVersion,
+            indexMode,
+            indexSourceKeepMode
+        ).dimension(dimension).metric(metricType).allowMultipleValues(allowMultipleValues).init(this);
     }
 
     @Override
@@ -2186,10 +2227,23 @@ public void doValidate(MappingLookup lookup) {
         }
     }
 
+    private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
+        if (offsetsFieldName != null) {
+            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+            layers.add(new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, type::writeValue));
+            if (ignoreMalformed.value()) {
+                layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
+            }
+            return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
+        } else {
+            return type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value());
+        }
+    }
+
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
         if (hasDocValues) {
-            return new SyntheticSourceSupport.Native(() -> type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()));
+            return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
         }
 
         return super.syntheticSourceSupport();