Store arrays offsets for unsigned long fields natively with synthetic source (elastic#125709)

jordan-powers · omricohenn · commit 2818cddef808 · 2025-03-28T23:53:55.000+03:00
This patch builds on the work in elastic#113757, elastic#122999, elastic#124594, and elastic#125529 to natively store array offsets for unsigned long fields instead of falling back to ignored source when synthetic_source_keep: arrays.
diff --git a/docs/changelog/125709.yaml b/docs/changelog/125709.yaml
@@ -0,0 +1,5 @@
+pr: 125709
+summary: Store arrays offsets for unsigned long fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []
diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
@@ -156,6 +156,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_NUMBER = def(9_016_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BOOLEAN = def(9_017_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion RESCORE_PARAMS_ALLOW_ZERO_TO_QUANTIZED_VECTORS = def(9_018_0_00, Version.LUCENE_10_1_0);
+    public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_UNSIGNED_LONG = def(9_019_0_00, Version.LUCENE_10_1_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
@@ -659,7 +659,7 @@ protected String contentType() {
 
     private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
         if (offsetsFieldName != null) {
-            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
             layers.add(
                 new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
                     fullPath(),
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java
@@ -28,14 +28,14 @@ public class FieldArrayContext {
     private static final String OFFSETS_FIELD_NAME_SUFFIX = ".offsets";
     private final Map<String, Offsets> offsetsPerField = new HashMap<>();
 
-    void recordOffset(String field, Comparable<?> value) {
+    public void recordOffset(String field, Comparable<?> value) {
         Offsets arrayOffsets = offsetsPerField.computeIfAbsent(field, k -> new Offsets());
         int nextOffset = arrayOffsets.currentOffset++;
         var offsets = arrayOffsets.valueToOffsets.computeIfAbsent(value, s -> new ArrayList<>(2));
         offsets.add(nextOffset);
     }
 
-    void recordNull(String field) {
+    public void recordNull(String field) {
         Offsets arrayOffsets = offsetsPerField.computeIfAbsent(field, k -> new Offsets());
         int nextOffset = arrayOffsets.currentOffset++;
         arrayOffsets.nullValueOffsets.add(nextOffset);
@@ -83,7 +83,7 @@ static int[] parseOffsetArray(StreamInput in) throws IOException {
         return offsetToOrd;
     }
 
-    static String getOffsetsFieldName(
+    public static String getOffsetsFieldName(
         MapperBuilderContext context,
         Mapper.SourceKeepMode indexSourceKeepMode,
         boolean hasDocValues,
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
@@ -644,7 +644,7 @@ public void doValidate(MappingLookup lookup) {
     protected SyntheticSourceSupport syntheticSourceSupport() {
         if (hasDocValues) {
             return new SyntheticSourceSupport.Native(() -> {
-                var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+                var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
                 if (offsetsFieldName != null) {
                     layers.add(
                         new SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, IpFieldMapper::convert)
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -1279,7 +1279,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
     public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fullFieldName, String leafFieldName) {
         assert fieldType.stored() || hasDocValues;
 
-        var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+        var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
         if (fieldType.stored()) {
             layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(fullPath()) {
                 @Override
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
@@ -2274,7 +2274,7 @@ public void doValidate(MappingLookup lookup) {
 
     private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
         if (offsetsFieldName != null) {
-            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
+            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
             layers.add(new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, type::writeValue));
             if (ignoreMalformed.value()) {
                 layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer.java b/server/src/main/java/org/elasticsearch/index/mapper/SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer.java
@@ -18,9 +18,9 @@
 
 import java.io.IOException;
 
-class SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer implements CompositeSyntheticFieldLoader.DocValuesLayer {
+public class SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer implements CompositeSyntheticFieldLoader.DocValuesLayer {
     @FunctionalInterface
-    interface NumericValueWriter {
+    public interface NumericValueWriter {
         void writeLongValue(XContentBuilder b, long value) throws IOException;
     }
 
@@ -29,7 +29,11 @@ interface NumericValueWriter {
     private final NumericValueWriter valueWriter;
     private NumericDocValuesWithOffsetsLoader docValuesLoader;
 
-    SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(String fullPath, String offsetsFieldName, NumericValueWriter valueWriter) {
+    public SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
+        String fullPath,
+        String offsetsFieldName,
+        NumericValueWriter valueWriter
+    ) {
         this.fullPath = fullPath;
         this.offsetsFieldName = offsetsFieldName;
         this.valueWriter = valueWriter;
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/OffsetDocValuesLoaderTestCase.java
diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java
@@ -20,13 +20,16 @@
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData;
 import org.elasticsearch.index.mapper.BlockDocValuesReader;
 import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.BlockSourceReader;
+import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -37,6 +40,8 @@
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.SimpleMappedFieldType;
 import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader;
+import org.elasticsearch.index.mapper.SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
@@ -64,6 +69,7 @@
 import java.util.Set;
 import java.util.function.Function;
 
+import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
 import static org.elasticsearch.xpack.unsignedlong.UnsignedLongLeafFieldData.convertUnsignedLongToDouble;
 
 public class UnsignedLongFieldMapper extends FieldMapper {
@@ -97,13 +103,27 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
          */
         private final Parameter<MetricType> metric;
 
+        private final IndexVersion indexCreatedVersion;
         private final IndexMode indexMode;
+        private final SourceKeepMode indexSourceKeepMode;
 
-        public Builder(String name, Settings settings, IndexMode mode) {
-            this(name, IGNORE_MALFORMED_SETTING.get(settings), mode);
+        public Builder(
+            String name,
+            Settings settings,
+            IndexVersion indexCreatedVersion,
+            IndexMode mode,
+            SourceKeepMode indexSourceKeepMode
+        ) {
+            this(name, IGNORE_MALFORMED_SETTING.get(settings), indexCreatedVersion, mode, indexSourceKeepMode);
         }
 
-        public Builder(String name, boolean ignoreMalformedByDefault, IndexMode mode) {
+        public Builder(
+            String name,
+            boolean ignoreMalformedByDefault,
+            IndexVersion indexCreatedVersion,
+            IndexMode mode,
+            SourceKeepMode indexSourceKeepMode
+        ) {
             super(name);
             this.ignoreMalformed = Parameter.explicitBoolParam(
                 "ignore_malformed",
@@ -150,6 +170,9 @@ public Builder(String name, boolean ignoreMalformedByDefault, IndexMode mode) {
                     );
                 }
             }).precludesParameters(dimension);
+
+            this.indexCreatedVersion = indexCreatedVersion;
+            this.indexSourceKeepMode = indexSourceKeepMode;
         }
 
         private String parseNullValueAsString(Object o) {
@@ -211,11 +234,35 @@ public UnsignedLongFieldMapper build(MapperBuilderContext context) {
                 indexMode,
                 context.isSourceSynthetic()
             );
-            return new UnsignedLongFieldMapper(leafName(), fieldType, builderParams(this, context), context.isSourceSynthetic(), this);
+            String offsetsFieldName = getOffsetsFieldName(
+                context,
+                indexSourceKeepMode,
+                hasDocValues.getValue(),
+                stored.getValue(),
+                this,
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_UNSIGNED_LONG
+            );
+            return new UnsignedLongFieldMapper(
+                leafName(),
+                fieldType,
+                builderParams(this, context),
+                context.isSourceSynthetic(),
+                this,
+                offsetsFieldName
+            );
         }
     }
 
-    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings(), c.getIndexSettings().getMode()));
+    public static final TypeParser PARSER = new TypeParser(
+        (n, c) -> new Builder(
+            n,
+            c.getSettings(),
+            c.indexVersionCreated(),
+            c.getIndexSettings().getMode(),
+            c.getIndexSettings().sourceKeepMode()
+        )
+    );
 
     public static final class UnsignedLongFieldType extends SimpleMappedFieldType {
 
@@ -641,14 +688,18 @@ public MetricType getMetricType() {
     private final Long nullValueIndexed; // null value to use for indexing, represented as shifted to signed long range
     private final boolean dimension;
     private final MetricType metricType;
+    private final IndexVersion indexCreatedVersion;
     private final IndexMode indexMode;
+    private final String offsetsFieldName;
+    private final SourceKeepMode indexSourceKeepMode;
 
     private UnsignedLongFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
         BuilderParams builderParams,
         boolean isSourceSynthetic,
-        Builder builder
+        Builder builder,
+        String offsetsFieldName
     ) {
         super(simpleName, mappedFieldType, builderParams);
         this.isSourceSynthetic = isSourceSynthetic;
@@ -667,6 +718,9 @@ private UnsignedLongFieldMapper(
         this.dimension = builder.dimension.getValue();
         this.metricType = builder.metric.getValue();
         this.indexMode = builder.indexMode;
+        this.indexCreatedVersion = builder.indexCreatedVersion;
+        this.offsetsFieldName = offsetsFieldName;
+        this.indexSourceKeepMode = builder.indexSourceKeepMode;
     }
 
     @Override
@@ -679,6 +733,11 @@ public UnsignedLongFieldType fieldType() {
         return (UnsignedLongFieldType) super.fieldType();
     }
 
+    @Override
+    public String getOffsetFieldName() {
+        return offsetsFieldName;
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;
@@ -715,7 +774,6 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         boolean isNullValue = false;
         if (numericValue == null) {
             numericValue = nullValueIndexed;
-            if (numericValue == null) return;
             isNullValue = true;
         } else {
             numericValue = unsignedToSortableSignedLong(numericValue);
@@ -725,29 +783,41 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             context.getRoutingFields().addUnsignedLong(fieldType().name(), numericValue);
         }
 
-        List<Field> fields = new ArrayList<>();
-        if (indexed && hasDocValues) {
-            fields.add(new LongField(fieldType().name(), numericValue, Field.Store.NO));
-        } else if (hasDocValues) {
-            fields.add(new SortedNumericDocValuesField(fieldType().name(), numericValue));
-        } else if (indexed) {
-            fields.add(new LongPoint(fieldType().name(), numericValue));
-        }
-        if (stored) {
-            // for stored field, keeping original unsigned_long value in the String form
-            String storedValued = isNullValue ? nullValue : Long.toUnsignedString(unsignedToSortableSignedLong(numericValue));
-            fields.add(new StoredField(fieldType().name(), storedValued));
+        if (numericValue != null) {
+            List<Field> fields = new ArrayList<>();
+            if (indexed && hasDocValues) {
+                fields.add(new LongField(fieldType().name(), numericValue, Field.Store.NO));
+            } else if (hasDocValues) {
+                fields.add(new SortedNumericDocValuesField(fieldType().name(), numericValue));
+            } else if (indexed) {
+                fields.add(new LongPoint(fieldType().name(), numericValue));
+            }
+            if (stored) {
+                // for stored field, keeping original unsigned_long value in the String form
+                String storedValued = isNullValue ? nullValue : Long.toUnsignedString(unsignedToSortableSignedLong(numericValue));
+                fields.add(new StoredField(fieldType().name(), storedValued));
+            }
+            context.doc().addAll(fields);
+
+            if (hasDocValues == false && (stored || indexed)) {
+                context.addToFieldNames(fieldType().name());
+            }
         }
-        context.doc().addAll(fields);
 
-        if (hasDocValues == false && (stored || indexed)) {
-            context.addToFieldNames(fieldType().name());
+        if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
+            if (numericValue != null) {
+                context.getOffSetContext().recordOffset(offsetsFieldName, numericValue);
+            } else {
+                context.getOffSetContext().recordNull(offsetsFieldName);
+            }
         }
     }
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), ignoreMalformedByDefault, indexMode).dimension(dimension).metric(metricType).init(this);
+        return new Builder(leafName(), ignoreMalformedByDefault, indexCreatedVersion, indexMode, indexSourceKeepMode).dimension(dimension)
+            .metric(metricType)
+            .init(this);
     }
 
     /**
@@ -828,17 +898,34 @@ public void doValidate(MappingLookup lookup) {
         }
     }
 
+    private SourceLoader.SyntheticFieldLoader docValuesSyntheticFieldLoader() {
+        if (offsetsFieldName != null) {
+            var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>(2);
+            layers.add(
+                new SortedNumericWithOffsetsDocValuesSyntheticFieldLoaderLayer(
+                    fullPath(),
+                    offsetsFieldName,
+                    (b, value) -> b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value))
+                )
+            );
+            if (ignoreMalformed.value()) {
+                layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
+            }
+            return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers);
+        } else {
+            return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) {
+                @Override
+                protected void writeValue(XContentBuilder b, long value) throws IOException {
+                    b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value));
+                }
+            };
+        }
+    }
+
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
         if (hasDocValues) {
-            return new SyntheticSourceSupport.Native(
-                () -> new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) {
-                    @Override
-                    protected void writeValue(XContentBuilder b, long value) throws IOException {
-                        b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value));
-                    }
-                }
-            );
+            return new SyntheticSourceSupport.Native(this::docValuesSyntheticFieldLoader);
         }
 
         return super.syntheticSourceSupport();
diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java
@@ -385,7 +385,20 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed)
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) {
-        return syntheticSourceSupport(ignoreMalformed);
+        return new NumberSyntheticSourceSupport(ignoreMalformed) {
+            @Override
+            public SyntheticSourceExample example(int maxVals) {
+                var example = super.example(maxVals);
+                // Need the expectedForSyntheticSource as inputValue since MapperTestCase#testSyntheticSourceKeepArrays
+                // uses the inputValue as both the input and expected.
+                return new SyntheticSourceExample(
+                    example.expectedForSyntheticSource(),
+                    example.expectedForSyntheticSource(),
+                    example.expectedForBlockLoader(),
+                    example.mapping()
+                );
+            }
+        };
     }
 
     @Override
@@ -437,7 +450,7 @@ protected Function<Object, Object> loadBlockExpected() {
         };
     }
 
-    final class NumberSyntheticSourceSupport implements SyntheticSourceSupport {
+    class NumberSyntheticSourceSupport implements SyntheticSourceSupport {
         private final BigInteger nullValue = usually() ? null : BigInteger.valueOf(randomNonNegativeLong());
         private final boolean ignoreMalformedEnabled;
 
diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java
diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongOffsetDocValuesLoaderTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongOffsetDocValuesLoaderTests.java
diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongSyntheticSourceNativeArrayIntegrationTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongSyntheticSourceNativeArrayIntegrationTests.java
