Store arrays offsets for ip fields natively with synthetic source (#122999)

Follow up of #113757 and adds support to natively store array offsets for ip fields instead of falling back to ignored source.

Author: martijnvg

docs/changelog/122999.yaml

@@ -0,0 +1,5 @@
+pr: 122999
+summary: Store arrays offsets for ip fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -150,6 +150,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_011_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion TIME_SERIES_ID_DOC_VALUES_SPARSE_INDEX = def(9_012_0_00, Version.LUCENE_10_1_0);
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD = def(9_013_0_00, Version.LUCENE_10_1_0);
+    public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_IP = def(9_014_0_00, Version.LUCENE_10_1_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java

@@ -13,6 +13,8 @@
 import org.apache.lucene.util.BitUtil;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -23,9 +25,10 @@
 
 public class FieldArrayContext {
 
+    private static final String OFFSETS_FIELD_NAME_SUFFIX = ".offsets";
     private final Map<String, Offsets> offsetsPerField = new HashMap<>();
 
-    void recordOffset(String field, String value) {
+    void recordOffset(String field, Comparable<?> value) {
         Offsets arrayOffsets = offsetsPerField.computeIfAbsent(field, k -> new Offsets());
         int nextOffset = arrayOffsets.currentOffset++;
         var offsets = arrayOffsets.valueToOffsets.computeIfAbsent(value, s -> new ArrayList<>(2));
@@ -79,13 +82,53 @@ static int[] parseOffsetArray(StreamInput in) throws IOException {
         return offsetToOrd;
     }
 
+    static String getOffsetsFieldName(
+        MapperBuilderContext context,
+        Mapper.SourceKeepMode indexSourceKeepMode,
+        boolean hasDocValues,
+        boolean isStored,
+        FieldMapper.Builder fieldMapperBuilder,
+        IndexVersion indexCreatedVersion,
+        IndexVersion minSupportedVersionMain
+    ) {
+        var sourceKeepMode = fieldMapperBuilder.sourceKeepMode.orElse(indexSourceKeepMode);
+        if (context.isSourceSynthetic()
+            && sourceKeepMode == Mapper.SourceKeepMode.ARRAYS
+            && hasDocValues
+            && isStored == false
+            && fieldMapperBuilder.copyTo.copyToFields().isEmpty()
+            && fieldMapperBuilder.multiFieldsBuilder.hasMultiFields() == false
+            && indexVersionSupportStoringArraysNatively(indexCreatedVersion, minSupportedVersionMain)) {
+            // Skip stored, we will be synthesizing from stored fields, no point to keep track of the offsets
+            // Skip copy_to and multi fields, supporting that requires more work. However, copy_to usage is rare in metrics and
+            // logging use cases
+
+            // keep track of value offsets so that we can reconstruct arrays from doc values in order as was specified during indexing
+            // (if field is stored then there is no point of doing this)
+            return context.buildFullName(fieldMapperBuilder.leafName() + FieldArrayContext.OFFSETS_FIELD_NAME_SUFFIX);
+        } else {
+            return null;
+        }
+    }
+
+    private static boolean indexVersionSupportStoringArraysNatively(
+        IndexVersion indexCreatedVersion,
+        IndexVersion minSupportedVersionMain
+    ) {
+        return indexCreatedVersion.onOrAfter(minSupportedVersionMain)
+            || indexCreatedVersion.between(
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD_BACKPORT_8_X,
+                IndexVersions.UPGRADE_TO_LUCENE_10_0_0
+            );
+    }
+
     private static class Offsets {
 
         int currentOffset;
         // Need to use TreeMap here, so that we maintain the order in which each value (with offset) stored inserted,
         // (which is in the same order the document gets parsed) so we store offsets in right order. This is the same
         // order in what the values get stored in SortedSetDocValues.
-        final Map<String, List<Integer>> valueToOffsets = new TreeMap<>();
+        final Map<Comparable<?>, List<Integer>> valueToOffsets = new TreeMap<>();
         final List<Integer> nullValueOffsets = new ArrayList<>(2);
 
     }

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -55,6 +55,7 @@
 import java.util.Objects;
 import java.util.function.BiFunction;
 
+import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
 import static org.elasticsearch.index.mapper.IpPrefixAutomatonUtil.buildIpPrefixAutomaton;
 
 /**
@@ -92,8 +93,15 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
         private final boolean ignoreMalformedByDefault;
         private final IndexVersion indexCreatedVersion;
         private final ScriptCompiler scriptCompiler;
+        private final SourceKeepMode indexSourceKeepMode;
 
-        public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion) {
+        public Builder(
+            String name,
+            ScriptCompiler scriptCompiler,
+            boolean ignoreMalformedByDefault,
+            IndexVersion indexCreatedVersion,
+            SourceKeepMode indexSourceKeepMode
+        ) {
             super(name);
             this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
             this.ignoreMalformedByDefault = ignoreMalformedByDefault;
@@ -114,6 +122,7 @@ public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalform
                     );
                 }
             });
+            this.indexSourceKeepMode = indexSourceKeepMode;
         }
 
         Builder nullValue(String nullValue) {
@@ -184,6 +193,16 @@ public IpFieldMapper build(MapperBuilderContext context) {
             }
             hasScript = script.get() != null;
             onScriptError = onScriptErrorParam.getValue();
+
+            String offsetsFieldName = getOffsetsFieldName(
+                context,
+                indexSourceKeepMode,
+                hasDocValues.getValue(),
+                stored.getValue(),
+                this,
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_IP
+            );
             return new IpFieldMapper(
                 leafName(),
                 new IpFieldType(
@@ -198,15 +217,16 @@ public IpFieldMapper build(MapperBuilderContext context) {
                 ),
                 builderParams(this, context),
                 context.isSourceSynthetic(),
-                this
+                this,
+                offsetsFieldName
             );
         }
 
     }
 
     public static final TypeParser PARSER = createTypeParserWithLegacySupport((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
-        return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated());
+        return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated(), c.getIndexSettings().sourceKeepMode());
     });
 
     public static final class IpFieldType extends SimpleMappedFieldType {
@@ -501,13 +521,16 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi
     private final Script script;
     private final FieldValues<InetAddress> scriptValues;
     private final ScriptCompiler scriptCompiler;
+    private final SourceKeepMode indexSourceKeepMode;
+    private final String offsetsFieldName;
 
     private IpFieldMapper(
         String simpleName,
         MappedFieldType mappedFieldType,
         BuilderParams builderParams,
         boolean storeIgnored,
-        Builder builder
+        Builder builder,
+        String offsetsFieldName
     ) {
         super(simpleName, mappedFieldType, builderParams);
         this.ignoreMalformedByDefault = builder.ignoreMalformedByDefault;
@@ -523,6 +546,8 @@ private IpFieldMapper(
         this.scriptCompiler = builder.scriptCompiler;
         this.dimension = builder.dimension.getValue();
         this.storeIgnored = storeIgnored;
+        this.indexSourceKeepMode = builder.indexSourceKeepMode;
+        this.offsetsFieldName = offsetsFieldName;
     }
 
     @Override
@@ -561,6 +586,14 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         if (address != null) {
             indexValue(context, address);
         }
+        if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
+            if (address != null) {
+                BytesRef sortableValue = new BytesRef(InetAddressPoint.encode(address));
+                context.getOffSetContext().recordOffset(offsetsFieldName, sortableValue);
+            } else {
+                context.getOffSetContext().recordNull(offsetsFieldName);
+            }
+        }
     }
 
     private void indexValue(DocumentParserContext context, InetAddress address) {
@@ -593,7 +626,9 @@ protected void indexScriptValues(
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(dimension).init(this);
+        return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension(
+            dimension
+        ).init(this);
     }
 
     @Override
@@ -610,19 +645,24 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
         if (hasDocValues) {
             return new SyntheticSourceSupport.Native(() -> {
                 var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
-                layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) {
-                    @Override
-                    protected BytesRef convert(BytesRef value) {
-                        byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length);
-                        return new BytesRef(NetworkAddress.format(InetAddressPoint.decode(bytes)));
-                    }
-
-                    @Override
-                    protected BytesRef preserve(BytesRef value) {
-                        // No need to copy because convert has made a deep copy
-                        return value;
-                    }
-                });
+                if (offsetsFieldName != null) {
+                    layers.add(
+                        new SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, IpFieldMapper::convert)
+                    );
+                } else {
+                    layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) {
+                        @Override
+                        protected BytesRef convert(BytesRef value) {
+                            return IpFieldMapper.convert(value);
+                        }
+
+                        @Override
+                        protected BytesRef preserve(BytesRef value) {
+                            // No need to copy because convert has made a deep copy
+                            return value;
+                        }
+                    });
+                }
 
                 if (ignoreMalformed) {
                     layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
@@ -633,4 +673,14 @@ protected BytesRef preserve(BytesRef value) {
 
         return super.syntheticSourceSupport();
     }
+
+    static BytesRef convert(BytesRef value) {
+        byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length);
+        return new BytesRef(NetworkAddress.format(InetAddressPoint.decode(bytes)));
+    }
+
+    @Override
+    public String getOffsetFieldName() {
+        return offsetsFieldName;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -85,6 +85,7 @@
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
 import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER;
+import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
 
 /**
  * A field mapper for keywords. This mapper accepts strings and indexes them as-is.
@@ -95,7 +96,6 @@ public final class KeywordFieldMapper extends FieldMapper {
 
     public static final String CONTENT_TYPE = "keyword";
     private static final String HOST_NAME = "host.name";
-    public static final String OFFSETS_FIELD_NAME_SUFFIX = ".offsets";
 
     public static class Defaults {
         public static final FieldType FIELD_TYPE;
@@ -439,26 +439,15 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
             super.hasScript = script.get() != null;
             super.onScriptError = onScriptError.getValue();
 
-            var sourceKeepMode = this.sourceKeepMode.orElse(indexSourceKeepMode);
-            String offsetsFieldName;
-            if (context.isSourceSynthetic()
-                && sourceKeepMode == SourceKeepMode.ARRAYS
-                && hasDocValues()
-                && fieldtype.stored() == false
-                && copyTo.copyToFields().isEmpty()
-                && multiFieldsBuilder.hasMultiFields() == false
-                && indexVersionSupportStoringArraysNatively()) {
-                // Skip stored, we will be synthesizing from stored fields, no point to keep track of the offsets
-                // Skip copy_to and multi fields, supporting that requires more work. However, copy_to usage is rare in metrics and
-                // logging use cases
-
-                // keep track of value offsets so that we can reconstruct arrays from doc values in order as was specified during indexing
-                // (if field is stored then there is no point of doing this)
-                offsetsFieldName = context.buildFullName(leafName() + OFFSETS_FIELD_NAME_SUFFIX);
-            } else {
-                offsetsFieldName = null;
-            }
-
+            String offsetsFieldName = getOffsetsFieldName(
+                context,
+                indexSourceKeepMode,
+                hasDocValues.getValue(),
+                stored.getValue(),
+                this,
+                indexCreatedVersion,
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD
+            );
             return new KeywordFieldMapper(
                 leafName(),
                 fieldtype,
@@ -472,14 +461,6 @@ && indexVersionSupportStoringArraysNatively()) {
             );
         }
 
-        private boolean indexVersionSupportStoringArraysNatively() {
-            return indexCreatedVersion.onOrAfter(IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD)
-                || indexCreatedVersion.between(
-                    IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD_BACKPORT_8_X,
-                    IndexVersions.UPGRADE_TO_LUCENE_10_0_0
-                );
-        }
-
         private FieldType resolveFieldType(
             final boolean useDocValuesSkipper,
             final IndexVersion indexCreatedVersion,
@@ -1127,7 +1108,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         }
 
         boolean indexed = indexValue(context, value);
-        if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.getRecordedSource() == false) {
+        if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) {
             if (indexed) {
                 context.getOffSetContext().recordOffset(offsetsFieldName, value);
             } else if (value == null) {