Add block loader from stored field and source for ip field

docs/changelog/126644.yaml

@@ -0,0 +1,5 @@
+pr: 126644
+summary: Add block loader from stored field and source for ip field
+area: Mapping
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.SortedSetDocValues;
@@ -20,6 +21,7 @@
 import org.elasticsearch.search.fetch.StoredFieldsSpec;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -381,6 +383,46 @@ public String toString() {
         }
     }
 
+    /**
+     * Load {@code ip}s from {@code _source}.
+     */
+    public static class IpsBlockLoader extends SourceBlockLoader {
+        public IpsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
+            super(fetcher, lookup);
+        }
+
+        @Override
+        public Builder builder(BlockFactory factory, int expectedCount) {
+            return factory.bytesRefs(expectedCount);
+        }
+
+        @Override
+        public RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) {
+            return new Ips(fetcher, iter);
+        }
+
+        @Override
+        protected String name() {
+            return "Ips";
+        }
+    }
+
+    private static class Ips extends BlockSourceReader {
+        Ips(ValueFetcher fetcher, DocIdSetIterator iter) {
+            super(fetcher, iter);
+        }
+
+        @Override
+        protected void append(BlockLoader.Builder builder, Object v) {
+            ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(new BytesRef(InetAddressPoint.encode((InetAddress) v)));
+        }
+
+        @Override
+        public String toString() {
+            return "BlockSourceReader.Ips";
+        }
+    }
+
     /**
      * Convert a {@link String} into a utf-8 {@link BytesRef}.
      */

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -43,6 +43,7 @@
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.net.InetAddress;
@@ -51,8 +52,10 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.BiFunction;
 
 import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
@@ -213,7 +216,8 @@ public IpFieldMapper build(MapperBuilderContext context) {
                     parseNullValue(),
                     scriptValues(),
                     meta.getValue(),
-                    dimension.getValue()
+                    dimension.getValue(),
+                    context.isSourceSynthetic()
                 ),
                 builderParams(this, context),
                 context.isSourceSynthetic(),
@@ -234,6 +238,7 @@ public static final class IpFieldType extends SimpleMappedFieldType {
         private final InetAddress nullValue;
         private final FieldValues<InetAddress> scriptValues;
         private final boolean isDimension;
+        private final boolean isSyntheticSource;
 
         public IpFieldType(
             String name,
@@ -243,12 +248,14 @@ public IpFieldType(
             InetAddress nullValue,
             FieldValues<InetAddress> scriptValues,
             Map<String, String> meta,
-            boolean isDimension
+            boolean isDimension,
+            boolean isSyntheticSource
         ) {
             super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
             this.nullValue = nullValue;
             this.scriptValues = scriptValues;
             this.isDimension = isDimension;
+            this.isSyntheticSource = isSyntheticSource;
         }
 
         public IpFieldType(String name) {
@@ -260,7 +267,7 @@ public IpFieldType(String name, boolean isIndexed) {
         }
 
         public IpFieldType(String name, boolean isIndexed, boolean hasDocValues) {
-            this(name, isIndexed, false, hasDocValues, null, null, Collections.emptyMap(), false);
+            this(name, isIndexed, false, hasDocValues, null, null, Collections.emptyMap(), false, false);
         }
 
         @Override
@@ -452,10 +459,79 @@ public static Query rangeQuery(
 
         @Override
         public BlockLoader blockLoader(BlockLoaderContext blContext) {
-            if (hasDocValues()) {
+            if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSource)) {
                 return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name());
             }
-            return null;
+
+            if (isStored()) {
+                return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name());
+            }
+
+            if (isSyntheticSource) {
+                return blockLoaderFromFallbackSyntheticSource(blContext);
+            }
+
+            // see #indexValue
+            BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isIndexed()
+                ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
+                : BlockSourceReader.lookupMatchingAll();
+            return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
+        }
+
+        private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) {
+            var reader = new FallbackSyntheticSourceBlockLoader.SingleValueReader<InetAddress>(nullValue) {
+                @Override
+                public void convertValue(Object value, List<InetAddress> accumulator) {
+                    if (value instanceof InetAddress ia) {
+                        accumulator.add(ia);
+                    }
+
+                    try {
+                        var address = InetAddresses.forString(value.toString());
+                        accumulator.add(address);
+                    } catch (Exception e) {
+                        // Malformed value, skip it
+                    }
+                }
+
+                @Override
+                protected void parseNonNullValue(XContentParser parser, List<InetAddress> accumulator) throws IOException {
+                    // aligned with #parseCreateField()
+                    String value = parser.text();
+
+                    try {
+                        var address = InetAddresses.forString(value);
+                        accumulator.add(address);
+                    } catch (Exception e) {
+                        // Malformed value, skip it
+                    }
+                }
+
+                @Override
+                public void writeToBlock(List<InetAddress> values, BlockLoader.Builder blockBuilder) {
+                    var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder;
+
+                    for (var value : values) {
+                        bytesRefBuilder.appendBytesRef(new BytesRef(InetAddressPoint.encode(value)));
+                    }
+                }
+            };
+
+            return new FallbackSyntheticSourceBlockLoader(reader, name()) {
+                @Override
+                public Builder builder(BlockFactory factory, int expectedCount) {
+                    return factory.bytesRefs(expectedCount);
+                }
+            };
+        }
+
+        private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths) {
+            return new SourceValueFetcher(sourcePaths, nullValue) {
+                @Override
+                public InetAddress parseSourceValue(Object value) {
+                    return parse(value);
+                }
+            };
         }
 
         @Override

server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java

@@ -105,6 +105,7 @@ public void testTermQuery() {
             null,
             null,
             Collections.emptyMap(),
+            false,
             false
         );
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("::1", MOCK_CONTEXT));
@@ -339,6 +340,7 @@ public void testRangeQuery() {
             null,
             null,
             Collections.emptyMap(),
+            false,
             false
         );
         IllegalArgumentException e = expectThrows(

server/src/test/java/org/elasticsearch/index/mapper/blockloader/IpFieldBlockLoaderTests.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.mapper.blockloader;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.index.mapper.BlockLoaderTestCase;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.logsdb.datageneration.FieldType;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class IpFieldBlockLoaderTests extends BlockLoaderTestCase {
+    public IpFieldBlockLoaderTests(Params params) {
+        super(FieldType.IP.toString(), params);
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
+        var rawNullValue = (String) fieldMapping.get("null_value");
+        BytesRef nullValue = convert(rawNullValue, null);
+
+        if (value == null) {
+            return convert(null, nullValue);
+        }
+        if (value instanceof String s) {
+            return convert(s, nullValue);
+        }
+
+        boolean hasDocValues = hasDocValues(fieldMapping, true);
+        boolean useDocValues = params.preference() == MappedFieldType.FieldExtractPreference.NONE
+            || params.preference() == MappedFieldType.FieldExtractPreference.DOC_VALUES
+            || params.syntheticSource();
+        if (hasDocValues && useDocValues) {
+            var resultList = ((List<String>) value).stream()
+                .map(v -> convert(v, nullValue))
+                .filter(Objects::nonNull)
+                .distinct()
+                .sorted()
+                .toList();
+            return maybeFoldList(resultList);
+        }
+
+        // field is stored or using source
+        var resultList = ((List<String>) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).toList();
+        return maybeFoldList(resultList);
+    }
+
+    private static BytesRef convert(Object value, BytesRef nullValue) {
+        if (value == null) {
+            return nullValue;
+        }
+
+        if (value instanceof String s) {
+            try {
+                var address = InetAddresses.forString(s);
+                return new BytesRef(InetAddressPoint.encode(address));
+            } catch (Exception ex) {
+                // malformed
+                return null;
+            }
+        }
+
+        return null;
+    }
+}

server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java

@@ -1328,6 +1328,7 @@ public void testIpField() throws Exception {
             null,
             null,
             Collections.emptyMap(),
+            false,
             false
         );
         testCase(iw -> {

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.logsdb.datageneration.fields.leaf.GeoPointFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.HalfFloatFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.IntegerFieldDataGenerator;
+import org.elasticsearch.logsdb.datageneration.fields.leaf.IpFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.KeywordFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.LongFieldDataGenerator;
 import org.elasticsearch.logsdb.datageneration.fields.leaf.ScaledFloatFieldDataGenerator;
@@ -44,7 +45,8 @@ public enum FieldType {
     BOOLEAN("boolean"),
     DATE("date"),
     GEO_POINT("geo_point"),
-    TEXT("text");
+    TEXT("text"),
+    IP("ip");
 
     private final String name;
 
@@ -69,6 +71,7 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
             case DATE -> new DateFieldDataGenerator(dataSource);
             case GEO_POINT -> new GeoPointFieldDataGenerator(dataSource);
             case TEXT -> new TextFieldDataGenerator(dataSource);
+            case IP -> new IpFieldDataGenerator(dataSource);
         };
     }
 
@@ -89,6 +92,7 @@ public static FieldType tryParse(String name) {
             case "date" -> FieldType.DATE;
             case "geo_point" -> FieldType.GEO_POINT;
             case "text" -> FieldType.TEXT;
+            case "ip" -> FieldType.IP;
             default -> null;
         };
     }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java

@@ -74,6 +74,10 @@ default DataSourceResponse.PointGenerator handle(DataSourceRequest.PointGenerato
         return null;
     }
 
+    default DataSourceResponse.IpGenerator handle(DataSourceRequest.IpGenerator request) {
+        return null;
+    }
+
     default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) {
         return null;
     }

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java

@@ -120,6 +120,12 @@ public DataSourceResponse.PointGenerator accept(DataSourceHandler handler) {
         }
     }
 
+    record IpGenerator() implements DataSourceRequest<DataSourceResponse.IpGenerator> {
+        public DataSourceResponse.IpGenerator accept(DataSourceHandler handler) {
+            return handler.handle(this);
+        }
+    }
+
     record NullWrapper() implements DataSourceRequest<DataSourceResponse.NullWrapper> {
         public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) {
             return handler.handle(this);

test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.geometry.Geometry;
 
+import java.net.InetAddress;
 import java.time.Instant;
 import java.util.Map;
 import java.util.Optional;
@@ -50,6 +51,8 @@ record PointGenerator(Supplier<Object> generator) implements DataSourceResponse
 
     record GeoPointGenerator(Supplier<Object> generator) implements DataSourceResponse {}
 
+    record IpGenerator(Supplier<InetAddress> generator) implements DataSourceResponse {}
+
     record NullWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
 
     record ArrayWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}