Skip to content

Commit 20333d1

Browse files
lktslkts
authored
[8.x] Add block loader from stored field and source for ip field (elastic#126644) (elastic#126727)
* Add block loader from stored field and source for ip field (elastic#126644) (cherry picked from commit 9d18d52) # Conflicts: # x-pack/plugin/build.gradle # x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java * fix
1 parent 3d0dee5 commit 20333d1

File tree

16 files changed

+355
-11
lines changed

16 files changed

+355
-11
lines changed

‎docs/changelog/126644.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126644
2+
summary: Add block loader from stored field and source for ip field
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.index.mapper;
1111

12+
import org.apache.lucene.document.InetAddressPoint;
1213
import org.apache.lucene.index.LeafReaderContext;
1314
import org.apache.lucene.index.PostingsEnum;
1415
import org.apache.lucene.index.SortedSetDocValues;
@@ -20,6 +21,7 @@
2021
import org.elasticsearch.search.fetch.StoredFieldsSpec;
2122

2223
import java.io.IOException;
24+
import java.net.InetAddress;
2325
import java.util.ArrayList;
2426
import java.util.List;
2527

@@ -380,6 +382,46 @@ public String toString() {
380382
}
381383
}
382384

385+
/**
386+
* Load {@code ip}s from {@code _source}.
387+
*/
388+
public static class IpsBlockLoader extends SourceBlockLoader {
389+
public IpsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
390+
super(fetcher, lookup);
391+
}
392+
393+
@Override
394+
public Builder builder(BlockFactory factory, int expectedCount) {
395+
return factory.bytesRefs(expectedCount);
396+
}
397+
398+
@Override
399+
public RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) {
400+
return new Ips(fetcher, iter);
401+
}
402+
403+
@Override
404+
protected String name() {
405+
return "Ips";
406+
}
407+
}
408+
409+
private static class Ips extends BlockSourceReader {
410+
Ips(ValueFetcher fetcher, DocIdSetIterator iter) {
411+
super(fetcher, iter);
412+
}
413+
414+
@Override
415+
protected void append(BlockLoader.Builder builder, Object v) {
416+
((BlockLoader.BytesRefBuilder) builder).appendBytesRef(new BytesRef(InetAddressPoint.encode((InetAddress) v)));
417+
}
418+
419+
@Override
420+
public String toString() {
421+
return "BlockSourceReader.Ips";
422+
}
423+
}
424+
383425
/**
384426
* Convert a {@link String} into a utf-8 {@link BytesRef}.
385427
*/

‎server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java‎

Lines changed: 80 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
4444
import org.elasticsearch.search.lookup.FieldValues;
4545
import org.elasticsearch.search.lookup.SearchLookup;
46+
import org.elasticsearch.xcontent.XContentParser;
4647

4748
import java.io.IOException;
4849
import java.net.InetAddress;
@@ -51,8 +52,10 @@
5152
import java.util.Arrays;
5253
import java.util.Collection;
5354
import java.util.Collections;
55+
import java.util.List;
5456
import java.util.Map;
5557
import java.util.Objects;
58+
import java.util.Set;
5659
import java.util.function.BiFunction;
5760

5861
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
@@ -213,7 +216,8 @@ public IpFieldMapper build(MapperBuilderContext context) {
213216
parseNullValue(),
214217
scriptValues(),
215218
meta.getValue(),
216-
dimension.getValue()
219+
dimension.getValue(),
220+
context.isSourceSynthetic()
217221
),
218222
builderParams(this, context),
219223
context.isSourceSynthetic(),
@@ -236,6 +240,7 @@ public static final class IpFieldType extends SimpleMappedFieldType {
236240
private final InetAddress nullValue;
237241
private final FieldValues<InetAddress> scriptValues;
238242
private final boolean isDimension;
243+
private final boolean isSyntheticSource;
239244

240245
public IpFieldType(
241246
String name,
@@ -245,12 +250,14 @@ public IpFieldType(
245250
InetAddress nullValue,
246251
FieldValues<InetAddress> scriptValues,
247252
Map<String, String> meta,
248-
boolean isDimension
253+
boolean isDimension,
254+
boolean isSyntheticSource
249255
) {
250256
super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
251257
this.nullValue = nullValue;
252258
this.scriptValues = scriptValues;
253259
this.isDimension = isDimension;
260+
this.isSyntheticSource = isSyntheticSource;
254261
}
255262

256263
public IpFieldType(String name) {
@@ -262,7 +269,7 @@ public IpFieldType(String name, boolean isIndexed) {
262269
}
263270

264271
public IpFieldType(String name, boolean isIndexed, boolean hasDocValues) {
265-
this(name, isIndexed, false, hasDocValues, null, null, Collections.emptyMap(), false);
272+
this(name, isIndexed, false, hasDocValues, null, null, Collections.emptyMap(), false, false);
266273
}
267274

268275
@Override
@@ -457,7 +464,76 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
457464
if (hasDocValues()) {
458465
return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name());
459466
}
460-
return null;
467+
468+
if (isStored()) {
469+
return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name());
470+
}
471+
472+
if (isSyntheticSource) {
473+
return blockLoaderFromFallbackSyntheticSource(blContext);
474+
}
475+
476+
// see #indexValue
477+
BlockSourceReader.LeafIteratorLookup lookup = hasDocValues() == false && isIndexed()
478+
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
479+
: BlockSourceReader.lookupMatchingAll();
480+
return new BlockSourceReader.IpsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
481+
}
482+
483+
private BlockLoader blockLoaderFromFallbackSyntheticSource(BlockLoaderContext blContext) {
484+
var reader = new FallbackSyntheticSourceBlockLoader.SingleValueReader<InetAddress>(nullValue) {
485+
@Override
486+
public void convertValue(Object value, List<InetAddress> accumulator) {
487+
if (value instanceof InetAddress ia) {
488+
accumulator.add(ia);
489+
}
490+
491+
try {
492+
var address = InetAddresses.forString(value.toString());
493+
accumulator.add(address);
494+
} catch (Exception e) {
495+
// Malformed value, skip it
496+
}
497+
}
498+
499+
@Override
500+
protected void parseNonNullValue(XContentParser parser, List<InetAddress> accumulator) throws IOException {
501+
// aligned with #parseCreateField()
502+
String value = parser.text();
503+
504+
try {
505+
var address = InetAddresses.forString(value);
506+
accumulator.add(address);
507+
} catch (Exception e) {
508+
// Malformed value, skip it
509+
}
510+
}
511+
512+
@Override
513+
public void writeToBlock(List<InetAddress> values, BlockLoader.Builder blockBuilder) {
514+
var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder;
515+
516+
for (var value : values) {
517+
bytesRefBuilder.appendBytesRef(new BytesRef(InetAddressPoint.encode(value)));
518+
}
519+
}
520+
};
521+
522+
return new FallbackSyntheticSourceBlockLoader(reader, name()) {
523+
@Override
524+
public Builder builder(BlockFactory factory, int expectedCount) {
525+
return factory.bytesRefs(expectedCount);
526+
}
527+
};
528+
}
529+
530+
private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths) {
531+
return new SourceValueFetcher(sourcePaths, nullValue) {
532+
@Override
533+
public InetAddress parseSourceValue(Object value) {
534+
return parse(value);
535+
}
536+
};
461537
}
462538

463539
@Override

‎server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ public void testTermQuery() {
105105
null,
106106
null,
107107
Collections.emptyMap(),
108+
false,
108109
false
109110
);
110111
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("::1", MOCK_CONTEXT));
@@ -339,6 +340,7 @@ public void testRangeQuery() {
339340
null,
340341
null,
341342
Collections.emptyMap(),
343+
false,
342344
false
343345
);
344346
IllegalArgumentException e = expectThrows(

‎server/src/test/java/org/elasticsearch/index/mapper/blockloader/IpFieldBlockLoaderTests.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper.blockloader;
11+
12+
import org.apache.lucene.document.InetAddressPoint;
13+
import org.apache.lucene.util.BytesRef;
14+
import org.elasticsearch.common.network.InetAddresses;
15+
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
16+
import org.elasticsearch.logsdb.datageneration.FieldType;
17+
18+
import java.util.List;
19+
import java.util.Map;
20+
import java.util.Objects;
21+
22+
public class IpFieldBlockLoaderTests extends BlockLoaderTestCase {
23+
public IpFieldBlockLoaderTests(Params params) {
24+
super(FieldType.IP.toString(), params);
25+
}
26+
27+
@Override
28+
@SuppressWarnings("unchecked")
29+
protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
30+
var rawNullValue = (String) fieldMapping.get("null_value");
31+
BytesRef nullValue = convert(rawNullValue, null);
32+
33+
if (value == null) {
34+
return convert(null, nullValue);
35+
}
36+
if (value instanceof String s) {
37+
return convert(s, nullValue);
38+
}
39+
40+
boolean hasDocValues = hasDocValues(fieldMapping, true);
41+
if (hasDocValues) {
42+
var resultList = ((List<String>) value).stream()
43+
.map(v -> convert(v, nullValue))
44+
.filter(Objects::nonNull)
45+
.distinct()
46+
.sorted()
47+
.toList();
48+
return maybeFoldList(resultList);
49+
}
50+
51+
// field is stored or using source
52+
var resultList = ((List<String>) value).stream().map(v -> convert(v, nullValue)).filter(Objects::nonNull).toList();
53+
return maybeFoldList(resultList);
54+
}
55+
56+
private static BytesRef convert(Object value, BytesRef nullValue) {
57+
if (value == null) {
58+
return nullValue;
59+
}
60+
61+
if (value instanceof String s) {
62+
try {
63+
var address = InetAddresses.forString(s);
64+
return new BytesRef(InetAddressPoint.encode(address));
65+
} catch (Exception ex) {
66+
// malformed
67+
return null;
68+
}
69+
}
70+
71+
return null;
72+
}
73+
}

‎server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1328,6 +1328,7 @@ public void testIpField() throws Exception {
13281328
null,
13291329
null,
13301330
Collections.emptyMap(),
1331+
false,
13311332
false
13321333
);
13331334
testCase(iw -> {

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/FieldType.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.elasticsearch.logsdb.datageneration.fields.leaf.GeoPointFieldDataGenerator;
2020
import org.elasticsearch.logsdb.datageneration.fields.leaf.HalfFloatFieldDataGenerator;
2121
import org.elasticsearch.logsdb.datageneration.fields.leaf.IntegerFieldDataGenerator;
22+
import org.elasticsearch.logsdb.datageneration.fields.leaf.IpFieldDataGenerator;
2223
import org.elasticsearch.logsdb.datageneration.fields.leaf.KeywordFieldDataGenerator;
2324
import org.elasticsearch.logsdb.datageneration.fields.leaf.LongFieldDataGenerator;
2425
import org.elasticsearch.logsdb.datageneration.fields.leaf.ScaledFloatFieldDataGenerator;
@@ -44,7 +45,8 @@ public enum FieldType {
4445
BOOLEAN("boolean"),
4546
DATE("date"),
4647
GEO_POINT("geo_point"),
47-
TEXT("text");
48+
TEXT("text"),
49+
IP("ip");
4850

4951
private final String name;
5052

@@ -69,6 +71,7 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
6971
case DATE -> new DateFieldDataGenerator(dataSource);
7072
case GEO_POINT -> new GeoPointFieldDataGenerator(dataSource);
7173
case TEXT -> new TextFieldDataGenerator(dataSource);
74+
case IP -> new IpFieldDataGenerator(dataSource);
7275
};
7376
}
7477

@@ -89,6 +92,7 @@ public static FieldType tryParse(String name) {
8992
case "date" -> FieldType.DATE;
9093
case "geo_point" -> FieldType.GEO_POINT;
9194
case "text" -> FieldType.TEXT;
95+
case "ip" -> FieldType.IP;
9296
default -> null;
9397
};
9498
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceHandler.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@ default DataSourceResponse.PointGenerator handle(DataSourceRequest.PointGenerato
7474
return null;
7575
}
7676

77+
default DataSourceResponse.IpGenerator handle(DataSourceRequest.IpGenerator request) {
78+
return null;
79+
}
80+
7781
default DataSourceResponse.NullWrapper handle(DataSourceRequest.NullWrapper request) {
7882
return null;
7983
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,12 @@ public DataSourceResponse.PointGenerator accept(DataSourceHandler handler) {
120120
}
121121
}
122122

123+
record IpGenerator() implements DataSourceRequest<DataSourceResponse.IpGenerator> {
124+
public DataSourceResponse.IpGenerator accept(DataSourceHandler handler) {
125+
return handler.handle(this);
126+
}
127+
}
128+
123129
record NullWrapper() implements DataSourceRequest<DataSourceResponse.NullWrapper> {
124130
public DataSourceResponse.NullWrapper accept(DataSourceHandler handler) {
125131
return handler.handle(this);

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceResponse.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.elasticsearch.geometry.Geometry;
1313

14+
import java.net.InetAddress;
1415
import java.time.Instant;
1516
import java.util.Map;
1617
import java.util.Optional;
@@ -50,6 +51,8 @@ record PointGenerator(Supplier<Object> generator) implements DataSourceResponse
5051

5152
record GeoPointGenerator(Supplier<Object> generator) implements DataSourceResponse {}
5253

54+
record IpGenerator(Supplier<InetAddress> generator) implements DataSourceResponse {}
55+
5356
record NullWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}
5457

5558
record ArrayWrapper(Function<Supplier<Object>, Supplier<Object>> wrapper) implements DataSourceResponse {}

0 commit comments

Comments
 (0)