Skip to content

Commit efc2fc8

Browse files
martijnvgmartijnvg
committed
Store arrays offsets for ip fields natively with synthetic source
Follow up of elastic#113757 and adds support to natively store array offsets for ip fields instead of falling back to ignored source.
1 parent 43665f0 commit efc2fc8

File tree

9 files changed

+395
-239
lines changed

9 files changed

+395
-239
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/FieldArrayContext.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@
2323

2424
public class FieldArrayContext {
2525

26+
public static final String OFFSETS_FIELD_NAME_SUFFIX = ".offsets";
2627
private final Map<String, Offsets> offsetsPerField = new HashMap<>();
2728

28-
void recordOffset(String field, String value) {
29+
void recordOffset(String field, Comparable<?> value) {
2930
Offsets arrayOffsets = offsetsPerField.computeIfAbsent(field, k -> new Offsets());
3031
int nextOffset = arrayOffsets.currentOffset++;
3132
var offsets = arrayOffsets.valueToOffsets.computeIfAbsent(value, s -> new ArrayList<>(2));
@@ -85,7 +86,7 @@ private static class Offsets {
8586
// Need to use TreeMap here, so that we maintain the order in which each value (with offset) stored inserted,
8687
// (which is in the same order the document gets parsed) so we store offsets in right order. This is the same
8788
// order in what the values get stored in SortedSetDocValues.
88-
final Map<String, List<Integer>> valueToOffsets = new TreeMap<>();
89+
final Map<Comparable<?>, List<Integer>> valueToOffsets = new TreeMap<>();
8990
final List<Integer> nullValueOffsets = new ArrayList<>(2);
9091

9192
}

‎server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java‎

Lines changed: 78 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,15 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
9292
private final boolean ignoreMalformedByDefault;
9393
private final IndexVersion indexCreatedVersion;
9494
private final ScriptCompiler scriptCompiler;
95+
private final SourceKeepMode indexSourceKeepMode;
9596

96-
public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, IndexVersion indexCreatedVersion) {
97+
public Builder(
98+
String name,
99+
ScriptCompiler scriptCompiler,
100+
boolean ignoreMalformedByDefault,
101+
IndexVersion indexCreatedVersion,
102+
SourceKeepMode indexSourceKeepMode
103+
) {
97104
super(name);
98105
this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
99106
this.ignoreMalformedByDefault = ignoreMalformedByDefault;
@@ -114,6 +121,7 @@ public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalform
114121
);
115122
}
116123
});
124+
this.indexSourceKeepMode = indexSourceKeepMode;
117125
}
118126

119127
Builder nullValue(String nullValue) {
@@ -184,6 +192,27 @@ public IpFieldMapper build(MapperBuilderContext context) {
184192
}
185193
hasScript = script.get() != null;
186194
onScriptError = onScriptErrorParam.getValue();
195+
196+
var sourceKeepMode = this.sourceKeepMode.orElse(indexSourceKeepMode);
197+
String offsetsFieldName;
198+
if (context.isSourceSynthetic()
199+
&& sourceKeepMode == SourceKeepMode.ARRAYS
200+
&& hasDocValues.get()
201+
&& stored.get() == false
202+
&& copyTo.copyToFields().isEmpty()
203+
&& multiFieldsBuilder.hasMultiFields() == false
204+
&& indexCreatedVersion.onOrAfter(IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD)) {
205+
// Skip stored, we will be synthesizing from stored fields, no point to keep track of the offsets
206+
// Skip copy_to and multi fields, supporting that requires more work. However, copy_to usage is rare in metrics and
207+
// logging use cases
208+
209+
// keep track of value offsets so that we can reconstruct arrays from doc values in order as was specified during indexing
210+
// (if field is stored then there is no point of doing this)
211+
offsetsFieldName = context.buildFullName(leafName() + FieldArrayContext.OFFSETS_FIELD_NAME_SUFFIX);
212+
} else {
213+
offsetsFieldName = null;
214+
}
215+
187216
return new IpFieldMapper(
188217
leafName(),
189218
new IpFieldType(
@@ -198,15 +227,16 @@ public IpFieldMapper build(MapperBuilderContext context) {
198227
),
199228
builderParams(this, context),
200229
context.isSourceSynthetic(),
201-
this
230+
this,
231+
offsetsFieldName
202232
);
203233
}
204234

205235
}
206236

207237
public static final TypeParser PARSER = createTypeParserWithLegacySupport((n, c) -> {
208238
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
209-
return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated());
239+
return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated(), c.getIndexSettings().sourceKeepMode());
210240
});
211241

212242
public static final class IpFieldType extends SimpleMappedFieldType {
@@ -501,13 +531,16 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi
501531
private final Script script;
502532
private final FieldValues<InetAddress> scriptValues;
503533
private final ScriptCompiler scriptCompiler;
534+
private final SourceKeepMode indexSourceKeepMode;
535+
private final String offsetsFieldName;
504536

505537
private IpFieldMapper(
506538
String simpleName,
507539
MappedFieldType mappedFieldType,
508540
BuilderParams builderParams,
509541
boolean storeIgnored,
510-
Builder builder
542+
Builder builder,
543+
String offsetsFieldName
511544
) {
512545
super(simpleName, mappedFieldType, builderParams);
513546
this.ignoreMalformedByDefault = builder.ignoreMalformedByDefault;
@@ -523,6 +556,8 @@ private IpFieldMapper(
523556
this.scriptCompiler = builder.scriptCompiler;
524557
this.dimension = builder.dimension.getValue();
525558
this.storeIgnored = storeIgnored;
559+
this.indexSourceKeepMode = builder.indexSourceKeepMode;
560+
this.offsetsFieldName = offsetsFieldName;
526561
}
527562

528563
@Override
@@ -561,6 +596,14 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
561596
if (address != null) {
562597
indexValue(context, address);
563598
}
599+
if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.getRecordedSource() == false) {
600+
if (address != null) {
601+
BytesRef sortableValue = new BytesRef(InetAddressPoint.encode(address));
602+
context.getOffSetContext().recordOffset(offsetsFieldName, sortableValue);
603+
} else {
604+
context.getOffSetContext().recordNull(offsetsFieldName);
605+
}
606+
}
564607
}
565608

566609
private void indexValue(DocumentParserContext context, InetAddress address) {
@@ -593,7 +636,9 @@ protected void indexScriptValues(
593636

594637
@Override
595638
public FieldMapper.Builder getMergeBuilder() {
596-
return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion).dimension(dimension).init(this);
639+
return new Builder(leafName(), scriptCompiler, ignoreMalformedByDefault, indexCreatedVersion, indexSourceKeepMode).dimension(
640+
dimension
641+
).init(this);
597642
}
598643

599644
@Override
@@ -610,19 +655,24 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
610655
if (hasDocValues) {
611656
return new SyntheticSourceSupport.Native(() -> {
612657
var layers = new ArrayList<CompositeSyntheticFieldLoader.Layer>();
613-
layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) {
614-
@Override
615-
protected BytesRef convert(BytesRef value) {
616-
byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length);
617-
return new BytesRef(NetworkAddress.format(InetAddressPoint.decode(bytes)));
618-
}
619-
620-
@Override
621-
protected BytesRef preserve(BytesRef value) {
622-
// No need to copy because convert has made a deep copy
623-
return value;
624-
}
625-
});
658+
if (offsetsFieldName != null) {
659+
layers.add(
660+
new SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(fullPath(), offsetsFieldName, IpFieldMapper::convert)
661+
);
662+
} else {
663+
layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) {
664+
@Override
665+
protected BytesRef convert(BytesRef value) {
666+
return IpFieldMapper.convert(value);
667+
}
668+
669+
@Override
670+
protected BytesRef preserve(BytesRef value) {
671+
// No need to copy because convert has made a deep copy
672+
return value;
673+
}
674+
});
675+
}
626676

627677
if (ignoreMalformed) {
628678
layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()));
@@ -633,4 +683,14 @@ protected BytesRef preserve(BytesRef value) {
633683

634684
return super.syntheticSourceSupport();
635685
}
686+
687+
static BytesRef convert(BytesRef value) {
688+
byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length);
689+
return new BytesRef(NetworkAddress.format(InetAddressPoint.decode(bytes)));
690+
}
691+
692+
@Override
693+
public String getOffsetFieldName() {
694+
return offsetsFieldName;
695+
}
636696
}

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,6 @@ public final class KeywordFieldMapper extends FieldMapper {
9595

9696
public static final String CONTENT_TYPE = "keyword";
9797
private static final String HOST_NAME = "host.name";
98-
public static final String OFFSETS_FIELD_NAME_SUFFIX = ".offsets";
9998

10099
public static class Defaults {
101100
public static final FieldType FIELD_TYPE;
@@ -454,7 +453,7 @@ && hasDocValues()
454453

455454
// keep track of value offsets so that we can reconstruct arrays from doc values in order as was specified during indexing
456455
// (if field is stored then there is no point of doing this)
457-
offsetsFieldName = context.buildFullName(leafName() + OFFSETS_FIELD_NAME_SUFFIX);
456+
offsetsFieldName = context.buildFullName(leafName() + FieldArrayContext.OFFSETS_FIELD_NAME_SUFFIX);
458457
} else {
459458
offsetsFieldName = null;
460459
}

‎server/src/main/java/org/elasticsearch/index/mapper/SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import java.io.IOException;
2222
import java.util.Objects;
23+
import java.util.function.Function;
2324

2425
/**
2526
* Load {@code _source} fields from {@link SortedSetDocValues} and associated {@link BinaryDocValues}. The former contains the unique values
@@ -30,11 +31,17 @@ final class SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer implements Co
3031

3132
private final String name;
3233
private final String offsetsFieldName;
34+
private final Function<BytesRef, BytesRef> converter;
3335
private DocValuesWithOffsetsLoader docValues;
3436

3537
SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(String name, String offsetsFieldName) {
38+
this(name, offsetsFieldName, Function.identity());
39+
}
40+
41+
SortedSetWithOffsetsDocValuesSyntheticFieldLoaderLayer(String name, String offsetsFieldName, Function<BytesRef, BytesRef> converter) {
3642
this.name = Objects.requireNonNull(name);
3743
this.offsetsFieldName = Objects.requireNonNull(offsetsFieldName);
44+
this.converter = Objects.requireNonNull(converter);
3845
}
3946

4047
@Override
@@ -47,7 +54,7 @@ public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf
4754
SortedSetDocValues valueDocValues = DocValues.getSortedSet(leafReader, name);
4855
SortedDocValues offsetDocValues = DocValues.getSorted(leafReader, offsetsFieldName);
4956

50-
return docValues = new DocValuesWithOffsetsLoader(valueDocValues, offsetDocValues);
57+
return docValues = new DocValuesWithOffsetsLoader(valueDocValues, offsetDocValues, converter);
5158
}
5259

5360
@Override
@@ -78,15 +85,21 @@ public void write(XContentBuilder b) throws IOException {
7885
static final class DocValuesWithOffsetsLoader implements DocValuesLoader {
7986
private final SortedDocValues offsetDocValues;
8087
private final SortedSetDocValues valueDocValues;
88+
private final Function<BytesRef, BytesRef> converter;
8189
private final ByteArrayStreamInput scratch = new ByteArrayStreamInput();
8290

8391
private boolean hasValue;
8492
private boolean hasOffset;
8593
private int[] offsetToOrd;
8694

87-
DocValuesWithOffsetsLoader(SortedSetDocValues valueDocValues, SortedDocValues offsetDocValues) {
95+
DocValuesWithOffsetsLoader(
96+
SortedSetDocValues valueDocValues,
97+
SortedDocValues offsetDocValues,
98+
Function<BytesRef, BytesRef> converter
99+
) {
88100
this.valueDocValues = valueDocValues;
89101
this.offsetDocValues = offsetDocValues;
102+
this.converter = converter;
90103
}
91104

92105
@Override
@@ -146,7 +159,7 @@ public void write(XContentBuilder b) throws IOException {
146159

147160
long ord = ords[offset];
148161
BytesRef c = valueDocValues.lookupOrd(ord);
149-
// This is keyword specific and needs to be updated once support is added for other field types:
162+
c = converter.apply(c);
150163
b.utf8Value(c.bytes, c.offset, c.length);
151164
}
152165
} else if (offsetToOrd != null) {

‎server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,4 +439,9 @@ public void execute() {
439439
protected Function<Object, Object> loadBlockExpected() {
440440
return v -> InetAddresses.toAddrString(InetAddressPoint.decode(BytesRef.deepCopyOf((BytesRef) v).bytes));
441441
}
442+
443+
@Override
444+
protected String randomSyntheticSourceKeep() {
445+
return "all";
446+
}
442447
}

‎server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -349,16 +349,24 @@ public void testRangeQuery() {
349349
}
350350

351351
public void testFetchSourceValue() throws IOException {
352-
MappedFieldType mapper = new IpFieldMapper.Builder("field", ScriptCompiler.NONE, true, IndexVersion.current()).build(
353-
MapperBuilderContext.root(false, false)
354-
).fieldType();
352+
MappedFieldType mapper = new IpFieldMapper.Builder(
353+
"field",
354+
ScriptCompiler.NONE,
355+
true,
356+
IndexVersion.current(),
357+
Mapper.SourceKeepMode.NONE
358+
).build(MapperBuilderContext.root(false, false)).fieldType();
355359
assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8::2:1"));
356360
assertEquals(List.of("2001:db8::2:1"), fetchSourceValue(mapper, "2001:db8:0:0:0:0:2:1"));
357361
assertEquals(List.of("::1"), fetchSourceValue(mapper, "0:0:0:0:0:0:0:1"));
358362

359-
MappedFieldType nullValueMapper = new IpFieldMapper.Builder("field", ScriptCompiler.NONE, true, IndexVersion.current()).nullValue(
360-
"2001:db8:0:0:0:0:2:7"
361-
).build(MapperBuilderContext.root(false, false)).fieldType();
363+
MappedFieldType nullValueMapper = new IpFieldMapper.Builder(
364+
"field",
365+
ScriptCompiler.NONE,
366+
true,
367+
IndexVersion.current(),
368+
Mapper.SourceKeepMode.NONE
369+
).nullValue("2001:db8:0:0:0:0:2:7").build(MapperBuilderContext.root(false, false)).fieldType();
362370
assertEquals(List.of("2001:db8::2:7"), fetchSourceValue(nullValueMapper, null));
363371
}
364372
}

‎server/src/test/java/org/elasticsearch/index/mapper/IpOffsetDocValuesLoaderTests.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
import org.elasticsearch.common.network.NetworkAddress;
13+
14+
public class IpOffsetDocValuesLoaderTests extends OffsetDocValuesLoaderTestCase {
15+
16+
public void testOffsetArray() throws Exception {
17+
verifyOffsets("{\"field\":[\"192.168.1.1\",\"192.168.1.3\",\"192.168.1.2\",\"192.168.1.1\",\"192.168.1.9\",\"192.168.1.3\"]}");
18+
verifyOffsets("{\"field\":[\"192.168.1.4\",null,\"192.168.1.3\",\"192.168.1.2\",null,\"192.168.1.1\"]}");
19+
}
20+
21+
public void testOffsetNestedArray() throws Exception {
22+
verifyOffsets(
23+
"{\"field\":[\"192.168.1.2\",[\"192.168.1.1\"],[\"192.168.1.0\"],null,\"192.168.1.0\"]}",
24+
"{\"field\":[\"192.168.1.2\",\"192.168.1.1\",\"192.168.1.0\",null,\"192.168.1.0\"]}"
25+
);
26+
verifyOffsets(
27+
"{\"field\":[\"192.168.1.6\",[\"192.168.1.5\", [\"192.168.1.4\"]],[\"192.168.1.3\", [\"192.168.1.2\"]],null,\"192.168.1.1\"]}",
28+
"{\"field\":[\"192.168.1.6\",\"192.168.1.5\",\"192.168.1.4\",\"192.168.1.3\",\"192.168.1.2\",null,\"192.168.1.1\"]}"
29+
);
30+
}
31+
32+
@Override
33+
protected String getFieldTypeName() {
34+
return "ip";
35+
}
36+
37+
@Override
38+
protected String randomValue() {
39+
return NetworkAddress.format(randomIp(true));
40+
}
41+
}

0 commit comments

Comments
 (0)