Skip to content

Commit 5251092

Browse files
lktslkts
committed
support null_value, lots of debugging
1 parent 2a64271 commit 5251092

File tree

5 files changed

+83
-66
lines changed

5 files changed

+83
-66
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -207,16 +207,8 @@ private void parseFieldFromParent(IgnoredSourceFieldMapper.NameValue nameValue,
207207
}
208208

209209
private void parseWithReader(XContentParser parser, List<T> blockValues) throws IOException {
210-
if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
211-
return;
212-
}
213-
214210
if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
215211
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
216-
if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
217-
continue;
218-
}
219-
220212
reader.parse(parser, blockValues);
221213
}
222214
return;
@@ -253,4 +245,26 @@ public interface Reader<T> {
253245

254246
void writeToBlock(List<T> values, Builder blockBuilder);
255247
}
248+
249+
public abstract static class ReaderWithNullValueSupport<T> implements Reader<T> {
250+
private final T nullValue;
251+
252+
public ReaderWithNullValueSupport(T nullValue) {
253+
this.nullValue = nullValue;
254+
}
255+
256+
@Override
257+
public void parse(XContentParser parser, List<T> accumulator) throws IOException {
258+
if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
259+
if (nullValue != null) {
260+
convertValue(nullValue, accumulator);
261+
}
262+
return;
263+
}
264+
265+
parseNonNullValue(parser, accumulator);
266+
}
267+
268+
abstract void parseNonNullValue(XContentParser parser, List<T> accumulator) throws IOException;
269+
}
256270
}

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 34 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -742,47 +742,50 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
742742
}
743743

744744
if (isSyntheticSource) {
745-
var reader = new FallbackSyntheticSourceBlockLoader.Reader<BytesRef>() {
745+
return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) {
746746
@Override
747-
public void convertValue(Object value, List<BytesRef> accumulator) {
748-
String stringValue = ((BytesRef) value).utf8ToString();
749-
String adjusted = applyIgnoreAboveAndNormalizer(stringValue);
750-
if (adjusted != null) {
751-
// TODO what if the value didn't change?
752-
accumulator.add(new BytesRef(adjusted));
753-
}
747+
public Builder builder(BlockFactory factory, int expectedCount) {
748+
return factory.bytesRefs(expectedCount);
754749
}
750+
};
751+
}
755752

756-
@Override
757-
public void parse(XContentParser parser, List<BytesRef> accumulator) throws IOException {
758-
assert parser.currentToken() == XContentParser.Token.VALUE_STRING : "Unexpected token " + parser.currentToken();
753+
SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()));
754+
return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext));
755+
}
759756

760-
var value = applyIgnoreAboveAndNormalizer(parser.text());
761-
if (value != null) {
762-
accumulator.add(new BytesRef(value));
763-
}
757+
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
758+
var nullValueBytes = nullValue != null ? new BytesRef(nullValue) : null;
759+
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<>(nullValueBytes) {
760+
@Override
761+
public void convertValue(Object value, List<BytesRef> accumulator) {
762+
String stringValue = ((BytesRef) value).utf8ToString();
763+
String adjusted = applyIgnoreAboveAndNormalizer(stringValue);
764+
if (adjusted != null) {
765+
// TODO what if the value didn't change?
766+
accumulator.add(new BytesRef(adjusted));
764767
}
768+
}
765769

766-
@Override
767-
public void writeToBlock(List<BytesRef> values, BlockLoader.Builder blockBuilder) {
768-
var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder;
770+
@Override
771+
public void parseNonNullValue(XContentParser parser, List<BytesRef> accumulator) throws IOException {
772+
assert parser.currentToken() == XContentParser.Token.VALUE_STRING : "Unexpected token " + parser.currentToken();
769773

770-
for (var value : values) {
771-
bytesRefBuilder.appendBytesRef(value);
772-
}
774+
var value = applyIgnoreAboveAndNormalizer(parser.text());
775+
if (value != null) {
776+
accumulator.add(new BytesRef(value));
773777
}
774-
};
778+
}
775779

776-
return new FallbackSyntheticSourceBlockLoader(reader, name()) {
777-
@Override
778-
public Builder builder(BlockFactory factory, int expectedCount) {
779-
return factory.bytesRefs(expectedCount);
780-
}
781-
};
782-
}
780+
@Override
781+
public void writeToBlock(List<BytesRef> values, BlockLoader.Builder blockBuilder) {
782+
var bytesRefBuilder = (BlockLoader.BytesRefBuilder) blockBuilder;
783783

784-
SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()));
785-
return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext));
784+
for (var value : values) {
785+
bytesRefBuilder.appendBytesRef(value);
786+
}
787+
}
788+
};
786789
}
787790

788791
private BlockSourceReader.LeafIteratorLookup sourceBlockLoaderLookup(BlockLoaderContext blContext) {

‎server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
1414
import org.elasticsearch.logsdb.datageneration.FieldType;
1515

16-
import java.util.HashSet;
1716
import java.util.List;
1817
import java.util.Map;
1918
import java.util.Objects;
@@ -28,29 +27,30 @@ public KeywordFieldBlockLoaderTests() {
2827
@SuppressWarnings("unchecked")
2928
@Override
3029
protected Object expected(Map<String, Object> fieldMapping, Object value, boolean syntheticSource) {
31-
if (value == null) {
32-
return null;
33-
}
30+
var nullValue = (String) fieldMapping.get("null_value");
3431

3532
var ignoreAbove = fieldMapping.get("ignore_above") == null
3633
? Integer.MAX_VALUE
3734
: ((Number) fieldMapping.get("ignore_above")).intValue();
3835

36+
if (value == null) {
37+
return convert(null, nullValue, ignoreAbove);
38+
}
39+
3940
if (value instanceof String s) {
40-
return convert(s, ignoreAbove);
41+
return convert(s, nullValue, ignoreAbove);
4142
}
4243

43-
Function<Stream<String>, Stream<BytesRef>> convertValues = s -> s.map(v -> convert(v, ignoreAbove)).filter(Objects::nonNull);
44+
Function<Stream<String>, Stream<BytesRef>> convertValues = s -> s.map(v -> convert(v, nullValue, ignoreAbove))
45+
.filter(Objects::nonNull);
4446

4547
if ((boolean) fieldMapping.getOrDefault("doc_values", false)) {
4648
// Sorted and no duplicates
4749

48-
var values = new HashSet<>((List<String>) value);
49-
var resultList = Function.identity()
50-
.andThen(s -> values.stream().filter(Objects::nonNull).sorted())
51-
.andThen(convertValues)
50+
var resultList = convertValues.andThen(Stream::distinct)
51+
.andThen(Stream::sorted)
5252
.andThen(Stream::toList)
53-
.apply(values.stream());
53+
.apply(((List<String>) value).stream());
5454
return maybeFoldList(resultList);
5555
}
5656

@@ -71,9 +71,13 @@ private Object maybeFoldList(List<?> list) {
7171
return list;
7272
}
7373

74-
private BytesRef convert(String value, int ignoreAbove) {
74+
private BytesRef convert(String value, String nullValue, int ignoreAbove) {
7575
if (value == null) {
76-
return null;
76+
if (nullValue != null) {
77+
value = nullValue;
78+
} else {
79+
return null;
80+
}
7781
}
7882

7983
return value.length() <= ignoreAbove ? new BytesRef(value) : null;

‎test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import java.util.List;
3737
import java.util.Map;
3838
import java.util.Set;
39+
import java.util.stream.Stream;
3940

4041
public abstract class BlockLoaderTestCase extends MapperServiceTestCase {
4142
private final FieldType fieldType;
@@ -114,29 +115,21 @@ private void runTest(Template template, String fieldName) throws IOException {
114115
protected abstract Object expected(Map<String, Object> fieldMapping, Object value, boolean syntheticSource);
115116

116117
private Object getFieldValue(Map<String, Object> document, String fieldName) {
117-
var results = new ArrayList<>();
118-
processLevel(document, fieldName, results);
118+
var rawValues = new ArrayList<>();
119+
processLevel(document, fieldName, rawValues);
119120

120-
if (results.isEmpty()) {
121-
return null;
122-
}
123-
if (results.size() == 1) {
124-
return results.get(0);
121+
if (rawValues.size() == 1) {
122+
return rawValues.get(0);
125123
}
126124

127-
return results;
125+
return rawValues.stream().flatMap(v -> v instanceof List<?> l ? l.stream() : Stream.of(v)).toList();
128126
}
129127

130128
@SuppressWarnings("unchecked")
131129
private void processLevel(Map<String, Object> level, String field, ArrayList<Object> values) {
132130
if (field.contains(".") == false) {
133131
var value = level.get(field);
134-
if (value instanceof List<?> l) {
135-
values.addAll(l);
136-
} else {
137-
values.add(value);
138-
}
139-
132+
values.add(value);
140133
return;
141134
}
142135

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ private Supplier<Map<String, Object>> keywordMapping(
6868
if (ESTestCase.randomDouble() <= 0.2) {
6969
injected.put("ignore_above", ESTestCase.randomIntBetween(1, 100));
7070
}
71+
if (ESTestCase.randomDouble() <= 0.2) {
72+
injected.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
73+
}
7174

7275
return injected;
7376
};

0 commit comments

Comments
 (0)