Skip to content

Commit 0a33390

Browse files
lktslkts
committed
Getting closer to reading from parent object
1 parent ada0166 commit 0a33390

File tree

3 files changed

+138
-49
lines changed

3 files changed

+138
-49
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java‎

Lines changed: 119 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@
1616
import org.elasticsearch.xcontent.XContentParserConfiguration;
1717

1818
import java.io.IOException;
19+
import java.util.ArrayList;
20+
import java.util.HashMap;
21+
import java.util.HashSet;
22+
import java.util.List;
23+
import java.util.Map;
1924
import java.util.Optional;
2025
import java.util.Set;
2126

@@ -66,60 +71,132 @@ public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException
6671
throw new UnsupportedOperationException();
6772
}
6873

69-
private static class IgnoredSourceRowStrideReader implements RowStrideReader {
70-
private final String fieldName;
71-
private final Reader reader;
72-
73-
private IgnoredSourceRowStrideReader(String fieldName, Reader reader) {
74-
this.fieldName = fieldName;
75-
this.reader = reader;
76-
}
77-
74+
private record IgnoredSourceRowStrideReader(String fieldName, Reader reader) implements RowStrideReader {
7875
@Override
7976
public void read(int docId, StoredFields storedFields, Builder builder) throws IOException {
8077
var ignoredSource = storedFields.storedFields().get(IgnoredSourceFieldMapper.NAME);
8178
if (ignoredSource == null) {
8279
return;
8380
}
8481

85-
boolean written = false;
82+
Map<String, List<IgnoredSourceFieldMapper.NameValue>> valuesForFieldAndParents = new HashMap<>();
83+
84+
// Contains name of the field and all its parents
85+
Set<String> fieldNames = new HashSet<>() {
86+
{
87+
add("_doc");
88+
}
89+
};
90+
91+
var current = new StringBuilder();
92+
for (String part : fieldName.split("\\.")) {
93+
current.append(part);
94+
fieldNames.add(current.toString());
95+
}
96+
8697
for (Object value : ignoredSource) {
8798
IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(value);
88-
if (nameValue.name().equals(fieldName)) {
89-
// Leaf field is stored directly (not as a part of a parent object), let's try to decode it.
90-
Optional<Object> singleValue = XContentDataHelper.decode(nameValue.value());
91-
if (singleValue.isPresent()) {
92-
reader.readValue(singleValue.get(), builder);
93-
written = true;
94-
continue;
95-
}
96-
97-
// We have a value for this field but it's an array or an object
98-
var type = XContentDataHelper.decodeType(nameValue.value());
99-
assert type.isPresent();
100-
101-
var filterParserConfig = XContentParserConfiguration.EMPTY.withFiltering("", Set.of(fieldName), Set.of(), true);
102-
try (
103-
XContentParser parser = type.get()
104-
.xContent()
105-
.createParser(
106-
filterParserConfig,
107-
nameValue.value().bytes,
108-
nameValue.value().offset + 1,
109-
nameValue.value().length - 1
110-
)
111-
) {
112-
parser.nextToken();
113-
reader.parse(parser, builder);
114-
}
115-
written = true;
99+
if (fieldNames.contains(nameValue.name())) {
100+
valuesForFieldAndParents.computeIfAbsent(nameValue.name(), k -> new ArrayList<>()).add(nameValue);
101+
}
102+
}
103+
104+
// TODO figure out how to handle XContentDataHelper#voidValue()
105+
106+
if (readFromFieldValue(valuesForFieldAndParents.get(fieldName), builder)) {
107+
return;
108+
}
109+
if (readFromParentValue(valuesForFieldAndParents, builder)) {
110+
return;
111+
}
112+
113+
builder.appendNull();
114+
}
115+
116+
private boolean readFromFieldValue(List<IgnoredSourceFieldMapper.NameValue> nameValues, Builder builder) throws IOException {
117+
if (nameValues == null || nameValues.isEmpty()) {
118+
return false;
119+
}
120+
121+
// TODO this is not working properly
122+
if (nameValues.size() > 1) {
123+
builder.beginPositionEntry();
124+
}
125+
126+
for (var nameValue : nameValues) {
127+
// Leaf field is stored directly (not as a part of a parent object), let's try to decode it.
128+
Optional<Object> singleValue = XContentDataHelper.decode(nameValue.value());
129+
if (singleValue.isPresent()) {
130+
reader.readValue(singleValue.get(), builder);
131+
continue;
132+
}
133+
134+
// We have a value for this field but it's an array or an object
135+
var type = XContentDataHelper.decodeType(nameValue.value());
136+
assert type.isPresent();
137+
138+
try (
139+
XContentParser parser = type.get()
140+
.xContent()
141+
.createParser(XContentParserConfiguration.EMPTY, nameValue.value().bytes, nameValue.value().offset + 1, nameValue.value().length - 1)
142+
) {
143+
parser.nextToken();
144+
reader.parse(parser, builder);
116145
}
117-
// It is possible that the field is stored as part of the parent object, we'll need to look at those too and use something
118-
// similar to reader.parse()
119146
}
120147

121-
if (written == false) {
122-
builder.appendNull();
148+
if (nameValues.size() > 1) {
149+
builder.endPositionEntry();
150+
}
151+
152+
return true;
153+
}
154+
155+
private boolean readFromParentValue(Map<String, List<IgnoredSourceFieldMapper.NameValue>> valuesForFieldAndParents, Builder builder) throws IOException {
156+
if (valuesForFieldAndParents.isEmpty()) {
157+
return false;
158+
}
159+
160+
// If a parent object is stored at a particular level its children won't be stored.
161+
// So we should only ever have one parent here.
162+
assert valuesForFieldAndParents.size() == 1 : "_ignored_source field contains multiple levels of the same object";
163+
var parentValues = valuesForFieldAndParents.values().iterator().next();
164+
if (parentValues.size() > 1) {
165+
builder.beginPositionEntry();
166+
}
167+
168+
for (var nameValue : parentValues) {
169+
parseFieldFromParent(nameValue, builder);
170+
}
171+
172+
173+
if (parentValues.size() > 1) {
174+
builder.endPositionEntry();
175+
}
176+
177+
return true;
178+
}
179+
180+
private void parseFieldFromParent(IgnoredSourceFieldMapper.NameValue nameValue, Builder builder) throws IOException {
181+
var type = XContentDataHelper.decodeType(nameValue.value());
182+
assert type.isPresent();
183+
184+
var filterParserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, Set.of(fieldName), Set.of(), true);
185+
try (
186+
XContentParser parser = type.get()
187+
.xContent()
188+
.createParser(filterParserConfig, nameValue.value().bytes, nameValue.value().offset + 1, nameValue.value().length - 1)
189+
) {
190+
parser.nextToken();
191+
// boolean found = false;
192+
// do {
193+
// var token = parser.nextToken();
194+
// if (token == XContentParser.Token.FIELD_NAME && parser.currentName().equals(fieldName)) {
195+
// found = true;
196+
// }
197+
//
198+
// } while (found == false);
199+
reader.parse(parser, builder);
123200
}
124201
}
125202

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -648,13 +648,11 @@ public void parse(XContentParser parser, BlockLoader.Builder builder) throws IOE
648648

649649
var bytesRefBuilder = (BlockLoader.BytesRefBuilder) builder;
650650

651-
bytesRefBuilder.beginPositionEntry();
652651
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
653652
assert parser.currentToken() == XContentParser.Token.VALUE_STRING;
654653

655654
bytesRefBuilder.appendBytesRef(new BytesRef(parser.charBuffer()));
656655
}
657-
bytesRefBuilder.endPositionEntry();
658656
}
659657
};
660658

‎test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public void testBlockLoader() throws IOException {
6868

6969
var fieldValue = generator.generateValue();
7070

71-
Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, fieldValue);
71+
Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, fieldName, fieldValue);
7272
Object expected = expected(mapping.lookup().get(fieldName), fieldValue, syntheticSource);
7373
assertEquals(expected, blockLoaderResult);
7474
}
@@ -82,14 +82,28 @@ public void testSynth() throws IOException {
8282

8383
var fieldValue = generator.generateValue();
8484

85-
Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, fieldValue);
85+
Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, fieldName, fieldValue);
8686
Object expected = expected(mapping.lookup().get(fieldName), fieldValue, true);
8787
assertEquals(expected, blockLoaderResult);
8888
}
8989

90+
public void testSynthInObject() throws IOException {
91+
Map<String, Object> raw = Map.of("type", "keyword", "doc_values", false, "store", false);
92+
var mapping = new Mapping(Map.of("_doc", Map.of("properties", Map.of("obj", Map.of("enabled", "false", "properties", Map.of(fieldName, raw))))), Map.of("obj", Map.of("enabled", "false", "obj." + fieldName, raw)));
93+
var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw());
94+
95+
var mapperService = createSytheticSourceMapperService(mappingXContent);
96+
97+
var fieldValue = generator.generateValue();
98+
99+
Object blockLoaderResult = setupAndInvokeBlockLoader(mapperService, "obj." + fieldName, fieldValue);
100+
Object expected = expected(mapping.lookup().get("obj." + fieldName), fieldValue, true);
101+
assertEquals(expected, blockLoaderResult);
102+
}
103+
90104
protected abstract Object expected(Map<String, Object> fieldMapping, Object value, boolean syntheticSource);
91105

92-
private Object setupAndInvokeBlockLoader(MapperService mapperService, Object fieldValue) throws IOException {
106+
private Object setupAndInvokeBlockLoader(MapperService mapperService, String fieldName, Object fieldValue) throws IOException {
93107
try (Directory directory = newDirectory()) {
94108
RandomIndexWriter iw = new RandomIndexWriter(random(), directory);
95109

@@ -103,7 +117,7 @@ private Object setupAndInvokeBlockLoader(MapperService mapperService, Object fie
103117

104118
try (DirectoryReader reader = DirectoryReader.open(directory)) {
105119
LeafReaderContext context = reader.leaves().get(0);
106-
return load(createBlockLoader(mapperService), context, mapperService);
120+
return load(createBlockLoader(mapperService, fieldName), context, mapperService);
107121
}
108122
}
109123
}
@@ -137,7 +151,7 @@ private Object load(BlockLoader blockLoader, LeafReaderContext context, MapperSe
137151
return block.get(0);
138152
}
139153

140-
private BlockLoader createBlockLoader(MapperService mapperService) {
154+
private BlockLoader createBlockLoader(MapperService mapperService, String fieldName) {
141155
SearchLookup searchLookup = new SearchLookup(mapperService.mappingLookup().fieldTypesLookup()::get, null, null);
142156

143157
return mapperService.fieldType(fieldName).blockLoader(new MappedFieldType.BlockLoaderContext() {

0 commit comments

Comments
 (0)