Skip to content

Commit f5861a0

Browse files
jordan-powersjordan-powers
committed
Move field subset filtering into IgnoredSourceFormat
1 parent f6a68e9 commit f5861a0

File tree

6 files changed

+194
-184
lines changed

6 files changed

+194
-184
lines changed

‎server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ public Status needsField(FieldInfo fieldInfo) throws IOException {
121121

122122
@Override
123123
public void binaryField(FieldInfo fieldInfo, byte[] value) {
124-
var nameValues = IgnoredSourceFieldMapper.decodeMultipleValuesForField(new BytesRef(value));
124+
var nameValues = IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.decode(new BytesRef(value));
125125
assert nameValues.isEmpty() == false;
126126
String fieldPath = nameValues.getFirst().name();
127127

‎server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java‎

Lines changed: 163 additions & 110 deletions
Large diffs are not rendered by default.

‎server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import java.util.Set;
2727
import java.util.function.Consumer;
2828

29-
import static org.elasticsearch.index.mapper.IgnoredSourceFieldMapper.encodeMultipleValuesForField;
3029
import static org.hamcrest.Matchers.containsInAnyOrder;
3130

3231
/**
@@ -70,7 +69,7 @@ public void testSupports() {
7069
assertFalse(IgnoredSourceFieldLoader.supports(StoredFieldsSpec.NO_REQUIREMENTS));
7170
}
7271

73-
public IgnoredSourceFieldMapper.NameValue[] nameValue(String name, String... values) {
72+
private IgnoredSourceFieldMapper.NameValue[] nameValue(String name, String... values) {
7473
var nameValues = new IgnoredSourceFieldMapper.NameValue[values.length];
7574
for (int i = 0; i < values.length; i++) {
7675
nameValues[i] = new IgnoredSourceFieldMapper.NameValue(name, 0, new BytesRef(values[i]), null);
@@ -81,7 +80,7 @@ public IgnoredSourceFieldMapper.NameValue[] nameValue(String name, String... val
8180
public void testLoadSingle() throws IOException {
8281
var fooValue = nameValue("foo", "lorem ipsum");
8382
Document doc = new Document();
84-
doc.add(new StoredField("_ignored_source", encodeMultipleValuesForField(List.of(fooValue))));
83+
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue))));
8584
testLoader(doc, Set.of("foo"), ignoredSourceEntries -> {
8685
assertThat(ignoredSourceEntries, containsInAnyOrder(containsInAnyOrder(fooValue)));
8786
});
@@ -91,8 +90,8 @@ public void testLoadMultiple() throws IOException {
9190
var fooValue = nameValue("foo", "lorem ipsum");
9291
var barValue = nameValue("bar", "dolor sit amet");
9392
Document doc = new Document();
94-
doc.add(new StoredField("_ignored_source", encodeMultipleValuesForField(List.of(fooValue))));
95-
doc.add(new StoredField("_ignored_source", encodeMultipleValuesForField(List.of(barValue))));
93+
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue))));
94+
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(barValue))));
9695
testLoader(doc, Set.of("foo", "bar"), ignoredSourceEntries -> {
9796
assertThat(ignoredSourceEntries, containsInAnyOrder(containsInAnyOrder(fooValue), containsInAnyOrder(barValue)));
9897
});
@@ -103,8 +102,8 @@ public void testLoadSubset() throws IOException {
103102
var barValue = nameValue("bar", "dolor sit amet");
104103

105104
Document doc = new Document();
106-
doc.add(new StoredField("_ignored_source", encodeMultipleValuesForField(List.of(fooValue))));
107-
doc.add(new StoredField("_ignored_source", encodeMultipleValuesForField(List.of(barValue))));
105+
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue))));
106+
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(barValue))));
108107

109108
testLoader(doc, Set.of("foo"), ignoredSourceEntries -> {
110109
assertThat(ignoredSourceEntries, containsInAnyOrder(containsInAnyOrder(fooValue)));
@@ -114,7 +113,7 @@ public void testLoadSubset() throws IOException {
114113
public void testLoadFromParent() throws IOException {
115114
var fooValue = new IgnoredSourceFieldMapper.NameValue("parent", 7, new BytesRef("lorem ipsum"), null);
116115
Document doc = new Document();
117-
doc.add(new StoredField("_ignored_source", encodeMultipleValuesForField(List.of(fooValue))));
116+
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue))));
118117
testLoader(doc, Set.of("parent.foo"), ignoredSourceEntries -> {
119118
assertThat(ignoredSourceEntries, containsInAnyOrder(containsInAnyOrder(fooValue)));
120119
});

‎server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public void testEmpty() throws IOException {
6464
public void testSingleIgnoredSourceNewFormat() throws IOException {
6565
var fooValue = new IgnoredSourceFieldMapper.NameValue("foo", 0, new BytesRef("lorem ipsum"), null);
6666
var doc = new Document();
67-
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.encodeMultipleValuesForField(List.of(fooValue))));
67+
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue))));
6868
testIgnoredSourceLoader(
6969
doc,
7070
fieldsSpec(Set.of(), Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE),
@@ -88,8 +88,8 @@ public void testMultiValueIgnoredSourceNewFormat() throws IOException {
8888
var fooValue = new IgnoredSourceFieldMapper.NameValue("foo", 0, new BytesRef("lorem ipsum"), null);
8989
var barValue = new IgnoredSourceFieldMapper.NameValue("bar", 0, new BytesRef("dolor sit amet"), null);
9090
Document doc = new Document();
91-
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.encodeMultipleValuesForField(List.of(fooValue))));
92-
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.encodeMultipleValuesForField(List.of(barValue))));
91+
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue))));
92+
doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(barValue))));
9393
testIgnoredSourceLoader(
9494
doc,
9595
fieldsSpec(Set.of(), Set.of("foo", "bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE),

‎server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
package org.elasticsearch.index.mapper;
1111

1212
import org.apache.lucene.index.DirectoryReader;
13-
import org.apache.lucene.util.ArrayUtil;
1413
import org.elasticsearch.common.Strings;
1514
import org.elasticsearch.common.settings.Settings;
1615
import org.elasticsearch.core.CheckedConsumer;
@@ -343,11 +342,11 @@ public void testEncodeFieldToMap() throws IOException {
343342
String value = randomAlphaOfLength(5);
344343
ParsedDocument parsedDocument = getParsedDocumentWithFieldLimit(b -> b.field("my_value", value));
345344
IgnoredSourceFieldMapper.MappedNameValue mappedNameValue;
346-
byte[] bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue().bytes;
345+
var bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue();
347346
if (IgnoredSourceFieldMapper.COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled()) {
348-
mappedNameValue = IgnoredSourceFieldMapper.decodeAsMapMultipleFieldValues(bytes).getFirst();
347+
mappedNameValue = IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.decodeAsMap(bytes).getFirst();
349348
} else {
350-
mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(bytes);
349+
mappedNameValue = IgnoredSourceFieldMapper.LegacyIgnoredSourceEncoding.decodeAsMap(bytes);
351350
}
352351
assertEquals("my_value", mappedNameValue.nameValue().name());
353352
assertEquals(value, mappedNameValue.map().get("my_value"));
@@ -359,20 +358,19 @@ public void testEncodeObjectToMapAndDecode() throws IOException {
359358
ParsedDocument parsedDocument = getParsedDocumentWithFieldLimit(
360359
b -> { b.startObject("my_object").field("my_value", value).endObject(); }
361360
);
362-
var byteRef = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue();
363-
byte[] bytes = ArrayUtil.copyOfSubArray(byteRef.bytes, byteRef.offset, byteRef.length);
361+
var bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue();
364362
IgnoredSourceFieldMapper.MappedNameValue mappedNameValue;
365363
if (IgnoredSourceFieldMapper.COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled()) {
366-
mappedNameValue = IgnoredSourceFieldMapper.decodeAsMapMultipleFieldValues(bytes).getFirst();
364+
mappedNameValue = IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.decodeAsMap(bytes).getFirst();
367365
} else {
368-
mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(bytes);
366+
mappedNameValue = IgnoredSourceFieldMapper.LegacyIgnoredSourceEncoding.decodeAsMap(bytes);
369367
}
370368
assertEquals("my_object", mappedNameValue.nameValue().name());
371369
assertEquals(value, ((Map<String, ?>) mappedNameValue.map().get("my_object")).get("my_value"));
372370
if (IgnoredSourceFieldMapper.COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled()) {
373-
assertArrayEquals(bytes, IgnoredSourceFieldMapper.encodeFromMapMultipleFieldValues(List.of(mappedNameValue)));
371+
assertEquals(bytes, IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encodeFromMap(List.of(mappedNameValue)));
374372
} else {
375-
assertArrayEquals(bytes, IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue));
373+
assertEquals(bytes, IgnoredSourceFieldMapper.LegacyIgnoredSourceEncoding.encodeFromMap(mappedNameValue));
376374
}
377375
}
378376

@@ -383,20 +381,19 @@ public void testEncodeArrayToMapAndDecode() throws IOException {
383381
b.startObject().field("int_value", 20).endObject();
384382
b.endArray();
385383
});
386-
var byteRef = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue();
387-
byte[] bytes = ArrayUtil.copyOfSubArray(byteRef.bytes, byteRef.offset, byteRef.length);
384+
var bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue();
388385
IgnoredSourceFieldMapper.MappedNameValue mappedNameValue;
389386
if (IgnoredSourceFieldMapper.COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled()) {
390-
mappedNameValue = IgnoredSourceFieldMapper.decodeAsMapMultipleFieldValues(bytes).getFirst();
387+
mappedNameValue = IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.decodeAsMap(bytes).getFirst();
391388
} else {
392-
mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(bytes);
389+
mappedNameValue = IgnoredSourceFieldMapper.LegacyIgnoredSourceEncoding.decodeAsMap(bytes);
393390
}
394391
assertEquals("my_array", mappedNameValue.nameValue().name());
395392
assertThat((List<?>) mappedNameValue.map().get("my_array"), Matchers.contains(Map.of("int_value", 10), Map.of("int_value", 20)));
396393
if (IgnoredSourceFieldMapper.COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled()) {
397-
assertArrayEquals(bytes, IgnoredSourceFieldMapper.encodeFromMapMultipleFieldValues(List.of(mappedNameValue)));
394+
assertEquals(bytes, IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encodeFromMap(List.of(mappedNameValue)));
398395
} else {
399-
assertArrayEquals(bytes, IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue));
396+
assertEquals(bytes, IgnoredSourceFieldMapper.LegacyIgnoredSourceEncoding.encodeFromMap(mappedNameValue));
400397
}
401398
}
402399

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java‎

Lines changed: 7 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.lucene.index.Terms;
3030
import org.apache.lucene.index.TermsEnum;
3131
import org.apache.lucene.search.KnnCollector;
32+
import org.apache.lucene.util.ArrayUtil;
3233
import org.apache.lucene.util.Bits;
3334
import org.apache.lucene.util.BytesRef;
3435
import org.apache.lucene.util.FilterIterator;
@@ -402,52 +403,12 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
402403
XContentBuilder xContentBuilder = XContentBuilder.builder(result.v1().xContent()).map(transformedSource);
403404
visitor.binaryField(fieldInfo, BytesReference.toBytes(BytesReference.bytes(xContentBuilder)));
404405
} else if (IgnoredSourceFieldMapper.NAME.equals(fieldInfo.name)) {
405-
// TODO: move logic to IgnoredSourceFormat
406-
if (ignoredSourceFormat == IgnoredSourceFieldMapper.IgnoredSourceFormat.LEGACY_SINGLE_IGNORED_SOURCE) {
407-
// for _ignored_source, parse, filter out the field and its contents, and serialize back downstream
408-
IgnoredSourceFieldMapper.MappedNameValue mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(value);
409-
Map<String, Object> transformedField = filter(mappedNameValue.map(), filter, 0);
410-
if (transformedField.isEmpty() == false) {
411-
// The unfiltered map contains at least one element, the field name with its value. If the field contains
412-
// an object or an array, the value of the first element is a map or a list, respectively. Otherwise,
413-
// it's a single leaf value, e.g. a string or a number.
414-
var topValue = mappedNameValue.map().values().iterator().next();
415-
if (topValue instanceof Map<?, ?> || topValue instanceof List<?>) {
416-
// The field contains an object or an array, reconstruct it from the transformed map in case
417-
// any subfield has been filtered out.
418-
visitor.binaryField(
419-
fieldInfo,
420-
IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue.withMap(transformedField))
421-
);
422-
} else {
423-
// The field contains a leaf value, and it hasn't been filtered out. It is safe to propagate the original value.
424-
visitor.binaryField(fieldInfo, value);
425-
}
426-
}
427-
} else if (ignoredSourceFormat == IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) {
428-
List<IgnoredSourceFieldMapper.MappedNameValue> mappedNameValues = IgnoredSourceFieldMapper
429-
.decodeAsMapMultipleFieldValues(value);
430-
List<IgnoredSourceFieldMapper.MappedNameValue> filteredNameValues = new ArrayList<>(mappedNameValues.size());
431-
boolean didFilter = false;
432-
for (var mappedNameValue : mappedNameValues) {
433-
Map<String, Object> transformedField = filter(mappedNameValue.map(), filter, 0);
434-
if (transformedField.isEmpty()) {
435-
didFilter = true;
436-
continue;
437-
}
438-
var topValue = mappedNameValue.map().values().iterator().next();
439-
if (topValue instanceof Map<?, ?> || topValue instanceof List<?>) {
440-
didFilter = true;
441-
}
442-
filteredNameValues.add(mappedNameValue.withMap(transformedField));
443-
}
444-
if (didFilter) {
445-
if (filteredNameValues.isEmpty() == false) {
446-
visitor.binaryField(fieldInfo, IgnoredSourceFieldMapper.encodeFromMapMultipleFieldValues(filteredNameValues));
447-
}
448-
} else {
449-
assert false : "invalid ignored source format " + ignoredSourceFormat.name();
450-
}
406+
assert ignoredSourceFormat != IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE;
407+
BytesRef valueRef = new BytesRef(value);
408+
BytesRef filtered = ignoredSourceFormat.filterValue(valueRef, v -> filter(v, filter, 0));
409+
if (filtered != null) {
410+
byte[] filteredBytes = ArrayUtil.copyOfSubArray(filtered.bytes, filtered.offset, filtered.offset + filtered.length);
411+
visitor.binaryField(fieldInfo, filteredBytes);
451412
}
452413
} else {
453414
visitor.binaryField(fieldInfo, value);

0 commit comments

Comments
 (0)