diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java index 0675632c0cc10..11100884076f5 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java @@ -14,11 +14,13 @@ import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -38,8 +40,8 @@ class IgnoredSourceFieldLoader extends StoredFieldLoader { HashMap> potentialFieldsInIgnoreSource = new HashMap<>(); for (String requiredIgnoredField : spec.ignoredFieldsSpec().requiredIgnoredFields()) { - for (String potentialStoredField : spec.ignoredFieldsSpec().format().requiredStoredFields(requiredIgnoredField)) { - potentialFieldsInIgnoreSource.computeIfAbsent(potentialStoredField, k -> new HashSet<>()).add(requiredIgnoredField); + for (String potentialIgnoredField : FallbackSyntheticSourceBlockLoader.splitIntoFieldPaths(requiredIgnoredField)) { + potentialFieldsInIgnoreSource.computeIfAbsent(potentialIgnoredField, k -> new HashSet<>()).add(requiredIgnoredField); } } this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource; @@ -82,7 +84,7 @@ public String routing() { @Override public Map> storedFields() { - return visitor.values; + return Map.of(IgnoredSourceFieldMapper.NAME, Collections.unmodifiableList(visitor.values)); } }; } @@ -93,7 +95,7 @@ public List fieldsToLoad() { } static class SFV extends StoredFieldVisitor { - final Map> values = new HashMap<>(); + List> values = new ArrayList<>(); final Set fieldNames; private final Set unvisitedFields; final Map> potentialFieldsInIgnoreSource; @@ -110,18 +112,26 @@ public Status needsField(FieldInfo fieldInfo) throws IOException { return Status.STOP; } - Set foundFields = potentialFieldsInIgnoreSource.get(fieldInfo.name); - if (foundFields == null) { - return Status.NO; + if (fieldInfo.name.equals(IgnoredSourceFieldMapper.NAME)) { + return Status.YES; } - unvisitedFields.removeAll(foundFields); - return Status.YES; + return Status.NO; } @Override public void binaryField(FieldInfo fieldInfo, byte[] value) { - values.computeIfAbsent(fieldInfo.name, k -> new ArrayList<>()).add(new BytesRef(value)); + var nameValues = IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.decode(new BytesRef(value)); + assert nameValues.isEmpty() == false; + String fieldPath = nameValues.getFirst().name(); + + Set foundValues = potentialFieldsInIgnoreSource.get(fieldPath); + if (foundValues == null) { + return; + } + + unvisitedFields.removeAll(foundValues); + values.add(nameValues); } void reset() { @@ -133,6 +143,6 @@ void reset() { static boolean supports(StoredFieldsSpec spec) { return spec.onlyRequiresIgnoredFields() - && spec.ignoredFieldsSpec().format() == IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE; + && spec.ignoredFieldsSpec().format() == IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE; } } diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java index effb3ce8f6fff..aa3357408f38e 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.fieldvisitor; -import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.StoredFieldVisitor; @@ -17,7 +16,6 @@ import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; -import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; @@ -202,29 +200,13 @@ private static class ReaderStoredFieldLoader implements LeafStoredFieldLoader { private final CustomFieldsVisitor visitor; private int doc = -1; - private static CustomFieldsVisitor getFieldsVisitor(Set fields, boolean loadSource) { - if (fields.contains(IgnoredSourceFieldMapper.NAME)) { - return new CustomFieldsVisitor(fields, loadSource) { - @Override - public Status needsField(FieldInfo fieldInfo) { - if (fieldInfo.name.startsWith(IgnoredSourceFieldMapper.NAME)) { - return Status.YES; - } - return super.needsField(fieldInfo); - } - }; - } - - return new CustomFieldsVisitor(fields, loadSource); - } - ReaderStoredFieldLoader( CheckedBiConsumer reader, boolean loadSource, Set fields ) { this.reader = reader; - this.visitor = getFieldsVisitor(fields, loadSource); + this.visitor = new CustomFieldsVisitor(fields, loadSource); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java index e7531da294489..b8c842eefb836 100644 --- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java +++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java @@ -244,7 +244,7 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params for (DocumentField field : metaFields.values()) { // TODO: can we avoid having an exception here? - if (field.getName().equals(IgnoredFieldMapper.NAME) || field.getName().startsWith(IgnoredSourceFieldMapper.NAME)) { + if (field.getName().equals(IgnoredFieldMapper.NAME) || field.getName().equals(IgnoredSourceFieldMapper.NAME)) { builder.field(field.getName(), field.getValues()); } else { builder.field(field.getName(), field.getValue()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java index cdbc7621024c0..19d2b37c9a308 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java @@ -13,7 +13,6 @@ import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; /** * Defines which fields need to be loaded from _ignored_source during a fetch. @@ -54,7 +53,6 @@ public IgnoredFieldsSpec merge(IgnoredFieldsSpec other) { * Get the set of stored fields required to load the specified fields from _ignored_source. */ public Set requiredStoredFields() { - return requiredIgnoredFields.stream().flatMap(field -> format.requiredStoredFields(field).stream()).collect(Collectors.toSet()); - + return Set.of(IgnoredSourceFieldMapper.NAME); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 24f667d38bbc7..423289250e08a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -11,7 +11,6 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.bytes.BytesArray; @@ -41,7 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; +import java.util.function.Function; import java.util.stream.Stream; /** @@ -76,7 +75,7 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { "mapper.ignored_source.always_store_object_arrays_in_nested" ); - public static final FeatureFlag IGNORED_SOURCE_FIELDS_PER_ENTRY_FF = new FeatureFlag("ignored_source_fields_per_entry"); + public static final FeatureFlag COALESCE_IGNORED_SOURCE_ENTRIES = new FeatureFlag("ignored_source_fields_per_entry"); /* Setting to disable encoding and writing values for this field. @@ -178,99 +177,118 @@ public void postParse(DocumentParserContext context) { ignoredSourceFormat(context.indexSettings().getIndexVersionCreated()).writeIgnoredFields(context.getIgnoredFieldValues()); } - public static String ignoredFieldName(String fieldName) { - return NAME + "." + fieldName; - } - - static BytesRef encodeMultipleValuesForField(List values) { - assert values.isEmpty() == false; - try { - BytesStreamOutput stream = new BytesStreamOutput(); - stream.writeVInt(values.size()); - String fieldName = values.getFirst().name; - stream.writeString(fieldName); - for (var value : values) { - assert fieldName.equals(value.name); - stream.writeVInt(value.parentOffset); - stream.writeBytesRef(value.value); - } - return stream.bytes().toBytesRef(); - } catch (IOException e) { - throw new ElasticsearchException("Failed to encode _ignored_source", e); + // In rare cases decoding values stored in this field can fail leading to entire source + // not being available. + // We would like to have an option to lose some values in synthetic source + // but have search not fail. + public static Set ensureLoaded(Set fieldsToLoadForSyntheticSource, IndexSettings indexSettings) { + if (indexSettings.getSkipIgnoredSourceRead() == false) { + fieldsToLoadForSyntheticSource.add(NAME); } + + return fieldsToLoadForSyntheticSource; } - static List decodeMultipleValuesForField(BytesRef value) { - try { - StreamInput stream = new BytesArray(value).streamInput(); - var count = stream.readVInt(); - assert count >= 1; - String fieldName = stream.readString(); - List values = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - int parentOffset = stream.readVInt(); - BytesRef valueBytes = stream.readBytesRef(); - values.add(new NameValue(fieldName, parentOffset, valueBytes, null)); - } - return values; - } catch (IOException e) { - throw new ElasticsearchException("Failed to decode _ignored_source", e); + public static class LegacyIgnoredSourceEncoding { + public static BytesRef encode(NameValue values) { + assert values.parentOffset < PARENT_OFFSET_IN_NAME_OFFSET; + assert values.parentOffset * (long) PARENT_OFFSET_IN_NAME_OFFSET < Integer.MAX_VALUE; + + byte[] nameBytes = values.name.getBytes(StandardCharsets.UTF_8); + byte[] bytes = new byte[4 + nameBytes.length + values.value.length]; + ByteUtils.writeIntLE(values.name.length() + PARENT_OFFSET_IN_NAME_OFFSET * values.parentOffset, bytes, 0); + System.arraycopy(nameBytes, 0, bytes, 4, nameBytes.length); + System.arraycopy(values.value.bytes, values.value.offset, bytes, 4 + nameBytes.length, values.value.length); + return new BytesRef(bytes); } - } - static byte[] encode(NameValue values) { - assert values.parentOffset < PARENT_OFFSET_IN_NAME_OFFSET; - assert values.parentOffset * (long) PARENT_OFFSET_IN_NAME_OFFSET < Integer.MAX_VALUE; + public static NameValue decode(Object field) { + byte[] bytes = ((BytesRef) field).bytes; + int encodedSize = ByteUtils.readIntLE(bytes, 0); + int nameSize = encodedSize % PARENT_OFFSET_IN_NAME_OFFSET; + int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET; - byte[] nameBytes = values.name.getBytes(StandardCharsets.UTF_8); - byte[] bytes = new byte[4 + nameBytes.length + values.value.length]; - ByteUtils.writeIntLE(values.name.length() + PARENT_OFFSET_IN_NAME_OFFSET * values.parentOffset, bytes, 0); - System.arraycopy(nameBytes, 0, bytes, 4, nameBytes.length); - System.arraycopy(values.value.bytes, values.value.offset, bytes, 4 + nameBytes.length, values.value.length); - return bytes; - } + String decoded = new String(bytes, 4, bytes.length - 4, StandardCharsets.UTF_8); + String name = decoded.substring(0, nameSize); + int nameByteCount = name.getBytes(StandardCharsets.UTF_8).length; - static NameValue decode(Object field) { - byte[] bytes = ((BytesRef) field).bytes; - int encodedSize = ByteUtils.readIntLE(bytes, 0); - int nameSize = encodedSize % PARENT_OFFSET_IN_NAME_OFFSET; - int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET; + BytesRef value = new BytesRef(bytes, 4 + nameByteCount, bytes.length - nameByteCount - 4); + return new NameValue(name, parentOffset, value, null); + } - String decoded = new String(bytes, 4, bytes.length - 4, StandardCharsets.UTF_8); - String name = decoded.substring(0, nameSize); - int nameByteCount = name.getBytes(StandardCharsets.UTF_8).length; + public static BytesRef encodeFromMap(MappedNameValue mappedNameValue) throws IOException { + return encode(mappedToNameValue(mappedNameValue)); + } - BytesRef value = new BytesRef(bytes, 4 + nameByteCount, bytes.length - nameByteCount - 4); - return new NameValue(name, parentOffset, value, null); + public static MappedNameValue decodeAsMap(BytesRef value) throws IOException { + return nameValueToMapped(decode(value)); + } } - // In rare cases decoding values stored in this field can fail leading to entire source - // not being available. - // We would like to have an option to lose some values in synthetic source - // but have search not fail. - public static Set ensureLoaded(Set fieldsToLoadForSyntheticSource, IndexSettings indexSettings) { - if (indexSettings.getSkipIgnoredSourceRead() == false) { - fieldsToLoadForSyntheticSource.add(NAME); + public static class CoalescedIgnoredSourceEncoding { + public static BytesRef encode(List values) { + assert values.isEmpty() == false; + try { + BytesStreamOutput stream = new BytesStreamOutput(); + stream.writeVInt(values.size()); + String fieldName = values.getFirst().name; + stream.writeString(fieldName); + for (var value : values) { + assert fieldName.equals(value.name); + stream.writeVInt(value.parentOffset); + stream.writeBytesRef(value.value); + } + return stream.bytes().toBytesRef(); + } catch (IOException e) { + throw new ElasticsearchException("Failed to encode _ignored_source", e); + } } - return fieldsToLoadForSyntheticSource; + public static List decode(BytesRef value) { + try { + StreamInput stream = new BytesArray(value).streamInput(); + var count = stream.readVInt(); + assert count >= 1; + String fieldName = stream.readString(); + List values = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + int parentOffset = stream.readVInt(); + BytesRef valueBytes = stream.readBytesRef(); + values.add(new NameValue(fieldName, parentOffset, valueBytes, null)); + } + return values; + } catch (IOException e) { + throw new ElasticsearchException("Failed to decode _ignored_source", e); + } + } + + public static BytesRef encodeFromMap(List filteredValues) throws IOException { + List filteredNameValues = new ArrayList<>(filteredValues.size()); + for (var filteredValue : filteredValues) { + filteredNameValues.add(mappedToNameValue(filteredValue)); + } + return encode(filteredNameValues); + } + + public static List decodeAsMap(BytesRef value) throws IOException { + List nameValues = decode(value); + List mappedValues = new ArrayList<>(nameValues.size()); + for (var nameValue : nameValues) { + mappedValues.add(nameValueToMapped(nameValue)); + } + return mappedValues; + } } public enum IgnoredSourceFormat { NO_IGNORED_SOURCE { @Override - public Map> loadAllIgnoredFields( - SourceFilter filter, - Map> storedFields - ) { + public Map> loadAllIgnoredFields(SourceFilter filter, Map> storedFields) { return Map.of(); } @Override - public Map> loadSingleIgnoredField( - Set fieldPaths, - Map> storedFields - ) { + public Map> loadSingleIgnoredField(Set fieldPaths, Map> storedFields) { return Map.of(); } @@ -280,24 +298,22 @@ public void writeIgnoredFields(Collection ignoredFieldValues) { } @Override - public Set requiredStoredFields(String fieldName) { - return Set.of(); + public BytesRef filterValue(BytesRef value, Function, Map> filter) { + assert false : "cannot filter ignored source with format NO_IGNORED_SOURCE"; + return null; } }, - SINGLE_IGNORED_SOURCE { + LEGACY_SINGLE_IGNORED_SOURCE { @Override - public Map> loadAllIgnoredFields( - SourceFilter filter, - Map> storedFields - ) { - Map> objectsWithIgnoredFields = null; - List storedValues = storedFields.get(IgnoredSourceFieldMapper.NAME); + public Map> loadAllIgnoredFields(SourceFilter filter, Map> storedFields) { + Map> objectsWithIgnoredFields = null; + List storedValues = storedFields.get(NAME); if (storedValues != null) { for (Object value : storedValues) { if (objectsWithIgnoredFields == null) { objectsWithIgnoredFields = new HashMap<>(); } - IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(value); + NameValue nameValue = LegacyIgnoredSourceEncoding.decode(value); if (filter != null && filter.isPathFiltered(nameValue.name(), XContentDataHelper.isEncodedObject(nameValue.value()))) { // This path is filtered by the include/exclude rules @@ -310,15 +326,12 @@ public Map> loadAllIgnoredField } @Override - public Map> loadSingleIgnoredField( - Set fieldPaths, - Map> storedFields - ) { - Map> valuesForFieldAndParents = new HashMap<>(); - var ignoredSource = storedFields.get(IgnoredSourceFieldMapper.NAME); + public Map> loadSingleIgnoredField(Set fieldPaths, Map> storedFields) { + Map> valuesForFieldAndParents = new HashMap<>(); + var ignoredSource = storedFields.get(NAME); if (ignoredSource != null) { for (Object value : ignoredSource) { - IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(value); + NameValue nameValue = LegacyIgnoredSourceEncoding.decode(value); if (fieldPaths.contains(nameValue.name())) { valuesForFieldAndParents.computeIfAbsent(nameValue.name(), k -> new ArrayList<>()).add(nameValue); } @@ -330,67 +343,81 @@ public Map> loadSingleIgnoredFi @Override public void writeIgnoredFields(Collection ignoredFieldValues) { for (NameValue nameValue : ignoredFieldValues) { - nameValue.doc().add(new StoredField(NAME, encode(nameValue))); + nameValue.doc().add(new StoredField(NAME, LegacyIgnoredSourceEncoding.encode(nameValue))); } } @Override - public Set requiredStoredFields(String fieldName) { - return Set.of(IgnoredSourceFieldMapper.NAME); + public BytesRef filterValue(BytesRef value, Function, Map> filter) throws IOException { + // for _ignored_source, parse, filter out the field and its contents, and serialize back downstream + IgnoredSourceFieldMapper.MappedNameValue mappedNameValue = LegacyIgnoredSourceEncoding.decodeAsMap(value); + Map transformedField = filter.apply(mappedNameValue.map()); + if (transformedField.isEmpty()) { + // All values were filtered + return null; + } + // The unfiltered map contains at least one element, the field name with its value. If the field contains + // an object or an array, the value of the first element is a map or a list, respectively. Otherwise, + // it's a single leaf value, e.g. a string or a number. + var topValue = mappedNameValue.map().values().iterator().next(); + if (topValue instanceof Map || topValue instanceof List) { + // The field contains an object or an array, reconstruct it from the transformed map in case + // any subfield has been filtered out. + return LegacyIgnoredSourceEncoding.encodeFromMap(mappedNameValue.withMap(transformedField)); + } else { + // The field contains a leaf value, and it hasn't been filtered out. It is safe to propagate the original value. + return value; + } } }, - PER_FIELD_IGNORED_SOURCE { + COALESCED_SINGLE_IGNORED_SOURCE { @Override - public Map> loadAllIgnoredFields( - SourceFilter filter, - Map> storedFields - ) { - Map> objectsWithIgnoredFields = null; - for (Map.Entry> e : storedFields.entrySet()) { - if (e.getKey().startsWith(IgnoredSourceFieldMapper.NAME)) { - assert e.getValue().size() == 1; - - Object value = e.getValue().getFirst(); - if (objectsWithIgnoredFields == null) { - objectsWithIgnoredFields = new HashMap<>(); - } - List nameValues = IgnoredSourceFieldMapper.decodeMultipleValuesForField( - (BytesRef) value - ); - - for (var nameValue : nameValues) { - if (filter != null - && filter.isPathFiltered(nameValue.name(), XContentDataHelper.isEncodedObject(nameValue.value()))) { - // This path is filtered by the include/exclude rules - continue; - } - objectsWithIgnoredFields.computeIfAbsent(nameValue.getParentFieldName(), k -> new ArrayList<>()).add(nameValue); + public Map> loadAllIgnoredFields(SourceFilter filter, Map> storedFields) { + Map> objectsWithIgnoredFields = null; + var ignoredSource = storedFields.get(NAME); + if (ignoredSource == null) { + return objectsWithIgnoredFields; + } + for (var ignoredSourceEntry : ignoredSource) { + if (objectsWithIgnoredFields == null) { + objectsWithIgnoredFields = new HashMap<>(); + } + + @SuppressWarnings("unchecked") + List nameValues = (ignoredSourceEntry instanceof List) + ? (List) ignoredSourceEntry + : CoalescedIgnoredSourceEncoding.decode((BytesRef) ignoredSourceEntry); + assert nameValues.isEmpty() == false; + + for (var nameValue : nameValues) { + if (filter != null + && filter.isPathFiltered(nameValue.name(), XContentDataHelper.isEncodedObject(nameValue.value()))) { + // This path is filtered by the include/exclude rules + continue; } + objectsWithIgnoredFields.computeIfAbsent(nameValue.getParentFieldName(), k -> new ArrayList<>()).add(nameValue); } } return objectsWithIgnoredFields; } @Override - public Map> loadSingleIgnoredField( - Set fieldPaths, - Map> storedFields - ) { - Map> valuesForFieldAndParents = new HashMap<>(); - for (var parentPath : fieldPaths) { - var ignoredSource = storedFields.get(IgnoredSourceFieldMapper.ignoredFieldName(parentPath)); - if (ignoredSource == null) { - continue; - } - assert ignoredSource.size() == 1; - - List nameValues = IgnoredSourceFieldMapper.decodeMultipleValuesForField( - (BytesRef) ignoredSource.getFirst() - ); - - for (var nameValue : nameValues) { - assert fieldPaths.contains(nameValue.name()); - valuesForFieldAndParents.computeIfAbsent(nameValue.name(), k -> new ArrayList<>()).add(nameValue); + public Map> loadSingleIgnoredField(Set fieldPaths, Map> storedFields) { + Map> valuesForFieldAndParents = new HashMap<>(); + var ignoredSource = storedFields.get(NAME); + if (ignoredSource == null) { + return valuesForFieldAndParents; + } + for (var ignoredSourceEntry : ignoredSource) { + @SuppressWarnings("unchecked") + List nameValues = (ignoredSourceEntry instanceof List) + ? (List) ignoredSourceEntry + : CoalescedIgnoredSourceEncoding.decode((BytesRef) ignoredSourceEntry); + assert nameValues.isEmpty() == false; + String fieldPath = nameValues.getFirst().name(); + if (fieldPaths.contains(fieldPath)) { + assert valuesForFieldAndParents.containsKey(fieldPath) == false; + valuesForFieldAndParents.put(fieldPath, nameValues); } } @@ -402,44 +429,57 @@ public void writeIgnoredFields(Collection ignoredFieldValues) { Map>> entriesMap = new HashMap<>(); for (NameValue nameValue : ignoredFieldValues) { - String fieldName = ignoredFieldName(nameValue.name()); entriesMap.computeIfAbsent(nameValue.doc(), d -> new HashMap<>()) - .computeIfAbsent(fieldName, n -> new ArrayList<>()) + .computeIfAbsent(nameValue.name(), n -> new ArrayList<>()) .add(nameValue); } for (var docEntry : entriesMap.entrySet()) { for (var fieldEntry : docEntry.getValue().entrySet()) { - docEntry.getKey().add(new StoredField(fieldEntry.getKey(), encodeMultipleValuesForField(fieldEntry.getValue()))); + docEntry.getKey().add(new StoredField(NAME, CoalescedIgnoredSourceEncoding.encode(fieldEntry.getValue()))); } } } @Override - public Set requiredStoredFields(String fieldName) { - return FallbackSyntheticSourceBlockLoader.splitIntoFieldPaths(fieldName) - .stream() - .map(IgnoredSourceFieldMapper::ignoredFieldName) - .collect(Collectors.toSet()); + public BytesRef filterValue(BytesRef value, Function, Map> filter) throws IOException { + List mappedNameValues = CoalescedIgnoredSourceEncoding.decodeAsMap(value); + List filteredNameValues = new ArrayList<>(mappedNameValues.size()); + boolean maybeDidFilter = false; + for (var mappedNameValue : mappedNameValues) { + Map transformedField = filter.apply(mappedNameValue.map()); + if (transformedField.isEmpty()) { + maybeDidFilter = true; + continue; + } + var topValue = mappedNameValue.map().values().iterator().next(); + if (topValue instanceof Map || topValue instanceof List) { + // The field contains an object or an array in which some subfield may have been filtered out + maybeDidFilter = true; + } + filteredNameValues.add(mappedNameValue.withMap(transformedField)); + } + if (maybeDidFilter) { + if (filteredNameValues.isEmpty()) { + // All values were filtered + return null; + } else { + return CoalescedIgnoredSourceEncoding.encodeFromMap(filteredNameValues); + } + } else { + // The field contains a leaf value, and it hasn't been filtered out. It is safe to propagate the original value. + return value; + } } }; - public abstract Map> loadAllIgnoredFields( - SourceFilter filter, - Map> storedFields - ); + public abstract Map> loadAllIgnoredFields(SourceFilter filter, Map> storedFields); - public abstract Map> loadSingleIgnoredField( - Set fieldPaths, - Map> storedFields - ); + public abstract Map> loadSingleIgnoredField(Set fieldPaths, Map> storedFields); public abstract void writeIgnoredFields(Collection ignoredFieldValues); - /** - * Get the set of stored fields needed to retrieve the value for fieldName - */ - public abstract Set requiredStoredFields(String fieldName); + public abstract BytesRef filterValue(BytesRef value, Function, Map> filter) throws IOException; } public IgnoredSourceFormat ignoredSourceFormat() { @@ -448,9 +488,9 @@ public IgnoredSourceFormat ignoredSourceFormat() { public static IgnoredSourceFormat ignoredSourceFormat(IndexVersion indexCreatedVersion) { return indexCreatedVersion.onOrAfter(IndexVersions.IGNORED_SOURCE_FIELDS_PER_ENTRY_WITH_FF) - && IGNORED_SOURCE_FIELDS_PER_ENTRY_FF.isEnabled() - ? IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE - : IgnoredSourceFormat.SINGLE_IGNORED_SOURCE; + && COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled() + ? IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE + : IgnoredSourceFormat.LEGACY_SINGLE_IGNORED_SOURCE; } @Override @@ -499,36 +539,17 @@ public void reset() { }); } + /** + * A parsed NameValue alongside its value decoded to a map of maps that corresponds to the field-value subtree. There is only a single + * pair at the top level, with the key corresponding to the field name. If the field contains a single value, the map contains a single + * key-value pair. Otherwise, the value of the first pair will be another map etc. + */ public record MappedNameValue(NameValue nameValue, XContentType type, Map map) { public MappedNameValue withMap(Map map) { return new MappedNameValue(new NameValue(nameValue.name, nameValue.parentOffset, null, nameValue.doc), type, map); } } - /** - * Parses the passed byte array as a NameValue and converts its decoded value to a map of maps that corresponds to the field-value - * subtree. There is only a single pair at the top level, with the key corresponding to the field name. If the field contains a single - * value, the map contains a single key-value pair. Otherwise, the value of the first pair will be another map etc. - * @param value encoded NameValue - * @return MappedNameValue with the parsed NameValue, the XContentType to use for serializing its contents and the field-value map. - * @throws IOException - */ - public static MappedNameValue decodeAsMap(byte[] value) throws IOException { - BytesRef bytes = new BytesRef(value); - IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(bytes); - return nameValueToMapped(nameValue); - } - - public static List decodeAsMapMultipleFieldValues(byte[] value) throws IOException { - BytesRef bytes = new BytesRef(value); - List nameValues = decodeMultipleValuesForField(bytes); - List mappedValues = new ArrayList<>(nameValues.size()); - for (var nameValue : nameValues) { - mappedValues.add(nameValueToMapped(nameValue)); - } - return mappedValues; - } - private static MappedNameValue nameValueToMapped(NameValue nameValue) throws IOException { XContentBuilder xContentBuilder = XContentBuilder.builder(XContentDataHelper.getXContentType(nameValue.value()).xContent()); xContentBuilder.startObject().field(nameValue.name()); @@ -538,27 +559,7 @@ private static MappedNameValue nameValueToMapped(NameValue nameValue) throws IOE return new MappedNameValue(nameValue, result.v1(), result.v2()); } - /** - * Clones the passed NameValue, using the passed map to produce its value. - * @param mappedNameValue containing the NameValue to clone and the map containing a simple field-value pair, or a deeper - * field-value subtree for objects and arrays with fields - * @return a byte array containing the encoding form of the cloned NameValue - * @throws IOException - */ - public static byte[] encodeFromMap(MappedNameValue mappedNameValue) throws IOException { - return IgnoredSourceFieldMapper.encode(mappedToNameValue(mappedNameValue)); - } - - public static byte[] encodeFromMapMultipleFieldValues(List filteredValues) throws IOException { - List filteredNameValues = new ArrayList<>(filteredValues.size()); - for (var filteredValue : filteredValues) { - filteredNameValues.add(mappedToNameValue(filteredValue)); - } - var encoded = encodeMultipleValuesForField(filteredNameValues); - return ArrayUtil.copyOfSubArray(encoded.bytes, encoded.offset, encoded.length); - } - - private static IgnoredSourceFieldMapper.NameValue mappedToNameValue(MappedNameValue mappedNameValue) throws IOException { + private static NameValue mappedToNameValue(MappedNameValue mappedNameValue) throws IOException { // The first entry is the field name, we skip to get to the value to encode. assert mappedNameValue.map.size() == 1; Object content = mappedNameValue.map.values().iterator().next(); @@ -571,11 +572,12 @@ private static IgnoredSourceFieldMapper.NameValue mappedToNameValue(MappedNameVa // Clone the NameValue with the updated value. NameValue oldNameValue = mappedNameValue.nameValue(); - return new IgnoredSourceFieldMapper.NameValue( + return new NameValue( oldNameValue.name(), oldNameValue.parentOffset(), XContentDataHelper.encodeXContentBuilder(xContentBuilder), oldNameValue.doc() ); } + } diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index b16c00033292b..9a8d0bf4ecffd 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -878,7 +878,7 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t } // _ignored is the only multi-valued meta field // TODO: can we avoid having an exception here? - if (IgnoredFieldMapper.NAME.equals(field.getName()) || field.getName().startsWith(IgnoredSourceFieldMapper.NAME)) { + if (IgnoredFieldMapper.NAME.equals(field.getName()) || field.getName().equals(IgnoredSourceFieldMapper.NAME)) { builder.field(field.getName(), field.getValues()); } else { builder.field(field.getName(), field.getValue()); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java b/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java index d4f44f31d6b36..04190c1d8d0bd 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.SetOnce; import org.elasticsearch.index.mapper.IdFieldMapper; -import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.search.lookup.FieldLookup; import org.elasticsearch.search.lookup.LeafFieldLookupProvider; @@ -22,7 +21,6 @@ import java.util.Map; import java.util.Set; import java.util.function.Supplier; -import java.util.stream.Collectors; /** * Makes pre-loaded stored fields available via a LeafSearchLookup. @@ -47,16 +45,6 @@ public void populateFieldLookup(FieldLookup fieldLookup, int doc) throws IOExcep fieldLookup.setValues(Collections.singletonList(id)); return; } - if (field.equals(IgnoredSourceFieldMapper.NAME)) { - fieldLookup.setValues( - preloadedStoredFieldValues.entrySet() - .stream() - .filter(entry -> entry.getKey().startsWith(IgnoredSourceFieldMapper.NAME)) - .flatMap(entry -> entry.getValue().stream()) - .toList() - ); - return; - } if (preloadedStoredFieldNames.get().contains(field)) { fieldLookup.setValues(preloadedStoredFieldValues.get(field)); return; @@ -73,13 +61,7 @@ void setPreloadedStoredFieldNames(Set preloadedStoredFieldNames) { } void setPreloadedStoredFieldValues(String id, Map> preloadedStoredFieldValues) { - assert preloadedStoredFieldNames.get() - .containsAll( - preloadedStoredFieldValues.keySet() - .stream() - .filter(it -> it.startsWith(IgnoredSourceFieldMapper.NAME) == false) - .collect(Collectors.toSet()) - ) + assert preloadedStoredFieldNames.get().containsAll(preloadedStoredFieldValues.keySet()) : "Provided stored field that was not expected to be preloaded? " + preloadedStoredFieldValues.keySet() + " - " diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java index 37c6f4f5658a3..e460b5a460194 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoaderTests.java @@ -23,15 +23,10 @@ import java.io.IOException; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.function.Consumer; import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.not; /** * Test that the {@link IgnoredSourceFieldLoader} loads the correct stored values. @@ -44,7 +39,7 @@ public void testSupports() { false, false, Set.of(), - new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) ) ) ); @@ -55,7 +50,7 @@ public void testSupports() { false, false, Set.of(), - new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) ) ) ); @@ -66,7 +61,7 @@ public void testSupports() { true, false, Set.of(), - new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) + new IgnoredFieldsSpec(Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) ) ) ); @@ -74,60 +69,67 @@ public void testSupports() { assertFalse(IgnoredSourceFieldLoader.supports(StoredFieldsSpec.NO_REQUIREMENTS)); } + private IgnoredSourceFieldMapper.NameValue[] nameValue(String name, String... values) { + var nameValues = new IgnoredSourceFieldMapper.NameValue[values.length]; + for (int i = 0; i < values.length; i++) { + nameValues[i] = new IgnoredSourceFieldMapper.NameValue(name, 0, new BytesRef(values[i]), null); + } + return nameValues; + } + public void testLoadSingle() throws IOException { - // Note: normally the stored value is encoded in the ignored source format - // (see IgnoredSourceFieldMapper#encodeMultipleValuesForField), but these tests are only verifying the loader, not the encoding. - BytesRef value = new BytesRef("lorem ipsum"); + var fooValue = nameValue("foo", "lorem ipsum"); Document doc = new Document(); - doc.add(new StoredField("_ignored_source.foo", value)); - testLoader(doc, Set.of("foo"), storedFields -> { - assertThat(storedFields, hasEntry(equalTo("_ignored_source.foo"), containsInAnyOrder(value))); + doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue)))); + testLoader(doc, Set.of("foo"), ignoredSourceEntries -> { + assertThat(ignoredSourceEntries, containsInAnyOrder(containsInAnyOrder(fooValue))); }); } public void testLoadMultiple() throws IOException { - BytesRef fooValue = new BytesRef("lorem ipsum"); - BytesRef barValue = new BytesRef("dolor sit amet"); + var fooValue = nameValue("foo", "lorem ipsum"); + var barValue = nameValue("bar", "dolor sit amet"); Document doc = new Document(); - doc.add(new StoredField("_ignored_source.foo", fooValue)); - doc.add(new StoredField("_ignored_source.bar", barValue)); - testLoader(doc, Set.of("foo", "bar"), storedFields -> { - assertThat(storedFields, hasEntry(equalTo("_ignored_source.foo"), containsInAnyOrder(fooValue))); - assertThat(storedFields, hasEntry(equalTo("_ignored_source.bar"), containsInAnyOrder(barValue))); + doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue)))); + doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(barValue)))); + testLoader(doc, Set.of("foo", "bar"), ignoredSourceEntries -> { + assertThat(ignoredSourceEntries, containsInAnyOrder(containsInAnyOrder(fooValue), containsInAnyOrder(barValue))); }); } public void testLoadSubset() throws IOException { - BytesRef fooValue = new BytesRef("lorem ipsum"); - BytesRef barValue = new BytesRef("dolor sit amet"); + var fooValue = nameValue("foo", "lorem ipsum"); + var barValue = nameValue("bar", "dolor sit amet"); Document doc = new Document(); - doc.add(new StoredField("_ignored_source.foo", fooValue)); - doc.add(new StoredField("_ignored_source.bar", barValue)); + doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue)))); + doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(barValue)))); - testLoader(doc, Set.of("foo"), storedFields -> { - assertThat(storedFields, hasEntry(equalTo("_ignored_source.foo"), containsInAnyOrder(fooValue))); - assertThat(storedFields, not(hasKey("_ignored_source.bar"))); + testLoader(doc, Set.of("foo"), ignoredSourceEntries -> { + assertThat(ignoredSourceEntries, containsInAnyOrder(containsInAnyOrder(fooValue))); }); } public void testLoadFromParent() throws IOException { - BytesRef fooValue = new BytesRef("lorem ipsum"); + var fooValue = new IgnoredSourceFieldMapper.NameValue("parent", 7, new BytesRef("lorem ipsum"), null); Document doc = new Document(); - doc.add(new StoredField("_ignored_source.parent", fooValue)); - testLoader(doc, Set.of("parent.foo"), storedFields -> { - assertThat(storedFields, hasEntry(equalTo("_ignored_source.parent"), containsInAnyOrder(fooValue))); + doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue)))); + testLoader(doc, Set.of("parent.foo"), ignoredSourceEntries -> { + assertThat(ignoredSourceEntries, containsInAnyOrder(containsInAnyOrder(fooValue))); }); } - private void testLoader(Document doc, Set fieldsToLoad, Consumer>> storedFieldsTest) - throws IOException { + private void testLoader( + Document doc, + Set fieldsToLoad, + Consumer>> ignoredSourceTest + ) throws IOException { try (Directory dir = newDirectory(); IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { StoredFieldsSpec spec = new StoredFieldsSpec( false, false, Set.of(), - new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE) + new IgnoredFieldsSpec(fieldsToLoad, IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE) ); assertTrue(IgnoredSourceFieldLoader.supports(spec)); iw.addDocument(doc); @@ -135,7 +137,10 @@ private void testLoader(Document doc, Set fieldsToLoad, Consumer>) (Object) leafLoader.storedFields() + .get("_ignored_source"); + ignoredSourceTest.accept(ignoredSourceEntries); } } } diff --git a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java index 1a3b62ed74be2..0255506816799 100644 --- a/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoaderTests.java @@ -53,8 +53,8 @@ private StoredFieldsSpec fieldsSpec( public void testEmpty() throws IOException { testStoredFieldLoader( - doc("foo", "lorem ipsum", "_ignored_source.bar", "dolor sit amet"), - fieldsSpec(Set.of(), Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + doc("foo", "lorem ipsum", "_ignored_source", "dolor sit amet"), + fieldsSpec(Set.of(), Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), storedFields -> { assertThat(storedFields, anEmptyMap()); } @@ -62,11 +62,14 @@ public void testEmpty() throws IOException { } public void testSingleIgnoredSourceNewFormat() throws IOException { + var fooValue = new IgnoredSourceFieldMapper.NameValue("foo", 0, new BytesRef("lorem ipsum"), null); + var doc = new Document(); + doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue)))); testIgnoredSourceLoader( - doc("_ignored_source.foo", "lorem ipsum"), - fieldsSpec(Set.of(), Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), - storedFields -> { - assertThat(storedFields, hasEntry(equalTo("_ignored_source.foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); + doc, + fieldsSpec(Set.of(), Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + ignoredFields -> { + assertThat(ignoredFields, containsInAnyOrder(containsInAnyOrder(fooValue))); } ); } @@ -74,7 +77,7 @@ public void testSingleIgnoredSourceNewFormat() throws IOException { public void testSingleIgnoredSourceOldFormat() throws IOException { testStoredFieldLoader( doc("_ignored_source", "lorem ipsum"), - fieldsSpec(Set.of(), Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.SINGLE_IGNORED_SOURCE), + fieldsSpec(Set.of(), Set.of("foo"), IgnoredSourceFieldMapper.IgnoredSourceFormat.LEGACY_SINGLE_IGNORED_SOURCE), storedFields -> { assertThat(storedFields, hasEntry(equalTo("_ignored_source"), containsInAnyOrder(new BytesRef("lorem ipsum")))); } @@ -82,12 +85,16 @@ public void testSingleIgnoredSourceOldFormat() throws IOException { } public void testMultiValueIgnoredSourceNewFormat() throws IOException { + var fooValue = new IgnoredSourceFieldMapper.NameValue("foo", 0, new BytesRef("lorem ipsum"), null); + var barValue = new IgnoredSourceFieldMapper.NameValue("bar", 0, new BytesRef("dolor sit amet"), null); + Document doc = new Document(); + doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(fooValue)))); + doc.add(new StoredField("_ignored_source", IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encode(List.of(barValue)))); testIgnoredSourceLoader( - doc("_ignored_source.foo", "lorem ipsum", "_ignored_source.bar", "dolor sit amet"), - fieldsSpec(Set.of(), Set.of("foo", "bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), - storedFields -> { - assertThat(storedFields, hasEntry(equalTo("_ignored_source.foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); - assertThat(storedFields, hasEntry(equalTo("_ignored_source.bar"), containsInAnyOrder(new BytesRef("dolor sit amet")))); + doc, + fieldsSpec(Set.of(), Set.of("foo", "bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), + ignoredFields -> { + assertThat(ignoredFields, containsInAnyOrder(containsInAnyOrder(fooValue), containsInAnyOrder(barValue))); } ); } @@ -95,7 +102,7 @@ public void testMultiValueIgnoredSourceNewFormat() throws IOException { public void testMultiValueIgnoredSourceOldFormat() throws IOException { testStoredFieldLoader( doc("_ignored_source", "lorem ipsum", "_ignored_source", "dolor sit amet"), - fieldsSpec(Set.of(), Set.of("foo", "bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.SINGLE_IGNORED_SOURCE), + fieldsSpec(Set.of(), Set.of("foo", "bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.LEGACY_SINGLE_IGNORED_SOURCE), storedFields -> { assertThat( storedFields, @@ -108,7 +115,7 @@ public void testMultiValueIgnoredSourceOldFormat() throws IOException { public void testSingleStoredField() throws IOException { testStoredFieldLoader( doc("foo", "lorem ipsum"), - fieldsSpec(Set.of("foo"), Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + fieldsSpec(Set.of("foo"), Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), storedFields -> { assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); } @@ -118,7 +125,7 @@ public void testSingleStoredField() throws IOException { public void testMultiValueStoredField() throws IOException { testStoredFieldLoader( doc("foo", "lorem ipsum", "bar", "dolor sit amet"), - fieldsSpec(Set.of("foo", "bar"), Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + fieldsSpec(Set.of("foo", "bar"), Set.of(), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), storedFields -> { assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); assertThat(storedFields, hasEntry(equalTo("bar"), containsInAnyOrder(new BytesRef("dolor sit amet")))); @@ -128,11 +135,11 @@ public void testMultiValueStoredField() throws IOException { public void testMixedStoredAndIgnoredFieldsNewFormat() throws IOException { testStoredFieldLoader( - doc("foo", "lorem ipsum", "_ignored_source.bar", "dolor sit amet"), - fieldsSpec(Set.of("foo"), Set.of("bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + doc("foo", "lorem ipsum", "_ignored_source", "dolor sit amet"), + fieldsSpec(Set.of("foo"), Set.of("bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), storedFields -> { assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); - assertThat(storedFields, hasEntry(equalTo("_ignored_source.bar"), containsInAnyOrder(new BytesRef("dolor sit amet")))); + assertThat(storedFields, hasEntry(equalTo("_ignored_source"), containsInAnyOrder(new BytesRef("dolor sit amet")))); } ); } @@ -140,7 +147,7 @@ public void testMixedStoredAndIgnoredFieldsNewFormat() throws IOException { public void testMixedStoredAndIgnoredFieldsOldFormat() throws IOException { testStoredFieldLoader( doc("foo", "lorem ipsum", "_ignored_source", "dolor sit amet"), - fieldsSpec(Set.of("foo"), Set.of("bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.SINGLE_IGNORED_SOURCE), + fieldsSpec(Set.of("foo"), Set.of("bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.LEGACY_SINGLE_IGNORED_SOURCE), storedFields -> { assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); assertThat(storedFields, hasEntry(equalTo("_ignored_source"), containsInAnyOrder(new BytesRef("dolor sit amet")))); @@ -150,11 +157,11 @@ public void testMixedStoredAndIgnoredFieldsOldFormat() throws IOException { public void testMixedStoredAndIgnoredFieldsLoadParent() throws IOException { testStoredFieldLoader( - doc("foo", "lorem ipsum", "_ignored_source.parent", "dolor sit amet"), - fieldsSpec(Set.of("foo"), Set.of("parent.bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE), + doc("foo", "lorem ipsum", "_ignored_source", "dolor sit amet"), + fieldsSpec(Set.of("foo"), Set.of("parent.bar"), IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE), storedFields -> { assertThat(storedFields, hasEntry(equalTo("foo"), containsInAnyOrder(new BytesRef("lorem ipsum")))); - assertThat(storedFields, hasEntry(equalTo("_ignored_source.parent"), containsInAnyOrder(new BytesRef("dolor sit amet")))); + assertThat(storedFields, hasEntry(equalTo("_ignored_source"), containsInAnyOrder(new BytesRef("dolor sit amet")))); } ); } @@ -164,9 +171,17 @@ private void testStoredFieldLoader(Document doc, StoredFieldsSpec spec, Consumer testLoader(doc, spec, StoredFieldLoader.class, storedFieldsTest); } - private void testIgnoredSourceLoader(Document doc, StoredFieldsSpec spec, Consumer>> storedFieldsTest) - throws IOException { - testLoader(doc, spec, IgnoredSourceFieldLoader.class, storedFieldsTest); + private void testIgnoredSourceLoader( + Document doc, + StoredFieldsSpec spec, + Consumer>> ignoredFieldEntriesTest + ) throws IOException { + testLoader(doc, spec, IgnoredSourceFieldLoader.class, storedFields -> { + @SuppressWarnings("unchecked") + List> ignoredFieldEntries = (List< + List>) (Object) storedFields.get("_ignored_source"); + ignoredFieldEntriesTest.accept(ignoredFieldEntries); + }); } private void testLoader( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java index 222d8e92b269f..1ba5f423d4b03 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -52,7 +52,7 @@ public void testDisableIgnoredSourceRead() throws IOException { var doc = mapperService.documentMapper().parse(source(inputDocument)); // Field was written. - assertTrue(doc.docs().get(0).getFields().stream().anyMatch(it -> it.name().startsWith(IgnoredSourceFieldMapper.NAME))); + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); // Values are not loaded. @@ -64,7 +64,7 @@ public void testDisableIgnoredSourceRead() throws IOException { doc = mapperService.documentMapper().parse(source(inputDocument)); // Field was written. - assertTrue(doc.docs().get(0).getFields().stream().anyMatch(it -> it.name().startsWith(IgnoredSourceFieldMapper.NAME))); + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); // Values are loaded. @@ -104,7 +104,7 @@ public void testDisableIgnoredSourceWrite() throws IOException { var doc = mapperService.documentMapper().parse(source(inputDocument)); // Field is not written. - assertTrue(doc.docs().get(0).getFields().stream().noneMatch(it -> it.name().startsWith(IgnoredSourceFieldMapper.NAME))); + assertNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); // Values are not loaded. @@ -116,7 +116,7 @@ public void testDisableIgnoredSourceWrite() throws IOException { doc = mapperService.documentMapper().parse(source(inputDocument)); // Field was written. - assertTrue(doc.docs().get(0).getFields().stream().anyMatch(it -> it.name().startsWith(IgnoredSourceFieldMapper.NAME))); + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); // Values are loaded. diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 3484852a481e6..8351bdb3bfbd8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -10,7 +10,6 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; @@ -343,12 +342,11 @@ public void testEncodeFieldToMap() throws IOException { String value = randomAlphaOfLength(5); ParsedDocument parsedDocument = getParsedDocumentWithFieldLimit(b -> b.field("my_value", value)); IgnoredSourceFieldMapper.MappedNameValue mappedNameValue; - if (IgnoredSourceFieldMapper.IGNORED_SOURCE_FIELDS_PER_ENTRY_FF.isEnabled()) { - byte[] bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.ignoredFieldName("my_value")).binaryValue().bytes; - mappedNameValue = IgnoredSourceFieldMapper.decodeAsMapMultipleFieldValues(bytes).getFirst(); + var bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue(); + if (IgnoredSourceFieldMapper.COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled()) { + mappedNameValue = IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.decodeAsMap(bytes).getFirst(); } else { - byte[] bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue().bytes; - mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(bytes); + mappedNameValue = IgnoredSourceFieldMapper.LegacyIgnoredSourceEncoding.decodeAsMap(bytes); } assertEquals("my_value", mappedNameValue.nameValue().name()); assertEquals(value, mappedNameValue.map().get("my_value")); @@ -360,22 +358,19 @@ public void testEncodeObjectToMapAndDecode() throws IOException { ParsedDocument parsedDocument = getParsedDocumentWithFieldLimit( b -> { b.startObject("my_object").field("my_value", value).endObject(); } ); - byte[] bytes; + var bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue(); IgnoredSourceFieldMapper.MappedNameValue mappedNameValue; - if (IgnoredSourceFieldMapper.IGNORED_SOURCE_FIELDS_PER_ENTRY_FF.isEnabled()) { - var byteRef = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.ignoredFieldName("my_object")).binaryValue(); - bytes = ArrayUtil.copyOfSubArray(byteRef.bytes, byteRef.offset, byteRef.length); - mappedNameValue = IgnoredSourceFieldMapper.decodeAsMapMultipleFieldValues(bytes).getFirst(); + if (IgnoredSourceFieldMapper.COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled()) { + mappedNameValue = IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.decodeAsMap(bytes).getFirst(); } else { - bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue().bytes; - mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(bytes); + mappedNameValue = IgnoredSourceFieldMapper.LegacyIgnoredSourceEncoding.decodeAsMap(bytes); } assertEquals("my_object", mappedNameValue.nameValue().name()); assertEquals(value, ((Map) mappedNameValue.map().get("my_object")).get("my_value")); - if (IgnoredSourceFieldMapper.IGNORED_SOURCE_FIELDS_PER_ENTRY_FF.isEnabled()) { - assertArrayEquals(bytes, IgnoredSourceFieldMapper.encodeFromMapMultipleFieldValues(List.of(mappedNameValue))); + if (IgnoredSourceFieldMapper.COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled()) { + assertEquals(bytes, IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encodeFromMap(List.of(mappedNameValue))); } else { - assertArrayEquals(bytes, IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue)); + assertEquals(bytes, IgnoredSourceFieldMapper.LegacyIgnoredSourceEncoding.encodeFromMap(mappedNameValue)); } } @@ -386,22 +381,19 @@ public void testEncodeArrayToMapAndDecode() throws IOException { b.startObject().field("int_value", 20).endObject(); b.endArray(); }); - byte[] bytes; + var bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue(); IgnoredSourceFieldMapper.MappedNameValue mappedNameValue; - if (IgnoredSourceFieldMapper.IGNORED_SOURCE_FIELDS_PER_ENTRY_FF.isEnabled()) { - var byteRef = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.ignoredFieldName("my_array")).binaryValue(); - bytes = ArrayUtil.copyOfSubArray(byteRef.bytes, byteRef.offset, byteRef.length); - mappedNameValue = IgnoredSourceFieldMapper.decodeAsMapMultipleFieldValues(bytes).getFirst(); + if (IgnoredSourceFieldMapper.COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled()) { + mappedNameValue = IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.decodeAsMap(bytes).getFirst(); } else { - bytes = parsedDocument.rootDoc().getField(IgnoredSourceFieldMapper.NAME).binaryValue().bytes; - mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(bytes); + mappedNameValue = IgnoredSourceFieldMapper.LegacyIgnoredSourceEncoding.decodeAsMap(bytes); } assertEquals("my_array", mappedNameValue.nameValue().name()); assertThat((List) mappedNameValue.map().get("my_array"), Matchers.contains(Map.of("int_value", 10), Map.of("int_value", 20))); - if (IgnoredSourceFieldMapper.IGNORED_SOURCE_FIELDS_PER_ENTRY_FF.isEnabled()) { - assertArrayEquals(bytes, IgnoredSourceFieldMapper.encodeFromMapMultipleFieldValues(List.of(mappedNameValue))); + if (IgnoredSourceFieldMapper.COALESCE_IGNORED_SOURCE_ENTRIES.isEnabled()) { + assertEquals(bytes, IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.encodeFromMap(List.of(mappedNameValue))); } else { - assertArrayEquals(bytes, IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue)); + assertEquals(bytes, IgnoredSourceFieldMapper.LegacyIgnoredSourceEncoding.encodeFromMap(mappedNameValue)); } } @@ -2489,16 +2481,10 @@ public void testSingleDeepIgnoredField() throws IOException { assertEquals("{\"top\":{\"level1\":{\"level2\":{\"n\":25}}}}", syntheticSource); } - private static String getIgnoredSourceFieldMask() { - return IgnoredSourceFieldMapper.IGNORED_SOURCE_FIELDS_PER_ENTRY_FF.isEnabled() - ? IgnoredSourceFieldMapper.ignoredFieldName("*") - : IgnoredSourceFieldMapper.NAME; - } - private final Set roundtripMaskedFields = Set.of( SourceFieldMapper.RECOVERY_SOURCE_NAME, SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME, - getIgnoredSourceFieldMask(), + IgnoredSourceFieldMapper.NAME, "*.offsets" ); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java index 1ba51a91e9203..d1ab3c0907562 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java @@ -228,14 +228,7 @@ public void testSynthesizeRandomArrayInNestedContext() throws Exception { var reader = searcher.getDirectoryReader(); var document = reader.storedFields().document(0); Set storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList()); - assertThat( - storedFieldNames, - contains( - IgnoredSourceFieldMapper.IGNORED_SOURCE_FIELDS_PER_ENTRY_FF.isEnabled() - ? IgnoredSourceFieldMapper.ignoredFieldName("parent.field") - : IgnoredSourceFieldMapper.NAME - ) - ); + assertThat(storedFieldNames, contains(IgnoredSourceFieldMapper.NAME)); assertThat(FieldInfos.getMergedFieldInfos(reader).fieldInfo("parent.field.offsets"), nullValue()); } } @@ -375,15 +368,7 @@ protected void verifySyntheticObjectArray(List> documents) throws var document = reader.storedFields().document(i); // Verify that there is ignored source because of leaf array being wrapped by object array: List storedFieldNames = document.getFields().stream().map(IndexableField::name).toList(); - assertThat( - storedFieldNames, - contains( - "_id", - IgnoredSourceFieldMapper.IGNORED_SOURCE_FIELDS_PER_ENTRY_FF.isEnabled() - ? IgnoredSourceFieldMapper.ignoredFieldName("object") - : IgnoredSourceFieldMapper.NAME - ) - ); + assertThat(storedFieldNames, contains("_id", IgnoredSourceFieldMapper.NAME)); // Verify that there is no offset field: LeafReader leafReader = reader.leaves().get(0).reader(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java index adf12490a7d90..bddb274b21226 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java @@ -29,6 +29,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FilterIterator; @@ -71,33 +72,43 @@ public final class FieldSubsetReader extends SequentialStoredFieldsLeafReader { * @param in reader to filter * @param filter fields to filter. */ - public static DirectoryReader wrap(DirectoryReader in, CharacterRunAutomaton filter) throws IOException { - return new FieldSubsetDirectoryReader(in, filter); + public static DirectoryReader wrap( + DirectoryReader in, + CharacterRunAutomaton filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) throws IOException { + return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat); } // wraps subreaders with fieldsubsetreaders. static class FieldSubsetDirectoryReader extends FilterDirectoryReader { private final CharacterRunAutomaton filter; + private final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; - FieldSubsetDirectoryReader(DirectoryReader in, final CharacterRunAutomaton filter) throws IOException { + FieldSubsetDirectoryReader( + DirectoryReader in, + final CharacterRunAutomaton filter, + final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + ) throws IOException { super(in, new FilterDirectoryReader.SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { try { - return new FieldSubsetReader(reader, filter); + return new FieldSubsetReader(reader, filter, ignoredSourceFormat); } catch (IOException e) { throw new UncheckedIOException(e); } } }); this.filter = filter; + this.ignoredSourceFormat = ignoredSourceFormat; verifyNoOtherFieldSubsetDirectoryReaderIsWrapped(in); } @Override protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { - return new FieldSubsetDirectoryReader(in, filter); + return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat); } /** Return the automaton that is used to filter fields. */ @@ -127,13 +138,15 @@ public CacheHelper getReaderCacheHelper() { private final FieldInfos fieldInfos; /** An automaton that only accepts authorized fields. */ private final CharacterRunAutomaton filter; + private final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; /** {@link Terms} cache with filtered stats for the {@link FieldNamesFieldMapper} field. */ private volatile Optional fieldNamesFilterTerms; /** * Wrap a single segment, exposing a subset of its fields. */ - FieldSubsetReader(LeafReader in, CharacterRunAutomaton filter) throws IOException { + FieldSubsetReader(LeafReader in, CharacterRunAutomaton filter, IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat) + throws IOException { super(in); ArrayList filteredInfos = new ArrayList<>(); for (FieldInfo fi : in.getFieldInfos()) { @@ -143,6 +156,7 @@ public CacheHelper getReaderCacheHelper() { } fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()])); this.filter = filter; + this.ignoredSourceFormat = ignoredSourceFormat; } /** returns true if this field is allowed. */ @@ -178,7 +192,7 @@ public StoredFields storedFields() throws IOException { return new StoredFields() { @Override public void document(int docID, StoredFieldVisitor visitor) throws IOException { - storedFields.document(docID, new FieldSubsetStoredFieldVisitor(visitor)); + storedFields.document(docID, new FieldSubsetStoredFieldVisitor(visitor, ignoredSourceFormat)); } }; } @@ -255,7 +269,7 @@ private static int step(CharacterRunAutomaton automaton, String key, int state) @Override protected StoredFieldsReader doGetSequentialStoredFieldsReader(StoredFieldsReader reader) { - return new FieldSubsetStoredFieldsReader(reader); + return new FieldSubsetStoredFieldsReader(reader, ignoredSourceFormat); } @Override @@ -334,24 +348,26 @@ public CacheHelper getReaderCacheHelper() { */ class FieldSubsetStoredFieldsReader extends StoredFieldsReader { final StoredFieldsReader reader; + final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; - FieldSubsetStoredFieldsReader(StoredFieldsReader reader) { + FieldSubsetStoredFieldsReader(StoredFieldsReader reader, IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat) { this.reader = reader; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override public void document(int docID, StoredFieldVisitor visitor) throws IOException { - reader.document(docID, new FieldSubsetStoredFieldVisitor(visitor)); + reader.document(docID, new FieldSubsetStoredFieldVisitor(visitor, ignoredSourceFormat)); } @Override public StoredFieldsReader clone() { - return new FieldSubsetStoredFieldsReader(reader.clone()); + return new FieldSubsetStoredFieldsReader(reader.clone(), ignoredSourceFormat); } @Override public StoredFieldsReader getMergeInstance() { - return new FieldSubsetStoredFieldsReader(reader.getMergeInstance()); + return new FieldSubsetStoredFieldsReader(reader.getMergeInstance(), ignoredSourceFormat); } @Override @@ -370,9 +386,11 @@ public void close() throws IOException { */ class FieldSubsetStoredFieldVisitor extends StoredFieldVisitor { final StoredFieldVisitor visitor; + final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; - FieldSubsetStoredFieldVisitor(StoredFieldVisitor visitor) { + FieldSubsetStoredFieldVisitor(StoredFieldVisitor visitor, IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat) { this.visitor = visitor; + this.ignoredSourceFormat = ignoredSourceFormat; } @Override @@ -385,47 +403,12 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { XContentBuilder xContentBuilder = XContentBuilder.builder(result.v1().xContent()).map(transformedSource); visitor.binaryField(fieldInfo, BytesReference.toBytes(BytesReference.bytes(xContentBuilder))); } else if (IgnoredSourceFieldMapper.NAME.equals(fieldInfo.name)) { - // for _ignored_source, parse, filter out the field and its contents, and serialize back downstream - IgnoredSourceFieldMapper.MappedNameValue mappedNameValue = IgnoredSourceFieldMapper.decodeAsMap(value); - Map transformedField = filter(mappedNameValue.map(), filter, 0); - if (transformedField.isEmpty() == false) { - // The unfiltered map contains at least one element, the field name with its value. If the field contains - // an object or an array, the value of the first element is a map or a list, respectively. Otherwise, - // it's a single leaf value, e.g. a string or a number. - var topValue = mappedNameValue.map().values().iterator().next(); - if (topValue instanceof Map || topValue instanceof List) { - // The field contains an object or an array, reconstruct it from the transformed map in case - // any subfield has been filtered out. - visitor.binaryField(fieldInfo, IgnoredSourceFieldMapper.encodeFromMap(mappedNameValue.withMap(transformedField))); - } else { - // The field contains a leaf value, and it hasn't been filtered out. It is safe to propagate the original value. - visitor.binaryField(fieldInfo, value); - } - } - } else if (fieldInfo.name.startsWith(IgnoredSourceFieldMapper.NAME)) { - List mappedNameValues = IgnoredSourceFieldMapper.decodeAsMapMultipleFieldValues( - value - ); - List filteredNameValues = new ArrayList<>(mappedNameValues.size()); - boolean didFilter = false; - for (var mappedNameValue : mappedNameValues) { - Map transformedField = filter(mappedNameValue.map(), filter, 0); - if (transformedField.isEmpty()) { - didFilter = true; - continue; - } - var topValue = mappedNameValue.map().values().iterator().next(); - if (topValue instanceof Map || topValue instanceof List) { - didFilter = true; - } - filteredNameValues.add(mappedNameValue.withMap(transformedField)); - } - if (didFilter) { - if (filteredNameValues.isEmpty() == false) { - visitor.binaryField(fieldInfo, IgnoredSourceFieldMapper.encodeFromMapMultipleFieldValues(filteredNameValues)); - } - } else { - visitor.binaryField(fieldInfo, value); + assert ignoredSourceFormat != IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE; + BytesRef valueRef = new BytesRef(value); + BytesRef filtered = ignoredSourceFormat.filterValue(valueRef, v -> filter(v, filter, 0)); + if (filtered != null) { + byte[] filteredBytes = ArrayUtil.copyOfSubArray(filtered.bytes, filtered.offset, filtered.offset + filtered.length); + visitor.binaryField(fieldInfo, filteredBytes); } } else { visitor.binaryField(fieldInfo, value); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java index 71ba14b02667a..0645f888096a8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java @@ -96,7 +96,9 @@ public DirectoryReader apply(final DirectoryReader reader) { } } - return permissions.getFieldPermissions().filter(wrappedReader); + var indexVersionCreated = searchExecutionContextProvider.apply(shardId).indexVersionCreated(); + + return permissions.getFieldPermissions().filter(wrappedReader, indexVersionCreated); } catch (IOException e) { logger.error("Unable to apply field level security"); throw ExceptionsHelper.convertToElastic(e); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java index c46f1350776b1..3369774ad5c5a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java @@ -18,6 +18,8 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.lucene.util.automaton.MinimizationOperations; import org.elasticsearch.plugins.FieldPredicate; import org.elasticsearch.xpack.core.security.authz.accesscontrol.FieldSubsetReader; @@ -243,11 +245,11 @@ public boolean hasFieldLevelSecurity() { } /** Return a wrapped reader that only exposes allowed fields. */ - public DirectoryReader filter(DirectoryReader reader) throws IOException { + public DirectoryReader filter(DirectoryReader reader, IndexVersion indexVersionCreated) throws IOException { if (hasFieldLevelSecurity() == false) { return reader; } - return FieldSubsetReader.wrap(reader, permittedFieldsAutomaton); + return FieldSubsetReader.wrap(reader, permittedFieldsAutomaton, IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersionCreated)); } Automaton getIncludeAutomaton() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index 76bb146b55fcc..88a6be2c215a2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -111,7 +111,11 @@ public void testIndexed() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -142,7 +146,11 @@ public void testPoints() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -201,7 +209,11 @@ public void testKnnVectors() throws Exception { doc.add(new KnnFloatVectorField("fieldB", new float[] { 3.0f, 2.0f, 1.0f })); iw.addDocument(doc); - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); LeafReader leafReader = ir.leaves().get(0).reader(); // Check that fieldA behaves as normal @@ -236,7 +248,11 @@ public void testKnnByteVectors() throws Exception { doc.add(new KnnByteVectorField("fieldB", new byte[] { 3, 2, 1 })); iw.addDocument(doc); - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); LeafReader leafReader = ir.leaves().get(0).reader(); // Check that fieldA behaves as normal @@ -276,7 +292,11 @@ public void testStoredFieldsString() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field { @@ -303,7 +323,11 @@ public void testStoredFieldsBinary() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field { @@ -330,7 +354,11 @@ public void testStoredFieldsInt() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field { @@ -357,7 +385,11 @@ public void testStoredFieldsLong() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field { @@ -384,7 +416,11 @@ public void testStoredFieldsFloat() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field { @@ -411,7 +447,11 @@ public void testStoredFieldsDouble() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field { @@ -440,7 +480,11 @@ public void testVectors() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field Fields vectors = ir.termVectors().get(0); @@ -469,7 +513,11 @@ public void testNorms() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -495,7 +543,11 @@ public void testNumericDocValues() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -524,7 +576,11 @@ public void testBinaryDocValues() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -553,7 +609,11 @@ public void testSortedDocValues() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -582,7 +642,11 @@ public void testSortedSetDocValues() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -612,7 +676,11 @@ public void testSortedNumericDocValues() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -642,7 +710,11 @@ public void testFieldInfos() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -673,7 +745,11 @@ public void testSourceFilteringIntegration() throws Exception { // open reader Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", SourceFieldMapper.NAME)); - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton)); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field { @@ -712,18 +788,13 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { iw.addDocuments(doc.docs()); iw.close(); - String ignoredSourceFieldPattern = IgnoredSourceFieldMapper.ignoredSourceFormat( - indexVersion - ) == IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE - ? IgnoredSourceFieldMapper.ignoredFieldName("*") - : IgnoredSourceFieldMapper.NAME; - { - Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", ignoredSourceFieldPattern)); + Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", IgnoredSourceFieldMapper.NAME)); try ( DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), - new CharacterRunAutomaton(automaton) + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -740,7 +811,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { try ( DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), - new CharacterRunAutomaton(automaton) + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -750,11 +822,12 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { } { - Automaton automaton = Automatons.patterns(Arrays.asList("obj.fieldC", ignoredSourceFieldPattern)); + Automaton automaton = Automatons.patterns(Arrays.asList("obj.fieldC", IgnoredSourceFieldMapper.NAME)); try ( DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), - new CharacterRunAutomaton(automaton) + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -772,7 +845,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { try ( DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), - new CharacterRunAutomaton(automaton) + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -782,11 +856,12 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { } { - Automaton automaton = Automatons.patterns(Arrays.asList("arr.fieldD", ignoredSourceFieldPattern)); + Automaton automaton = Automatons.patterns(Arrays.asList("arr.fieldD", IgnoredSourceFieldMapper.NAME)); try ( DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), - new CharacterRunAutomaton(automaton) + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -804,7 +879,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { try ( DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), - new CharacterRunAutomaton(automaton) + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -977,7 +1053,11 @@ public void testFieldNames() throws Exception { Set fields = new HashSet<>(); fields.add("fieldA"); Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", FieldNamesFieldMapper.NAME)); - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton)); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -1035,7 +1115,11 @@ public void testFieldNamesThreeFields() throws Exception { // open reader Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", "fieldC", FieldNamesFieldMapper.NAME)); - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton)); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only two fields LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -1080,7 +1164,11 @@ public void testFieldNamesMissing() throws Exception { // open reader Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", "fieldC", FieldNamesFieldMapper.NAME)); - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton)); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -1114,7 +1202,11 @@ public void testFieldNamesOldIndex() throws Exception { // open reader Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", SourceFieldMapper.NAME)); - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton)); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see only one field LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -1142,7 +1234,11 @@ public void testCoreCacheKey() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("id"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("id")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); assertEquals(2, ir.numDocs()); assertEquals(1, ir.leaves().size()); @@ -1176,7 +1272,11 @@ public void testFilterAwayAllVectors() throws Exception { iw.addDocument(doc); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldB"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldB")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // sees no fields assertNull(ir.termVectors().get(0)); @@ -1195,7 +1295,11 @@ public void testEmpty() throws Exception { iw.addDocument(new Document()); // open reader - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA"))); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); // see no fields LeafReader segmentReader = ir.leaves().get(0).reader(); @@ -1226,11 +1330,16 @@ public void testWrapTwice() throws Exception { final DirectoryReader directoryReader = FieldSubsetReader.wrap( DirectoryReader.open(dir), - new CharacterRunAutomaton(Automata.makeString("fieldA")) + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE ); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> FieldSubsetReader.wrap(directoryReader, new CharacterRunAutomaton(Automata.makeString("fieldA"))) + () -> FieldSubsetReader.wrap( + directoryReader, + new CharacterRunAutomaton(Automata.makeString("fieldA")), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ) ); assertThat( e.getMessage(), @@ -1416,7 +1525,11 @@ public void testProducesStoredFieldsReader() throws Exception { // open reader Automaton automaton = Automatons.patterns(Arrays.asList("fieldA", SourceFieldMapper.NAME)); - DirectoryReader ir = FieldSubsetReader.wrap(DirectoryReader.open(iw), new CharacterRunAutomaton(automaton)); + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + new CharacterRunAutomaton(automaton), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); TestUtil.checkReader(ir); assertThat(ir.leaves().size(), greaterThanOrEqualTo(1)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperUnitTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperUnitTests.java index 5ac21a4c126ba..af8ae20ba7d16 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperUnitTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperUnitTests.java @@ -14,9 +14,11 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesModule; @@ -86,7 +88,16 @@ public void tearDown() throws Exception { } public void testDefaultMetaFields() { - securityIndexReaderWrapper = new SecurityIndexReaderWrapper(null, null, securityContext, licenseState, scriptService) { + var searchExecutionContext = mock(SearchExecutionContext.class); + when(searchExecutionContext.indexVersionCreated()).thenReturn(IndexVersion.current()); + + securityIndexReaderWrapper = new SecurityIndexReaderWrapper( + id -> searchExecutionContext, + null, + securityContext, + licenseState, + scriptService + ) { @Override protected IndicesAccessControl getIndicesAccessControl() { IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java index 4d46cba676049..d29200fdccc41 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -116,7 +117,11 @@ public void testSortedSetDVOrdinalsIndexFieldData_global() throws Exception { assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(numDocs)); assertThat(indexFieldDataCache.topLevelBuilds, equalTo(1)); - DirectoryReader ir = FieldSubsetReader.wrap(this.ir, new CharacterRunAutomaton(Automata.makeEmpty())); + DirectoryReader ir = FieldSubsetReader.wrap( + this.ir, + new CharacterRunAutomaton(Automata.makeEmpty()), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); global = sortedSetOrdinalsIndexFieldData.loadGlobal(ir); atomic = global.load(ir.leaves().get(0)); assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(0L)); @@ -129,7 +134,11 @@ public void testSortedSetDVOrdinalsIndexFieldData_segment() throws Exception { assertThat(atomic.getOrdinalsValues().getValueCount(), greaterThanOrEqualTo(1L)); } - DirectoryReader ir = FieldSubsetReader.wrap(this.ir, new CharacterRunAutomaton(Automata.makeEmpty())); + DirectoryReader ir = FieldSubsetReader.wrap( + this.ir, + new CharacterRunAutomaton(Automata.makeEmpty()), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); for (LeafReaderContext context : ir.leaves()) { LeafOrdinalsFieldData atomic = sortedSetOrdinalsIndexFieldData.load(context); assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(0L)); @@ -145,7 +154,11 @@ public void testPagedBytesIndexFieldData_global() throws Exception { assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(numDocs)); assertThat(indexFieldDataCache.topLevelBuilds, equalTo(1)); - DirectoryReader ir = FieldSubsetReader.wrap(this.ir, new CharacterRunAutomaton(Automata.makeEmpty())); + DirectoryReader ir = FieldSubsetReader.wrap( + this.ir, + new CharacterRunAutomaton(Automata.makeEmpty()), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); global = pagedBytesIndexFieldData.loadGlobal(ir); atomic = global.load(ir.leaves().get(0)); assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(0L)); @@ -160,7 +173,11 @@ public void testPagedBytesIndexFieldData_segment() throws Exception { } assertThat(indexFieldDataCache.leafLevelBuilds, equalTo(ir.leaves().size())); - DirectoryReader ir = FieldSubsetReader.wrap(this.ir, new CharacterRunAutomaton(Automata.makeEmpty())); + DirectoryReader ir = FieldSubsetReader.wrap( + this.ir, + new CharacterRunAutomaton(Automata.makeEmpty()), + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + ); for (LeafReaderContext context : ir.leaves()) { LeafOrdinalsFieldData atomic = pagedBytesIndexFieldData.load(context); assertThat(atomic.getOrdinalsValues().getValueCount(), equalTo(0L));