diff --git a/docs/changelog/137442.yaml b/docs/changelog/137442.yaml new file mode 100644 index 0000000000000..b999927dff74a --- /dev/null +++ b/docs/changelog/137442.yaml @@ -0,0 +1,5 @@ +pr: 137442 +summary: Handle ._original stored fields with fls +area: "Authorization" +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java index 8544ddd0194f3..aa7b395519802 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java @@ -26,6 +26,9 @@ * {@code _source}. */ public abstract class IgnoreMalformedStoredValues { + + public static final String IGNORE_MALFORMED_FIELD_NAME_SUFFIX = "._ignore_malformed"; + /** * Creates a stored field that stores malformed data to be used in synthetic source. * Name of the stored field is original name of the field with added conventional suffix. @@ -143,6 +146,6 @@ public void reset() { } public static String name(String fieldName) { - return fieldName + "._ignore_malformed"; + return fieldName + IGNORE_MALFORMED_FIELD_NAME_SUFFIX; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java index efd2bafd8e2c9..de03b116f0c19 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java @@ -20,6 +20,7 @@ */ public abstract class TextFamilyFieldType extends StringFieldType { + public static final String FALLBACK_FIELD_NAME_SUFFIX = "._original"; private final boolean isSyntheticSourceEnabled; private final boolean isWithinMultiField; @@ -51,7 +52,7 @@ public boolean isWithinMultiField() { * stored for whatever reason. */ public String syntheticSourceFallbackFieldName() { - return name() + "._original"; + return name() + FALLBACK_FIELD_NAME_SUFFIX; } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java index 1b25b36e1d2db..548db7e08f598 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java @@ -41,8 +41,10 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.TextFamilyFieldType; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -55,6 +57,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; /** * A {@link FilterLeafReader} that exposes only a subset @@ -69,15 +72,18 @@ public final class FieldSubsetReader extends SequentialStoredFieldsLeafReader { * Note that for convenience, the returned reader * can be used normally (e.g. passed to {@link DirectoryReader#openIfChanged(DirectoryReader)}) * and so on. - * @param in reader to filter - * @param filter fields to filter. + * + * @param in reader to filter + * @param filter fields to filter. + * @param isMapped whether a field is mapped or not. */ public static DirectoryReader wrap( DirectoryReader in, CharacterRunAutomaton filter, - IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat, + Function isMapped ) throws IOException { - return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat); + return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat, isMapped); } // wraps subreaders with fieldsubsetreaders. @@ -85,17 +91,19 @@ static class FieldSubsetDirectoryReader extends FilterDirectoryReader { private final CharacterRunAutomaton filter; private final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; + private final Function isMapped; FieldSubsetDirectoryReader( DirectoryReader in, final CharacterRunAutomaton filter, - final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat, + Function isMapped ) throws IOException { super(in, new FilterDirectoryReader.SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { try { - return new FieldSubsetReader(reader, filter, ignoredSourceFormat); + return new FieldSubsetReader(reader, filter, ignoredSourceFormat, isMapped); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -103,12 +111,13 @@ public LeafReader wrap(LeafReader reader) { }); this.filter = filter; this.ignoredSourceFormat = ignoredSourceFormat; + this.isMapped = isMapped; verifyNoOtherFieldSubsetDirectoryReaderIsWrapped(in); } @Override protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { - return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat); + return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat, isMapped); } /** Return the automaton that is used to filter fields. */ @@ -145,12 +154,25 @@ public CacheHelper getReaderCacheHelper() { /** * Wrap a single segment, exposing a subset of its fields. */ - FieldSubsetReader(LeafReader in, CharacterRunAutomaton filter, IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat) - throws IOException { + FieldSubsetReader( + LeafReader in, + CharacterRunAutomaton filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat, + Function isMapped + ) throws IOException { super(in); ArrayList filteredInfos = new ArrayList<>(); for (FieldInfo fi : in.getFieldInfos()) { - if (filter.run(fi.name)) { + String name = fi.name; + if (fi.getName().endsWith(TextFamilyFieldType.FALLBACK_FIELD_NAME_SUFFIX) && isMapped.apply(fi.getName()) == false) { + name = fi.getName().substring(0, fi.getName().length() - TextFamilyFieldType.FALLBACK_FIELD_NAME_SUFFIX.length()); + } + if (fi.getName().endsWith(IgnoreMalformedStoredValues.IGNORE_MALFORMED_FIELD_NAME_SUFFIX) + && isMapped.apply(fi.getName()) == false) { + name = fi.getName() + .substring(0, fi.getName().length() - IgnoreMalformedStoredValues.IGNORE_MALFORMED_FIELD_NAME_SUFFIX.length()); + } + if (filter.run(name)) { filteredInfos.add(fi); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java index abb13702afb86..db8e71295f155 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java @@ -96,9 +96,11 @@ public DirectoryReader apply(final DirectoryReader reader) { } } - var indexVersionCreated = searchExecutionContextProvider.apply(shardId).indexVersionCreated(); + var searchContext = searchExecutionContextProvider.apply(shardId); + var indexVersionCreated = searchContext.indexVersionCreated(); + Function isMapped = searchContext::isFieldMapped; - return permissions.getFieldPermissions().filter(wrappedReader, indexVersionCreated); + return permissions.getFieldPermissions().filter(wrappedReader, indexVersionCreated, isMapped); } catch (IOException e) { logger.error("Unable to apply field level security"); throw ExceptionsHelper.convertToElastic(e); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java index 3369774ad5c5a..828485066fca4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -245,11 +246,17 @@ public boolean hasFieldLevelSecurity() { } /** Return a wrapped reader that only exposes allowed fields. */ - public DirectoryReader filter(DirectoryReader reader, IndexVersion indexVersionCreated) throws IOException { + public DirectoryReader filter(DirectoryReader reader, IndexVersion indexVersionCreated, Function isMapped) + throws IOException { if (hasFieldLevelSecurity() == false) { return reader; } - return FieldSubsetReader.wrap(reader, permittedFieldsAutomaton, IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersionCreated)); + return FieldSubsetReader.wrap( + reader, + permittedFieldsAutomaton, + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersionCreated), + isMapped + ); } Automaton getIncludeAutomaton() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index 23a2792663ec0..01d7175e9ad52 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -115,7 +115,8 @@ public void testIndexed() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -150,7 +151,8 @@ public void testPoints() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -213,7 +215,8 @@ public void testKnnVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); LeafReader leafReader = ir.leaves().get(0).reader(); @@ -264,7 +267,8 @@ public void testKnnByteVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); LeafReader leafReader = ir.leaves().get(0).reader(); @@ -320,7 +324,8 @@ public void testStoredFieldsString() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -351,7 +356,8 @@ public void testStoredFieldsBinary() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -382,7 +388,8 @@ public void testStoredFieldsInt() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -413,7 +420,8 @@ public void testStoredFieldsLong() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -444,7 +452,8 @@ public void testStoredFieldsFloat() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -475,7 +484,8 @@ public void testStoredFieldsDouble() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -508,7 +518,8 @@ public void testVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -541,7 +552,8 @@ public void testNorms() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -571,7 +583,8 @@ public void testNumericDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -604,7 +617,8 @@ public void testBinaryDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -637,7 +651,8 @@ public void testSortedDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -670,7 +685,8 @@ public void testSortedSetDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -704,7 +720,8 @@ public void testSortedNumericDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -738,7 +755,8 @@ public void testFieldInfos() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -773,7 +791,8 @@ public void testSourceFilteringIntegration() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -819,7 +838,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -837,7 +857,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -852,7 +873,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -871,7 +893,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -886,7 +909,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -905,7 +929,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -916,6 +941,82 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { } } + public void testVisibilityOriginalFieldNames() throws Exception { + try (Directory dir = newDirectory()) { + try (IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null))) { + Document doc = new Document(); + doc.add(new StoredField("a._original", new BytesRef("a"))); + doc.add(new StoredField("b._ignore_malformed", new BytesRef("b"))); + doc.add(new StoredField("c", new BytesRef("c"))); + iw.addDocument(doc); + + // Field a is mapped: + + var filter = new CharacterRunAutomaton(Automatons.patterns(List.of("a", "c"))); + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(2, fields.getFields().size()); + assertEquals(new BytesRef("a"), fields.getBinaryValue("a._original")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + // Field a is not mapped: + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(1, fields.getFields().size()); + assertNull(fields.getBinaryValue("a._original")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + // Field b is mapped: + filter = new CharacterRunAutomaton(Automatons.patterns(List.of("b", "c"))); + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(2, fields.getFields().size()); + assertEquals(new BytesRef("b"), fields.getBinaryValue("b._ignore_malformed")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + // Field b is not mapped: + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(1, fields.getFields().size()); + assertNull(fields.getBinaryValue("b._ignore_malformed")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + } + } + } + public void testSourceFiltering() { // include on top-level value Map map = new HashMap<>(); @@ -1081,7 +1182,8 @@ public void testFieldNames() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only one field @@ -1143,7 +1245,8 @@ public void testFieldNamesThreeFields() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only two fields @@ -1192,7 +1295,8 @@ public void testFieldNamesMissing() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only one field @@ -1230,7 +1334,8 @@ public void testFieldNamesOldIndex() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only one field @@ -1262,7 +1367,8 @@ public void testCoreCacheKey() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("id")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); assertEquals(2, ir.numDocs()); assertEquals(1, ir.leaves().size()); @@ -1300,7 +1406,8 @@ public void testFilterAwayAllVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldB")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // sees no fields @@ -1323,7 +1430,8 @@ public void testEmpty() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see no fields @@ -1356,14 +1464,16 @@ public void testWrapTwice() throws Exception { final DirectoryReader directoryReader = FieldSubsetReader.wrap( DirectoryReader.open(dir), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> FieldSubsetReader.wrap( directoryReader, new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ) ); assertThat( @@ -1553,7 +1663,8 @@ public void testProducesStoredFieldsReader() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); TestUtil.checkReader(ir); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java index d29200fdccc41..d1f8e5096dd23 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java @@ -120,7 +120,8 @@ public void testSortedSetDVOrdinalsIndexFieldData_global() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); global = sortedSetOrdinalsIndexFieldData.loadGlobal(ir); atomic = global.load(ir.leaves().get(0)); @@ -137,7 +138,8 @@ public void testSortedSetDVOrdinalsIndexFieldData_segment() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); for (LeafReaderContext context : ir.leaves()) { LeafOrdinalsFieldData atomic = sortedSetOrdinalsIndexFieldData.load(context); @@ -157,7 +159,8 @@ public void testPagedBytesIndexFieldData_global() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); global = pagedBytesIndexFieldData.loadGlobal(ir); atomic = global.load(ir.leaves().get(0)); @@ -176,7 +179,8 @@ public void testPagedBytesIndexFieldData_segment() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); for (LeafReaderContext context : ir.leaves()) { LeafOrdinalsFieldData atomic = pagedBytesIndexFieldData.load(context); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml index 301cb01acd2d3..c038c33f68f5a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml @@ -24,7 +24,134 @@ Filter single field: name: type: keyword secret: + type: match_only_text + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "name" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + +--- +match_only_text field type grant all except secret field: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: type: keyword + secret: + type: match_only_text + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "secret" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + +--- +match_only_text field type grant name field: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: match_only_text + secret: + type: match_only_text - do: bulk: @@ -73,6 +200,136 @@ Filter single field: - match: { hits.hits.0._source.name: A } - is_false: "hits.hits.0._source.secret" +--- +keyword field type with ignore_above: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + secret: + type: keyword + ignore_above: 3 + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "secret" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + +--- +long field type with ignore_malformed: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + secret: + type: long + ignore_malformed: true + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "secret" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + --- Filter fields in object: - do: diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index f23536acba87b..ad28b0336d855 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -70,6 +70,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.SourceValueFetcher; +import org.elasticsearch.index.mapper.TextFamilyFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromCustomBinaryBlockLoader; @@ -1023,7 +1024,7 @@ private WildcardFieldMapper( this.indexVersionCreated = builder.indexCreatedVersion; this.ignoreAboveDefault = builder.ignoreAboveDefault; this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); - this.originalName = storeIgnored ? fullPath() + "._original" : null; + this.originalName = storeIgnored ? fullPath() + TextFamilyFieldType.FALLBACK_FIELD_NAME_SUFFIX : null; } @Override