Skip to content

Commit e4f7147

Browse files
RassyanRassyan
committed
Introduce index.mapping.source.auto_exclude_types to exclude field from source by type
1 parent 18286be commit e4f7147

File tree

9 files changed

+164
-15
lines changed

9 files changed

+164
-15
lines changed

‎server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
204204
IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING,
205205
IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING,
206206
InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT,
207+
IndexSettings.INDEX_MAPPER_SOURCE_AUTO_EXCLUDE_TYPES_SETTING,
207208

208209
// validate that built-in similarities don't get redefined
209210
Setting.groupSetting("index.similarity.", (s) -> {

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -856,6 +856,14 @@ private static String getIgnoreAboveDefaultValue(final Settings settings) {
856856
Property.ServerlessPublic
857857
);
858858

859+
public static final Setting<List<String>> INDEX_MAPPER_SOURCE_AUTO_EXCLUDE_TYPES_SETTING = Setting.listSetting(
860+
"index.mapping.source.auto_exclude_types",
861+
Collections.emptyList(),
862+
String::toString,
863+
Setting.Property.Final,
864+
Setting.Property.IndexScope
865+
);
866+
859867
private final Index index;
860868
private final IndexVersion version;
861869
private final Logger logger;

‎server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,9 @@ Mapping parse(@Nullable String type, MergeReason reason, Map<String, Object> map
123123
final MappingParserContext mappingParserContext = mappingParserContextSupplier.get();
124124

125125
RootObjectMapper.Builder rootObjectMapper = RootObjectMapper.parse(type, mappingSource, mappingParserContext);
126+
if (mappingParserContext.getAutoExcludes().isEmpty() == false && mappingSource.containsKey(SourceFieldMapper.NAME) == false) {
127+
mappingSource.put(SourceFieldMapper.NAME, new HashMap<>());
128+
}
126129

127130
Map<Class<? extends MetadataFieldMapper>, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get();
128131
Map<String, Object> meta = null;

‎server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
import org.elasticsearch.index.similarity.SimilarityProvider;
2222
import org.elasticsearch.script.ScriptCompiler;
2323

24+
import java.util.ArrayList;
25+
import java.util.List;
26+
import java.util.Stack;
2427
import java.util.function.Function;
2528
import java.util.function.Supplier;
2629

@@ -42,7 +45,8 @@ public class MappingParserContext {
4245
private final IdFieldMapper idFieldMapper;
4346
private final Function<Query, BitSetProducer> bitSetProducer;
4447
private final long mappingObjectDepthLimit;
45-
private long mappingObjectDepth = 0;
48+
private final Stack<String> mappingObjectPath = new Stack<>();
49+
private final List<String> autoExcludes = new ArrayList<>();
4650

4751
public MappingParserContext(
4852
Function<String, SimilarityProvider> similarityLookupService,
@@ -142,21 +146,43 @@ public BitSetProducer bitSetProducer(Query query) {
142146
return bitSetProducer.apply(query);
143147
}
144148

145-
void incrementMappingObjectDepth() throws MapperParsingException {
146-
mappingObjectDepth++;
147-
if (mappingObjectDepth > mappingObjectDepthLimit) {
149+
void incrementMappingObjectDepth(String name) throws MapperParsingException {
150+
mappingObjectPath.push(name);
151+
if (mappingObjectPath.size() > mappingObjectDepthLimit) {
148152
throw new MapperParsingException("Limit of mapping depth [" + mappingObjectDepthLimit + "] has been exceeded");
149153
}
150154
}
151155

152156
void decrementMappingObjectDepth() throws MapperParsingException {
153-
mappingObjectDepth--;
157+
mappingObjectPath.pop();
158+
}
159+
160+
public List<String> getAutoExcludes() {
161+
return autoExcludes;
162+
}
163+
164+
public void addAutoExclude(String name) {
165+
autoExcludes.add(name);
154166
}
155167

156168
public MappingParserContext createMultiFieldContext() {
157169
return new MultiFieldParserContext(this);
158170
}
159171

172+
public String getPath(String fieldName) {
173+
if (mappingObjectPath.isEmpty()) {
174+
return fieldName;
175+
} else {
176+
StringBuilder sb = new StringBuilder();
177+
for (String s : mappingObjectPath) {
178+
sb.append(s);
179+
sb.append(".");
180+
}
181+
sb.append(fieldName);
182+
return sb.toString();
183+
}
184+
}
185+
160186
private static class MultiFieldParserContext extends MappingParserContext {
161187
MultiFieldParserContext(MappingParserContext in) {
162188
super(

‎server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@
4646
import java.util.stream.Collectors;
4747
import java.util.stream.Stream;
4848

49+
import static org.elasticsearch.index.IndexSettings.INDEX_MAPPER_SOURCE_AUTO_EXCLUDE_TYPES_SETTING;
50+
4951
public class ObjectMapper extends Mapper {
5052
private static final Logger logger = LogManager.getLogger(ObjectMapper.class);
5153
private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ObjectMapper.class);
@@ -278,7 +280,7 @@ public boolean supportsVersion(IndexVersion indexCreatedVersion) {
278280
@Override
279281
public Mapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext)
280282
throws MapperParsingException {
281-
parserContext.incrementMappingObjectDepth(); // throws MapperParsingException if depth limit is exceeded
283+
parserContext.incrementMappingObjectDepth(name); // throws MapperParsingException if depth limit is exceeded
282284
Optional<Subobjects> subobjects = parseSubobjects(node);
283285
Builder builder = new Builder(name, subobjects);
284286
parseObjectFields(node, parserContext, builder);
@@ -407,6 +409,10 @@ protected static void parseProperties(Builder objBuilder, Map<String, Object> pr
407409
+ " Check the documentation."
408410
);
409411
}
412+
List<String> autoExcludes = INDEX_MAPPER_SOURCE_AUTO_EXCLUDE_TYPES_SETTING.get(parserContext.getSettings());
413+
if (autoExcludes.isEmpty() == false && autoExcludes.contains(type)) {
414+
parserContext.addAutoExclude(parserContext.getPath(fieldName));
415+
}
410416
Mapper.Builder fieldBuilder;
411417
if (objBuilder.subobjects.isPresent() && objBuilder.subobjects.get() != Subobjects.ENABLED) {
412418
fieldBuilder = typeParser.parse(fieldName, propNode, parserContext);

‎server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java‎

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import java.util.HashSet;
4949
import java.util.List;
5050
import java.util.Locale;
51+
import java.util.Objects;
5152
import java.util.Set;
5253

5354
public class SourceFieldMapper extends MetadataFieldMapper {
@@ -156,6 +157,19 @@ public static class Builder extends MetadataFieldMapper.Builder {
156157
false,
157158
m -> Arrays.asList(toType(m).excludes)
158159
);
160+
private final Parameter<List<String>> autoExcludes = new Parameter<>(
161+
"auto_excludes",
162+
true,
163+
Collections::emptyList,
164+
(n, c, o) -> c.getAutoExcludes(),
165+
m -> {
166+
Set<String> merge = new HashSet<>(this.autoExcludes.getValue());
167+
merge.addAll(Arrays.asList(toType(m).autoExcludes));
168+
return merge.stream().toList();
169+
},
170+
XContentBuilder::field,
171+
Objects::toString
172+
);
159173

160174
private final Settings settings;
161175

@@ -168,13 +182,15 @@ public static class Builder extends MetadataFieldMapper.Builder {
168182
public Builder(
169183
IndexMode indexMode,
170184
final Settings settings,
185+
final List<String> autoExcludes,
171186
boolean sourceModeIsNoop,
172187
boolean supportsCheckForNonDefaultParams,
173188
boolean serializeMode
174189
) {
175190
super(Defaults.NAME);
176191
this.settings = settings;
177192
this.indexMode = indexMode;
193+
this.autoExcludes.setValue(autoExcludes);
178194
this.supportsNonDefaultParameterValues = supportsCheckForNonDefaultParams == false
179195
|| settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true);
180196
this.sourceModeIsNoop = sourceModeIsNoop;
@@ -201,11 +217,14 @@ public Builder setSynthetic() {
201217

202218
@Override
203219
protected Parameter<?>[] getParameters() {
204-
return new Parameter<?>[] { enabled, mode, includes, excludes };
220+
return new Parameter<?>[] { enabled, mode, includes, excludes, autoExcludes };
205221
}
206222

207223
private boolean isDefault() {
208-
return enabled.get().value() && includes.getValue().isEmpty() && excludes.getValue().isEmpty();
224+
return enabled.get().value()
225+
&& includes.getValue().isEmpty()
226+
&& excludes.getValue().isEmpty()
227+
&& autoExcludes.getValue().isEmpty();
209228
}
210229

211230
@Override
@@ -259,6 +278,7 @@ public SourceFieldMapper build() {
259278
enabled.get(),
260279
includes.getValue().toArray(Strings.EMPTY_ARRAY),
261280
excludes.getValue().toArray(Strings.EMPTY_ARRAY),
281+
autoExcludes.getValue().toArray(Strings.EMPTY_ARRAY),
262282
serializeMode,
263283
sourceModeIsNoop
264284
);
@@ -329,6 +349,7 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) {
329349
c -> new Builder(
330350
c.getIndexSettings().getMode(),
331351
c.getSettings(),
352+
c.getAutoExcludes(),
332353
c.indexVersionCreated().onOrAfter(IndexVersions.SOURCE_MAPPER_MODE_ATTRIBUTE_NOOP),
333354
c.indexVersionCreated().onOrAfter(IndexVersions.SOURCE_MAPPER_LOSSY_PARAMS_CHECK),
334355
onOrAfterDeprecateModeVersion(c.indexVersionCreated()) == false
@@ -383,6 +404,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
383404

384405
private final String[] includes;
385406
private final String[] excludes;
407+
private final String[] autoExcludes;
386408
private final SourceFilter sourceFilter;
387409

388410
private SourceFieldMapper(
@@ -392,23 +414,39 @@ private SourceFieldMapper(
392414
String[] excludes,
393415
boolean serializeMode,
394416
boolean sourceModeIsNoop
417+
) {
418+
this(mode, enabled, includes, excludes, Strings.EMPTY_ARRAY, serializeMode, sourceModeIsNoop);
419+
}
420+
421+
private SourceFieldMapper(
422+
Mode mode,
423+
Explicit<Boolean> enabled,
424+
String[] includes,
425+
String[] excludes,
426+
String[] autoExcludes,
427+
boolean serializeMode,
428+
boolean sourceModeIsNoop
395429
) {
396430
super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED)));
397431
this.mode = mode;
398432
this.enabled = enabled;
399-
this.sourceFilter = buildSourceFilter(includes, excludes);
433+
this.autoExcludes = autoExcludes;
434+
this.sourceFilter = buildSourceFilter(includes, excludes, autoExcludes);
400435
this.includes = includes;
401436
this.excludes = excludes;
402437
this.complete = stored() && sourceFilter == null;
403438
this.serializeMode = serializeMode;
404439
this.sourceModeIsNoop = sourceModeIsNoop;
405440
}
406441

407-
private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) {
408-
if (CollectionUtils.isEmpty(includes) && CollectionUtils.isEmpty(excludes)) {
442+
private static SourceFilter buildSourceFilter(String[] includes, String[] excludes, String[] autoExcludes) {
443+
if (CollectionUtils.isEmpty(includes) && CollectionUtils.isEmpty(excludes) && CollectionUtils.isEmpty(autoExcludes)) {
409444
return null;
410445
}
411-
return new SourceFilter(includes, excludes);
446+
Set<String> excludesSet = new HashSet<>();
447+
excludesSet.addAll(Arrays.asList(excludes));
448+
excludesSet.addAll(Arrays.asList(autoExcludes));
449+
return new SourceFilter(includes, excludesSet.toArray(Strings.EMPTY_ARRAY));
412450
}
413451

414452
private boolean stored() {
@@ -550,7 +588,7 @@ protected String contentType() {
550588

551589
@Override
552590
public FieldMapper.Builder getMergeBuilder() {
553-
return new Builder(null, Settings.EMPTY, sourceModeIsNoop, false, serializeMode).init(this);
591+
return new Builder(null, Settings.EMPTY, List.of(autoExcludes), sourceModeIsNoop, false, serializeMode).init(this);
554592
}
555593

556594
public boolean isSynthetic() {
@@ -615,4 +653,9 @@ private static void removeSyntheticVectorFields(
615653
destination.copyCurrentEvent(parser);
616654
}
617655
}
656+
657+
// test
658+
String[] getAutoExcludes() {
659+
return autoExcludes;
660+
}
618661
}

‎server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.xcontent.json.JsonXContent;
1818

1919
import java.io.IOException;
20+
import java.util.Collections;
2021
import java.util.List;
2122
import java.util.Map;
2223
import java.util.Optional;
@@ -69,7 +70,9 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti
6970
XContentParser parser = createParser(JsonXContent.jsonXContent, source);
7071
SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON);
7172

72-
SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, false, false).setSynthetic().build();
73+
SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, Collections.emptyList(), false, false, false)
74+
.setSynthetic()
75+
.build();
7376
RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
7477
new PassThroughObjectMapper.Builder("labels").setPriority(0).setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE)
7578
).build(MapperBuilderContext.root(false, false));

‎server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.index.mapper;
1111

12+
import org.apache.lucene.index.IndexOptions;
1213
import org.apache.lucene.index.IndexableField;
1314
import org.apache.lucene.util.BytesRef;
1415
import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -31,6 +32,7 @@
3132
import java.util.Locale;
3233
import java.util.Map;
3334

35+
import static org.elasticsearch.index.IndexSettings.INDEX_MAPPER_SOURCE_AUTO_EXCLUDE_TYPES_SETTING;
3436
import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING;
3537
import static org.hamcrest.Matchers.containsString;
3638
import static org.hamcrest.Matchers.equalTo;
@@ -800,4 +802,59 @@ public void testRecoverySourceWithTimeSeriesCustom() throws IOException {
800802
assertNull(doc.rootDoc().getField("_recovery_source"));
801803
}
802804
}
805+
806+
public void testAutoExcludeTypesSetting() throws IOException {
807+
String mappings = """
808+
{
809+
"_doc" : {
810+
"properties": {
811+
"foo": {
812+
"type": "dense_vector",
813+
"similarity": "l2_norm"
814+
},
815+
"bar": {
816+
"type": "keyword"
817+
},
818+
"baz": {
819+
"type": "object",
820+
"properties": {
821+
"foo": {
822+
"type": "binary",
823+
"doc_values": true
824+
}
825+
}
826+
}
827+
}
828+
}
829+
}
830+
""";
831+
Settings settings = Settings.builder()
832+
.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName())
833+
.putList(INDEX_MAPPER_SOURCE_AUTO_EXCLUDE_TYPES_SETTING.getKey(), List.of("dense_vector", "binary"))
834+
.build();
835+
MapperService mapperService = createMapperService(settings, mappings);
836+
DocumentMapper docMapper = mapperService.documentMapper();
837+
final byte[] binaryValue = new byte[100];
838+
binaryValue[56] = 1;
839+
ParsedDocument doc = docMapper.parse(
840+
source("123", b -> b
841+
.field("foo", List.of(3f, 2f, 1.5f))
842+
.field("bar", "value")
843+
.field("baz.foo", binaryValue)
844+
, null));
845+
SourceFieldMapper source = (SourceFieldMapper) doc.dynamicMappingsUpdate().getMetadataMapperByName("_source");
846+
assertNotNull(source);
847+
assertTrue(List.of(source.getAutoExcludes()).contains("foo"));
848+
assertTrue(List.of(source.getAutoExcludes()).contains("baz.foo"));
849+
850+
IndexableField sourceField = doc.rootDoc().getField("_source");
851+
Map<String, Object> sourceAsMap;
852+
try (XContentParser parser = createParser(JsonXContent.jsonXContent, new BytesArray(sourceField.binaryValue()))) {
853+
sourceAsMap = parser.map();
854+
}
855+
assertThat(sourceAsMap.containsKey("foo"), equalTo(false));
856+
assertThat(sourceAsMap.containsKey("bar"), equalTo(true));
857+
assertThat(sourceAsMap.containsKey("baz.foo"), equalTo(false));
858+
}
803859
}
860+

‎server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,9 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() {
384384

385385
public void testSyntheticSourceSearchLookup() throws IOException {
386386
// Build a mapping using synthetic source
387-
SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, false, false).setSynthetic().build();
387+
SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, Collections.emptyList(), false, false, false)
388+
.setSynthetic()
389+
.build();
388390
RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
389391
new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100)
390392
).build(MapperBuilderContext.root(true, false));

0 commit comments

Comments
 (0)