Skip to content

Commit f0dd37e

Browse files
quux00quux00
committed
Most manual tests now pass. The two that still don't work right are:
1. One case with subobjects:false doesn't work as hoped: // this does NOT return an error - it just silently "fails" to create the _project entry PUT test333?error_trace=true { "mappings": { "subobjects": false, "properties" : { "_project" : { "type" : "object" } } } } // YAY - this one fails with correct error message PUT test333?error_trace=true { "mappings": { "subobjects": false, "properties": { "_project": { "type": "object", "properties": { "myfield": { "type": "keyword" } } } } } } 2. Creating a runtime mapping is not detected and prevented: PUT /rt-index { "mappings": { "runtime": { "_project": { "type": "keyword", "script": { "source": "emit(doc['existing_field'].value + ' some additional text')" } } } } }
1 parent 6f6df95 commit f0dd37e

File tree

8 files changed

+178
-38
lines changed

8 files changed

+178
-38
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/DefaultRootObjectMapperNamespaceValidator.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
/**
13+
* No-op Default of RootObjectMapperNamespaceValidator used in non-serverless Elasticsearch envs.
14+
*/
15+
public class DefaultRootObjectMapperNamespaceValidator implements RootObjectMapperNamespaceValidator {
16+
@Override
17+
public void validateNamespace(ObjectMapper.Subobjects subobjects, Mapper mapper) {}
18+
19+
// MP FIXME remove
20+
@Override
21+
public String toString() {
22+
return "I'm the DefaultRootObjectMapperNamespaceValidator{}";
23+
}
24+
}

‎server/src/main/java/org/elasticsearch/index/mapper/MapperService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,8 @@ public MapperService(
245245
indexAnalyzers,
246246
indexSettings,
247247
idFieldMapper,
248-
bitSetProducer
248+
bitSetProducer,
249+
mapperRegistry.getNamespaceValidator()
249250
);
250251
this.documentParser = new DocumentParser(parserConfiguration, this.mappingParserContextSupplier.get());
251252
Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(

‎server/src/main/java/org/elasticsearch/index/mapper/MappingParserContext.java‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ public class MappingParserContext {
4343
private final Function<Query, BitSetProducer> bitSetProducer;
4444
private final long mappingObjectDepthLimit;
4545
private long mappingObjectDepth = 0;
46+
private final RootObjectMapperNamespaceValidator namespaceValidator;
4647

4748
public MappingParserContext(
4849
Function<String, SimilarityProvider> similarityLookupService,
@@ -55,7 +56,8 @@ public MappingParserContext(
5556
IndexAnalyzers indexAnalyzers,
5657
IndexSettings indexSettings,
5758
IdFieldMapper idFieldMapper,
58-
Function<Query, BitSetProducer> bitSetProducer
59+
Function<Query, BitSetProducer> bitSetProducer,
60+
RootObjectMapperNamespaceValidator namespaceValidator
5961
) {
6062
this.similarityLookupService = similarityLookupService;
6163
this.typeParsers = typeParsers;
@@ -69,6 +71,41 @@ public MappingParserContext(
6971
this.idFieldMapper = idFieldMapper;
7072
this.mappingObjectDepthLimit = indexSettings.getMappingDepthLimit();
7173
this.bitSetProducer = bitSetProducer;
74+
this.namespaceValidator = namespaceValidator;
75+
}
76+
77+
// MP TODO: only used by tests, so remove this after tests are updated?
78+
public MappingParserContext(
79+
Function<String, SimilarityProvider> similarityLookupService,
80+
Function<String, Mapper.TypeParser> typeParsers,
81+
Function<String, RuntimeField.Parser> runtimeFieldParsers,
82+
IndexVersion indexVersionCreated,
83+
Supplier<TransportVersion> clusterTransportVersion,
84+
Supplier<SearchExecutionContext> searchExecutionContextSupplier,
85+
ScriptCompiler scriptCompiler,
86+
IndexAnalyzers indexAnalyzers,
87+
IndexSettings indexSettings,
88+
IdFieldMapper idFieldMapper,
89+
Function<Query, BitSetProducer> bitSetProducer
90+
) {
91+
this(
92+
similarityLookupService,
93+
typeParsers,
94+
runtimeFieldParsers,
95+
indexVersionCreated,
96+
clusterTransportVersion,
97+
searchExecutionContextSupplier,
98+
scriptCompiler,
99+
indexAnalyzers,
100+
indexSettings,
101+
idFieldMapper,
102+
bitSetProducer,
103+
null
104+
);
105+
}
106+
107+
public RootObjectMapperNamespaceValidator getNamespaceValidator() {
108+
return namespaceValidator;
72109
}
73110

74111
public IndexAnalyzers getIndexAnalyzers() {

‎server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -76,11 +76,17 @@ public static class Builder extends ObjectMapper.Builder {
7676
protected final Map<String, RuntimeField> runtimeFields = new HashMap<>();
7777
protected Explicit<Boolean> dateDetection = Defaults.DATE_DETECTION;
7878
protected Explicit<Boolean> numericDetection = Defaults.NUMERIC_DETECTION;
79+
protected RootObjectMapperNamespaceValidator namespaceValidator;
7980

8081
public Builder(String name, Optional<Subobjects> subobjects) {
8182
super(name, subobjects);
8283
}
8384

85+
public Builder addNamespaceValidator(RootObjectMapperNamespaceValidator namespaceValidator) {
86+
this.namespaceValidator = namespaceValidator;
87+
return this;
88+
}
89+
8490
public Builder dynamicDateTimeFormatter(Collection<DateFormatter> dateTimeFormatters) {
8591
this.dynamicDateTimeFormatters = new Explicit<>(dateTimeFormatters.toArray(new DateFormatter[0]), true);
8692
return this;
@@ -120,7 +126,8 @@ public RootObjectMapper build(MapperBuilderContext context) {
120126
dynamicDateTimeFormatters,
121127
dynamicTemplates,
122128
dateDetection,
123-
numericDetection
129+
numericDetection,
130+
namespaceValidator
124131
);
125132
}
126133
}
@@ -130,6 +137,7 @@ public RootObjectMapper build(MapperBuilderContext context) {
130137
private final Explicit<Boolean> numericDetection;
131138
private final Explicit<DynamicTemplate[]> dynamicTemplates;
132139
private final Map<String, RuntimeField> runtimeFields;
140+
private final RootObjectMapperNamespaceValidator namespaceValidator;
133141

134142
RootObjectMapper(
135143
String name,
@@ -142,14 +150,16 @@ public RootObjectMapper build(MapperBuilderContext context) {
142150
Explicit<DateFormatter[]> dynamicDateTimeFormatters,
143151
Explicit<DynamicTemplate[]> dynamicTemplates,
144152
Explicit<Boolean> dateDetection,
145-
Explicit<Boolean> numericDetection
153+
Explicit<Boolean> numericDetection,
154+
RootObjectMapperNamespaceValidator namespaceValidator
146155
) {
147156
super(name, name, enabled, subobjects, sourceKeepMode, dynamic, mappers);
148157
this.runtimeFields = runtimeFields;
149158
this.dynamicTemplates = dynamicTemplates;
150159
this.dynamicDateTimeFormatters = dynamicDateTimeFormatters;
151160
this.dateDetection = dateDetection;
152161
this.numericDetection = numericDetection;
162+
this.namespaceValidator = namespaceValidator;
153163
if (sourceKeepMode.orElse(SourceKeepMode.NONE) == SourceKeepMode.ALL) {
154164
throw new MapperParsingException(
155165
"root object can't be configured with [" + Mapper.SYNTHETIC_SOURCE_KEEP_PARAM + ":" + SourceKeepMode.ALL + "]"
@@ -178,7 +188,8 @@ RootObjectMapper withoutMappers() {
178188
dynamicDateTimeFormatters,
179189
dynamicTemplates,
180190
dateDetection,
181-
numericDetection
191+
numericDetection,
192+
namespaceValidator
182193
);
183194
}
184195

@@ -294,7 +305,8 @@ public RootObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeCo
294305
dynamicDateTimeFormatters,
295306
dynamicTemplates,
296307
dateDetection,
297-
numericDetection
308+
numericDetection,
309+
namespaceValidator
298310
);
299311
}
300312

@@ -451,6 +463,7 @@ public static RootObjectMapper.Builder parse(String name, Map<String, Object> no
451463
throws MapperParsingException {
452464
Optional<Subobjects> subobjects = parseSubobjects(node);
453465
RootObjectMapper.Builder builder = new Builder(name, subobjects);
466+
builder.addNamespaceValidator(parserContext.getNamespaceValidator());
454467
Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
455468
while (iterator.hasNext()) {
456469
Map.Entry<String, Object> entry = iterator.next();
@@ -541,19 +554,10 @@ public int getTotalFieldsCount() {
541554
return super.getTotalFieldsCount() - 1 + runtimeFields.size();
542555
}
543556

544-
// MP TODO: - this needs to move to a serverless class, right?
545-
private static final String RESERVED_NAMESPACE = "_project";
546-
547557
@Override
548558
protected void validateSubField(Mapper mapper, MappingLookup mappers) {
549-
if (subobjects() == Subobjects.ENABLED) {
550-
if (mapper.leafName().equals(RESERVED_NAMESPACE)) {
551-
throw new IllegalArgumentException("Reserved Namespace. Fields may not start with " + RESERVED_NAMESPACE);
552-
}
553-
} else {
554-
if (mapper.leafName().startsWith(RESERVED_NAMESPACE)) {
555-
throw new IllegalArgumentException("Reserved Namespace. Fields may not start with " + RESERVED_NAMESPACE);
556-
}
559+
if (namespaceValidator != null) {
560+
namespaceValidator.validateNamespace(subobjects(), mapper);
557561
}
558562
super.validateSubField(mapper, mappers);
559563
}

‎server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapperNamespaceValidator.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,12 @@
1010
package org.elasticsearch.index.mapper;
1111

1212
/**
13-
* TODO: DOCUMENT ME
13+
* SPI to inject additional rules around namespaces that are prohibited
14+
* in Elasticsearch mappings.
1415
*/
1516
public interface RootObjectMapperNamespaceValidator {
17+
/**
18+
* If the namespace in the mapper is not allowed, an Exception should be thrown.
19+
*/
1620
void validateNamespace(ObjectMapper.Subobjects subobjects, Mapper mapper);
1721
}

‎server/src/main/java/org/elasticsearch/index/mapper/ServerlessRootObjectMapperNamespaceValidator.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper;
11+
12+
import org.elasticsearch.common.Strings;
13+
14+
/**
15+
* In serverless, prohibits mappings that start with _project, including subfields such as _project.foo.
16+
* The _project "namespace" in serverless is reserved in order to allow users to include project metadata tags
17+
* (e.g., _project._region or _project._csp, etc.) which are useful in cross-project search and ES|QL queries.
18+
*/
19+
public class ServerlessRootObjectMapperNamespaceValidator implements RootObjectMapperNamespaceValidator {
20+
private static final String SERVERLESS_RESERVED_NAMESPACE = "_project";
21+
22+
// MP TODO: we can also pass in a MappingLookup - would that help here?
23+
@Override
24+
public void validateNamespace(ObjectMapper.Subobjects subobjects, Mapper mapper) { // TODO: stop passing in Mapper and pass in String
25+
// fieldName
26+
if (mapper.leafName().equals(SERVERLESS_RESERVED_NAMESPACE)) {
27+
System.err.println("XX: 1A: " + mapper.leafName());
28+
System.err.println("XX: 1B: " + mapper.fullPath());
29+
System.err.println("XX: 1C: " + mapper.typeName());
30+
System.err.println("XX: 1D: " + mapper);
31+
throw new IllegalArgumentException(generateErrorMessage());
32+
} else if (subobjects != ObjectMapper.Subobjects.ENABLED) {
33+
System.err.println("YYY: 2A: " + mapper.leafName());
34+
System.err.println("YYY: 2B: " + mapper.fullPath());
35+
System.err.println("YYY: 2C: " + mapper.typeName());
36+
System.err.println("YYY: 2D: " + mapper);
37+
// leafName here will be something like _project.myfield, rather than just _project
38+
if (mapper.leafName().startsWith(SERVERLESS_RESERVED_NAMESPACE + ".")) {
39+
throw new IllegalArgumentException(generateErrorMessage(mapper.leafName()));
40+
}
41+
}
42+
}
43+
44+
private String generateErrorMessage(String fieldName) {
45+
return Strings.format(
46+
"Mapping rejected%s. No mappings of [%s] are allowed in order to avoid conflicts with project metadata tags in serverless",
47+
fieldName == null ? "" : ": [" + fieldName + "]",
48+
SERVERLESS_RESERVED_NAMESPACE
49+
);
50+
}
51+
52+
private String generateErrorMessage() {
53+
return generateErrorMessage(null);
54+
}
55+
}

‎server/src/main/java/org/elasticsearch/indices/IndicesModule.java‎

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -95,33 +95,37 @@
9595
public class IndicesModule extends AbstractModule {
9696
private final MapperRegistry mapperRegistry;
9797

98-
// TODO: this needs to be loaded from serverless somehow
99-
private static final String RESERVED_NAMESPACE = "_project";
100-
101-
public IndicesModule(List<MapperPlugin> mapperPlugins) {
98+
public IndicesModule(List<MapperPlugin> mapperPlugins, RootObjectMapperNamespaceValidator namespaceValidator) {
99+
// this is the only place that the MapperRegistry is created
102100
this.mapperRegistry = new MapperRegistry(
103101
getMappers(mapperPlugins),
104102
getRuntimeFields(mapperPlugins),
105103
getMetadataMappers(mapperPlugins),
106104
getFieldFilter(mapperPlugins),
107-
new RootObjectMapperNamespaceValidator() {
108-
@Override
109-
public void validateNamespace(ObjectMapper.Subobjects subobjects, Mapper mapper) {
110-
// TODO: in the future, this will be a no-op on stateful and loaded somehow dynamically in serverless
111-
if (subobjects == ObjectMapper.Subobjects.ENABLED) {
112-
if (mapper.leafName().equals(RESERVED_NAMESPACE)) {
113-
throw new IllegalArgumentException("xx reserved namespace: [" + RESERVED_NAMESPACE + ']');
114-
}
115-
} else {
116-
if (mapper.leafName().startsWith(RESERVED_NAMESPACE)) {
117-
throw new IllegalArgumentException("xx reserved namespace: [" + RESERVED_NAMESPACE + ']');
118-
}
119-
}
120-
}
121-
}
105+
namespaceValidator
106+
// new RootObjectMapperNamespaceValidator() {
107+
// @Override
108+
// public void validateNamespace(ObjectMapper.Subobjects subobjects, Mapper mapper) {
109+
// // TODO: in the future, this will be a no-op on stateful and loaded somehow dynamically in serverless
110+
// if (subobjects == ObjectMapper.Subobjects.ENABLED) {
111+
// if (mapper.leafName().equals(RESERVED_NAMESPACE)) {
112+
// throw new IllegalArgumentException("xx reserved namespace: [" + RESERVED_NAMESPACE + ']');
113+
// }
114+
// } else {
115+
// if (mapper.leafName().startsWith(RESERVED_NAMESPACE)) {
116+
// throw new IllegalArgumentException("xx reserved namespace: [" + RESERVED_NAMESPACE + ']');
117+
// }
118+
// }
119+
// }
120+
// }
122121
);
123122
}
124123

124+
// MP TODO: remove this constructor once all tests have been updated
125+
public IndicesModule(List<MapperPlugin> mapperPlugins) {
126+
this(mapperPlugins, null);
127+
}
128+
125129
public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
126130
return Arrays.asList(
127131
new NamedWriteableRegistry.Entry(Condition.class, MinAgeCondition.NAME, MinAgeCondition::new),

‎server/src/main/java/org/elasticsearch/node/NodeConstruction.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@
119119
import org.elasticsearch.index.analysis.AnalysisRegistry;
120120
import org.elasticsearch.index.engine.MergeMetrics;
121121
import org.elasticsearch.index.mapper.MapperMetrics;
122+
import org.elasticsearch.index.mapper.RootObjectMapperNamespaceValidator;
123+
import org.elasticsearch.index.mapper.ServerlessRootObjectMapperNamespaceValidator;
122124
import org.elasticsearch.index.mapper.SourceFieldMetrics;
123125
import org.elasticsearch.index.search.stats.ShardSearchPhaseAPMMetrics;
124126
import org.elasticsearch.index.shard.SearchOperationListener;
@@ -695,6 +697,15 @@ private void construct(
695697

696698
modules.bindToInstance(Tracer.class, telemetryProvider.getTracer());
697699

700+
// serverless deployments plug-in the namespace validator that prohibits mappings with "_project"
701+
RootObjectMapperNamespaceValidator namespaceValidator = pluginsService.loadSingletonServiceProvider(
702+
RootObjectMapperNamespaceValidator.class,
703+
// () -> new DefaultRootObjectMapperNamespaceValidator()
704+
() -> new ServerlessRootObjectMapperNamespaceValidator() // MP FIXME - for this testing branch only
705+
);
706+
modules.bindToInstance(RootObjectMapperNamespaceValidator.class, namespaceValidator);
707+
logger.warn("XXX namespaceValidator loaded: " + namespaceValidator); // MP FIXME remove
708+
698709
assert nodeEnvironment.nodeId() != null : "node ID must be set before constructing the Node";
699710
TaskManager taskManager = new TaskManager(
700711
settings,
@@ -806,7 +817,7 @@ private void construct(
806817
)::onNewInfo
807818
);
808819

809-
IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class).toList());
820+
IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class).toList(), namespaceValidator);
810821
modules.add(indicesModule);
811822

812823
modules.add(new GatewayModule());

0 commit comments

Comments
 (0)