Skip to content

Commit 292407b

Browse files
bruce-hong-gleanbruce-hong-glean
authored
Introduce false_allow_templates as a dynamic mapping option (opensearch-project#19065)
Signed-off-by: Bruce Hong <[email protected]>
1 parent dc70bf6 commit 292407b

File tree

8 files changed

+913
-19
lines changed

8 files changed

+913
-19
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
77
### Added
88
- Expand fetch phase profiling to support inner hits and top hits aggregation phases ([##18936](https://github.com/opensearch-project/OpenSearch/pull/18936))
99
- Add temporal routing processors for time-based document routing ([#18920](https://github.com/opensearch-project/OpenSearch/issues/18920))
10+
- The dynamic mapping parameter supports false_allow_templates ([#19065](https://github.com/opensearch-project/OpenSearch/pull/19065))
1011

1112

1213
### Changed

‎rest-api-spec/src/main/resources/rest-api-spec/test/index/111_false_allow_templates.yml‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
---
2+
"Index documents with setting dynamic parameter to false_allow_templates in the mapping of the index":
3+
- skip:
4+
version: " - 3.2.99"
5+
reason: "introduced in 3.3.0"
6+
7+
- do:
8+
indices.create:
9+
index: test_1
10+
body:
11+
mappings:
12+
dynamic: false_allow_templates
13+
dynamic_templates: [
14+
{
15+
dates: {
16+
"match": "date_*",
17+
"match_mapping_type": "date",
18+
"mapping": {
19+
"type": "date"
20+
}
21+
}
22+
},
23+
{
24+
strings: {
25+
"match": "stringField*",
26+
"match_mapping_type": "string",
27+
"mapping": {
28+
"type": "keyword"
29+
}
30+
}
31+
},
32+
{
33+
object: {
34+
"match": "objectField*",
35+
"match_mapping_type": "object",
36+
"mapping": {
37+
"type": "object",
38+
"properties": {
39+
"bar1": {
40+
"type": "keyword"
41+
},
42+
"bar2": {
43+
"type": "text"
44+
}
45+
}
46+
}
47+
}
48+
},
49+
{
50+
boolean: {
51+
"match": "booleanField*",
52+
"match_mapping_type": "boolean",
53+
"mapping": {
54+
"type": "boolean"
55+
}
56+
}
57+
},
58+
{
59+
long: {
60+
"match": "longField*",
61+
"match_mapping_type": "long",
62+
"mapping": {
63+
"type": "long"
64+
}
65+
}
66+
},
67+
{
68+
double: {
69+
"match": "doubleField*",
70+
"match_mapping_type": "double",
71+
"mapping": {
72+
"type": "double"
73+
}
74+
}
75+
},
76+
{
77+
array: {
78+
"match": "arrayField*",
79+
"mapping": {
80+
"type": "keyword"
81+
}
82+
}
83+
}
84+
]
85+
properties:
86+
url:
87+
type: keyword
88+
89+
- do:
90+
index:
91+
index: test_1
92+
id: 1
93+
body: {
94+
url: "https://example.com",
95+
date_timestamp: "2024-06-25T05:11:51.243Z",
96+
stringField: "bar",
97+
objectField: {
98+
bar1: "bar1",
99+
bar2: "bar2"
100+
},
101+
booleanField: true,
102+
longField: 123456789,
103+
doubleField: 123.456,
104+
arrayField: ["item1", "item2", "item3"],
105+
author: "John Doe"
106+
}
107+
108+
- do:
109+
get:
110+
index: test_1
111+
id: 1
112+
- match:
113+
_source:
114+
url: "https://example.com"
115+
date_timestamp: "2024-06-25T05:11:51.243Z"
116+
stringField: "bar"
117+
objectField:
118+
bar1: "bar1"
119+
bar2: "bar2"
120+
booleanField: true
121+
longField: 123456789
122+
doubleField: 123.456
123+
arrayField: ["item1", "item2", "item3"]
124+
author: "John Doe"
125+
126+
- do:
127+
indices.get_mapping:
128+
index: test_1
129+
130+
- match: {test_1.mappings.dynamic: false_allow_templates}
131+
- match: {test_1.mappings.properties.url.type: keyword}
132+
- match: {test_1.mappings.properties.date_timestamp.type: date}
133+
- match: {test_1.mappings.properties.stringField.type: keyword}
134+
- match: {test_1.mappings.properties.objectField.properties.bar1.type: keyword}
135+
- match: {test_1.mappings.properties.objectField.properties.bar2.type: text}
136+
- match: {test_1.mappings.properties.booleanField.type: boolean}
137+
- match: {test_1.mappings.properties.longField.type: long}
138+
- match: {test_1.mappings.properties.doubleField.type: double}
139+
- match: {test_1.mappings.properties.arrayField.type: keyword}
140+
- match: {test_1.mappings.properties.author: null}

‎rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,3 +190,34 @@ setup:
190190

191191
- match: {test_index1.mappings.dynamic: strict_allow_templates}
192192
- match: {test_index1.mappings.properties.test1.type: text}
193+
194+
---
195+
"post a mapping with setting dynamic to false_allow_templates":
196+
- skip:
197+
version: " - 3.2.99"
198+
reason: "introduced in 3.3.0"
199+
- do:
200+
indices.put_mapping:
201+
index: test_index1
202+
body:
203+
dynamic: false_allow_templates
204+
dynamic_templates: [
205+
{
206+
strings: {
207+
"match": "foo*",
208+
"match_mapping_type": "string",
209+
"mapping": {
210+
"type": "keyword"
211+
}
212+
}
213+
}
214+
]
215+
properties:
216+
test1:
217+
type: text
218+
219+
- do:
220+
indices.get_mapping: {}
221+
222+
- match: {test_index1.mappings.dynamic: false_allow_templates}
223+
- match: {test_index1.mappings.properties.test1.type: text}

‎server/src/main/java/org/opensearch/index/mapper/DocumentParser.java‎

Lines changed: 59 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,7 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
554554
throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName);
555555
case TRUE:
556556
case STRICT_ALLOW_TEMPLATES:
557+
case FALSE_ALLOW_TEMPLATES:
557558
Mapper.Builder builder = findTemplateBuilder(
558559
context,
559560
currentFieldName,
@@ -563,6 +564,10 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper,
563564
);
564565

565566
if (builder == null) {
567+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
568+
context.parser().skipChildren();
569+
break;
570+
}
566571
builder = new ObjectMapper.Builder(currentFieldName).enabled(true);
567572
}
568573
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
@@ -614,6 +619,7 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
614619
);
615620
case TRUE:
616621
case STRICT_ALLOW_TEMPLATES:
622+
case FALSE_ALLOW_TEMPLATES:
617623
Mapper.Builder builder = findTemplateBuilder(
618624
context,
619625
arrayFieldName,
@@ -622,6 +628,10 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper,
622628
parentMapper.fullPath()
623629
);
624630
if (builder == null) {
631+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
632+
context.parser().skipChildren();
633+
break;
634+
}
625635
parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
626636
} else {
627637
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(
@@ -786,13 +796,13 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
786796
if (parseableAsLong && context.root().numericDetection()) {
787797
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath);
788798
if (builder == null) {
789-
builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings());
799+
return handleNoTemplateFound(dynamic, () -> newLongBuilder(currentFieldName, context.indexSettings().getSettings()));
790800
}
791801
return builder;
792802
} else if (parseableAsDouble && context.root().numericDetection()) {
793803
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath);
794804
if (builder == null) {
795-
builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings());
805+
return handleNoTemplateFound(dynamic, () -> newFloatBuilder(currentFieldName, context.indexSettings().getSettings()));
796806
}
797807
return builder;
798808
} else if (parseableAsLong == false && parseableAsDouble == false && context.root().dateDetection()) {
@@ -808,14 +818,16 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
808818
}
809819
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, dateTimeFormatter, dynamic, fullPath);
810820
if (builder == null) {
811-
boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
812-
builder = new DateFieldMapper.Builder(
813-
currentFieldName,
814-
DateFieldMapper.Resolution.MILLISECONDS,
815-
dateTimeFormatter,
816-
ignoreMalformed,
817-
IndexMetadata.indexCreated(context.indexSettings().getSettings())
818-
);
821+
return handleNoTemplateFound(dynamic, () -> {
822+
boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
823+
return new DateFieldMapper.Builder(
824+
currentFieldName,
825+
DateFieldMapper.Resolution.MILLISECONDS,
826+
dateTimeFormatter,
827+
ignoreMalformed,
828+
IndexMetadata.indexCreated(context.indexSettings().getSettings())
829+
);
830+
});
819831
}
820832
return builder;
821833

@@ -824,8 +836,11 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
824836

825837
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath);
826838
if (builder == null) {
827-
builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField(
828-
new KeywordFieldMapper.Builder("keyword").ignoreAbove(256)
839+
return handleNoTemplateFound(
840+
dynamic,
841+
() -> new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField(
842+
new KeywordFieldMapper.Builder("keyword").ignoreAbove(256)
843+
)
829844
);
830845
}
831846
return builder;
@@ -836,7 +851,7 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
836851
|| numberType == XContentParser.NumberType.BIG_INTEGER) {
837852
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath);
838853
if (builder == null) {
839-
builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings());
854+
return handleNoTemplateFound(dynamic, () -> newLongBuilder(currentFieldName, context.indexSettings().getSettings()));
840855
}
841856
return builder;
842857
} else if (numberType == XContentParser.NumberType.FLOAT
@@ -847,34 +862,48 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(
847862
// no templates are defined, we use float by default instead of double
848863
// since this is much more space-efficient and should be enough most of
849864
// the time
850-
builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings());
865+
return handleNoTemplateFound(
866+
dynamic,
867+
() -> newFloatBuilder(currentFieldName, context.indexSettings().getSettings())
868+
);
851869
}
852870
return builder;
853871
}
854872
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
855873
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN, dynamic, fullPath);
856874
if (builder == null) {
857-
builder = new BooleanFieldMapper.Builder(currentFieldName);
875+
return handleNoTemplateFound(dynamic, () -> new BooleanFieldMapper.Builder(currentFieldName));
858876
}
859877
return builder;
860878
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
861879
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY, dynamic, fullPath);
862880
if (builder == null) {
863-
builder = new BinaryFieldMapper.Builder(currentFieldName);
881+
return handleNoTemplateFound(dynamic, () -> new BinaryFieldMapper.Builder(currentFieldName));
864882
}
865883
return builder;
866884
} else {
867885
Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath);
868886
if (builder != null) {
869887
return builder;
870888
}
889+
return handleNoTemplateFound(dynamic, () -> null);
871890
}
872891
// TODO how do we identify dynamically that its a binary value?
873892
throw new IllegalStateException(
874893
"Can't handle serializing a dynamic type with content token [" + token + "] and field name [" + currentFieldName + "]"
875894
);
876895
}
877896

897+
private static Mapper.Builder<?> handleNoTemplateFound(
898+
ObjectMapper.Dynamic dynamic,
899+
java.util.function.Supplier<Mapper.Builder<?>> builderSupplier
900+
) {
901+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
902+
return null;
903+
}
904+
return builderSupplier.get();
905+
}
906+
878907
private static void parseDynamicValue(
879908
final ParseContext context,
880909
ObjectMapper parentMapper,
@@ -888,8 +917,16 @@ private static void parseDynamicValue(
888917
if (dynamic == ObjectMapper.Dynamic.FALSE) {
889918
return;
890919
}
891-
final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
892920
final Mapper.Builder<?> builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath());
921+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES && builder == null) {
922+
// For FALSE_ALLOW_TEMPLATES, if no template matches, we still need to consume the token
923+
// to maintain proper JSON parsing state
924+
if (token == XContentParser.Token.START_OBJECT || token == XContentParser.Token.START_ARRAY) {
925+
context.parser().skipChildren();
926+
}
927+
return;
928+
}
929+
final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path());
893930
Mapper mapper = builder.build(builderContext);
894931
context.addDynamicMapper(mapper);
895932

@@ -978,8 +1015,9 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(
9781015
switch (dynamic) {
9791016
case STRICT:
9801017
throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parent.fullPath(), paths[i]);
981-
case STRICT_ALLOW_TEMPLATES:
9821018
case TRUE:
1019+
case STRICT_ALLOW_TEMPLATES:
1020+
case FALSE_ALLOW_TEMPLATES:
9831021
Mapper.Builder builder = findTemplateBuilder(
9841022
context,
9851023
paths[i],
@@ -988,6 +1026,9 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(
9881026
parent.fullPath()
9891027
);
9901028
if (builder == null) {
1029+
if (dynamic == ObjectMapper.Dynamic.FALSE_ALLOW_TEMPLATES) {
1030+
return new Tuple<>(pathsAdded, parent);
1031+
}
9911032
builder = new ObjectMapper.Builder(paths[i]).enabled(true);
9921033
}
9931034
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(

‎server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,8 @@ public enum Dynamic {
9393
TRUE,
9494
FALSE,
9595
STRICT,
96-
STRICT_ALLOW_TEMPLATES
96+
STRICT_ALLOW_TEMPLATES,
97+
FALSE_ALLOW_TEMPLATES
9798
}
9899

99100
/**
@@ -313,6 +314,8 @@ protected static boolean parseObjectOrDocumentTypeProperties(
313314
builder.dynamic(Dynamic.STRICT);
314315
} else if (value.equalsIgnoreCase("strict_allow_templates")) {
315316
builder.dynamic(Dynamic.STRICT_ALLOW_TEMPLATES);
317+
} else if (value.equalsIgnoreCase("false_allow_templates")) {
318+
builder.dynamic(Dynamic.FALSE_ALLOW_TEMPLATES);
316319
} else {
317320
boolean dynamic = XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".dynamic");
318321
builder.dynamic(dynamic ? Dynamic.TRUE : Dynamic.FALSE);

0 commit comments

Comments
 (0)