Skip to content

Commit 9cf174f

Browse files
masseykemasseyke
authored
Adding support for simulate ingest mapping adddition for indices with mappings that do not come from templates (#115359) (#115369)
1 parent 8f568b5 commit 9cf174f

File tree

4 files changed

+186
-10
lines changed

4 files changed

+186
-10
lines changed

‎docs/changelog/115359.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 115359
2+
summary: Adding support for simulate ingest mapping adddition for indices with mappings
3+
that do not come from templates
4+
area: Ingest Node
5+
type: enhancement
6+
issues: []

‎qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml‎

Lines changed: 142 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1229,7 +1229,7 @@ setup:
12291229

12301230
- requires:
12311231
cluster_features: ["simulate.mapping.addition"]
1232-
reason: "ingest simulate mapping addition added in 8.16"
1232+
reason: "ingest simulate mapping addition added in 8.17"
12331233

12341234
- do:
12351235
headers:
@@ -1465,7 +1465,7 @@ setup:
14651465

14661466
- requires:
14671467
cluster_features: ["simulate.mapping.addition"]
1468-
reason: "ingest simulate mapping addition added in 8.16"
1468+
reason: "ingest simulate mapping addition added in 8.17"
14691469

14701470
- do:
14711471
indices.put_template:
@@ -1571,3 +1571,143 @@ setup:
15711571
- match: { docs.0.doc._source.foo: 3 }
15721572
- match: { docs.0.doc._source.bar: "not a boolean" }
15731573
- not_exists: docs.0.doc.error
1574+
1575+
---
1576+
"Test mapping addition works with indices without templates":
1577+
# In this test, we make sure that when we have an index that has mapping but was not built with a template, that the additional_mapping
1578+
# is merged in with that template.
1579+
1580+
- skip:
1581+
features:
1582+
- headers
1583+
- allowed_warnings
1584+
1585+
- requires:
1586+
cluster_features: ["simulate.support.non.template.mapping"]
1587+
reason: "ingest simulate support for indices with mappings that didn't come from templates added in 8.17"
1588+
1589+
# First, make sure that validation fails before we create the index (since we are only defining to bar field but trying to index a value
1590+
# for foo.
1591+
- do:
1592+
headers:
1593+
Content-Type: application/json
1594+
simulate.ingest:
1595+
index: foo-1
1596+
body: >
1597+
{
1598+
"docs": [
1599+
{
1600+
"_id": "asdf",
1601+
"_source": {
1602+
"foo": 3,
1603+
"bar": "some text value"
1604+
}
1605+
}
1606+
],
1607+
"mapping_addition": {
1608+
"dynamic": "strict",
1609+
"properties": {
1610+
"bar": {
1611+
"type": "keyword"
1612+
}
1613+
}
1614+
}
1615+
}
1616+
- length: { docs: 1 }
1617+
- match: { docs.0.doc._index: "foo-1" }
1618+
- match: { docs.0.doc._source.foo: 3 }
1619+
- match: { docs.0.doc._source.bar: "some text value" }
1620+
- match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" }
1621+
1622+
- do:
1623+
indices.create:
1624+
index: foo-1
1625+
body:
1626+
mappings:
1627+
dynamic: strict
1628+
properties:
1629+
foo:
1630+
type: integer
1631+
- match: { acknowledged: true }
1632+
1633+
# Now make sure that the mapping for the newly-created index is getting picked up. Validation fails because it only defined a mapping
1634+
# for foo, not for bar.
1635+
- do:
1636+
headers:
1637+
Content-Type: application/json
1638+
simulate.ingest:
1639+
index: foo-1
1640+
body: >
1641+
{
1642+
"docs": [
1643+
{
1644+
"_id": "asdf",
1645+
"_source": {
1646+
"foo": 3,
1647+
"bar": "some text value"
1648+
}
1649+
}
1650+
]
1651+
}
1652+
- length: { docs: 1 }
1653+
- match: { docs.0.doc._index: "foo-1" }
1654+
- match: { docs.0.doc._source.foo: 3 }
1655+
- match: { docs.0.doc._source.bar: "some text value" }
1656+
- match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" }
1657+
1658+
# Now we make sure that the index's mapping gets merged with the mapping_addition:
1659+
- do:
1660+
headers:
1661+
Content-Type: application/json
1662+
simulate.ingest:
1663+
index: foo-1
1664+
body: >
1665+
{
1666+
"docs": [
1667+
{
1668+
"_id": "asdf",
1669+
"_source": {
1670+
"foo": 3,
1671+
"bar": "some text value"
1672+
}
1673+
}
1674+
],
1675+
"mapping_addition": {
1676+
"dynamic": "strict",
1677+
"properties": {
1678+
"bar": {
1679+
"type": "keyword"
1680+
}
1681+
}
1682+
}
1683+
}
1684+
- length: { docs: 1 }
1685+
- match: { docs.0.doc._index: "foo-1" }
1686+
- match: { docs.0.doc._source.foo: 3 }
1687+
- match: { docs.0.doc._source.bar: "some text value" }
1688+
- not_exists: docs.0.doc.error
1689+
1690+
# This last call to simulate is just making sure that if there are no templates, no index mappings, no substitutions, and no mapping
1691+
# addition, then validation does not fail
1692+
- do:
1693+
headers:
1694+
Content-Type: application/json
1695+
simulate.ingest:
1696+
index: nonexistent
1697+
body: >
1698+
{
1699+
"docs": [
1700+
{
1701+
"_id": "asdf",
1702+
"_source": {
1703+
"foo": 3,
1704+
"bar": "some text value"
1705+
}
1706+
}
1707+
]
1708+
}
1709+
- length: { docs: 1 }
1710+
- match: { docs.0.doc._index: "nonexistent" }
1711+
- match: { docs.0.doc._source.foo: 3 }
1712+
- match: { docs.0.doc._source.bar: "some text value" }
1713+
- not_exists: docs.0.doc.error

‎server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_ADDITION;
2020
import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION;
2121
import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES;
22+
import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING;
2223

2324
public class BulkFeatures implements FeatureSpecification {
2425
public Set<NodeFeature> getFeatures() {
@@ -27,7 +28,8 @@ public Set<NodeFeature> getFeatures() {
2728
SIMULATE_MAPPING_VALIDATION_TEMPLATES,
2829
SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS,
2930
SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS,
30-
SIMULATE_MAPPING_ADDITION
31+
SIMULATE_MAPPING_ADDITION,
32+
SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING
3133
);
3234
}
3335
}

‎server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ public class TransportSimulateBulkAction extends TransportAbstractBulkAction {
8484
);
8585
public static final NodeFeature SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS = new NodeFeature("simulate.index.template.substitutions");
8686
public static final NodeFeature SIMULATE_MAPPING_ADDITION = new NodeFeature("simulate.mapping.addition");
87+
public static final NodeFeature SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING = new NodeFeature("simulate.support.non.template.mapping");
8788
private final IndicesService indicesService;
8889
private final NamedXContentRegistry xContentRegistry;
8990
private final Set<IndexSettingProvider> indexSettingProviders;
@@ -258,6 +259,10 @@ private Exception validateMappings(
258259

259260
String matchingTemplate = findV2Template(simulatedState.metadata(), request.index(), false);
260261
if (matchingTemplate != null) {
262+
/*
263+
* The index matches a v2 template (including possibly one or more of the substitutions passed in). So we use this
264+
* template, and then possibly apply the mapping addition if it is not null, and validate.
265+
*/
261266
final Template template = TransportSimulateIndexTemplateAction.resolveTemplate(
262267
matchingTemplate,
263268
request.index(),
@@ -273,13 +278,36 @@ private Exception validateMappings(
273278
validateUpdatedMappings(mappings, mergedMappings, request, sourceToParse);
274279
} else {
275280
List<IndexTemplateMetadata> matchingTemplates = findV1Templates(simulatedState.metadata(), request.index(), false);
276-
final Map<String, Object> mappingsMap = MetadataCreateIndexService.parseV1Mappings(
277-
"{}",
278-
matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
279-
xContentRegistry
280-
);
281-
final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
282-
validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
281+
if (matchingTemplates.isEmpty() == false) {
282+
/*
283+
* The index matches v1 mappings. These are not compatible with component_template_substitutions or
284+
* index_template_substitutions, but we can apply a mapping_addition.
285+
*/
286+
final Map<String, Object> mappingsMap = MetadataCreateIndexService.parseV1Mappings(
287+
"{}",
288+
matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
289+
xContentRegistry
290+
);
291+
final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
292+
validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
293+
} else if (indexAbstraction != null && mappingAddition.isEmpty() == false) {
294+
/*
295+
* The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we
296+
* merge in the mapping addition if it exists, and validate.
297+
*/
298+
MappingMetadata mappingFromIndex = clusterService.state().metadata().index(indexAbstraction.getName()).mapping();
299+
CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source();
300+
CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition);
301+
validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
302+
} else {
303+
/*
304+
* The index matched no templates and had no mapping of its own. If there were component template substitutions
305+
* or index template substitutions, they didn't match anything. So just apply the mapping addition if it exists,
306+
* and validate.
307+
*/
308+
final CompressedXContent combinedMappings = mergeMappings(null, mappingAddition);
309+
validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
310+
}
283311
}
284312
}
285313
} catch (Exception e) {

0 commit comments

Comments
 (0)