Skip to content

Commit 35f7efe

Browse files
masseykemasseyke
authored
Adding support for additional mapping to simulate ingest API (#114742) (#115284)
1 parent 6d0d3ba commit 35f7efe

File tree

12 files changed

+676
-131
lines changed

12 files changed

+676
-131
lines changed

‎docs/changelog/114742.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 114742
2+
summary: Adding support for additional mapping to simulate ingest API
3+
area: Ingest Node
4+
type: enhancement
5+
issues: []

‎docs/reference/ingest/apis/simulate-ingest.asciidoc‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,14 @@ POST /_ingest/_simulate
108108
"index_patterns": ["my-index-*"],
109109
"composed_of": ["component_template_1", "component_template_2"]
110110
}
111+
},
112+
"mapping_addition": { <4>
113+
"dynamic": "strict",
114+
"properties": {
115+
"foo": {
116+
"type": "keyword"
117+
}
118+
}
111119
}
112120
}
113121
----
@@ -117,6 +125,7 @@ POST /_ingest/_simulate
117125
These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result.
118126
<3> This replaces the existing `my-index-template` index template with the contents given here for the duration of this request.
119127
These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result.
128+
<4> This mapping is merged into the index's final mapping just before validation. It is used only for the duration of this request.
120129

121130
[[simulate-ingest-api-request]]
122131
==== {api-request-title}
@@ -246,6 +255,10 @@ include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=request-body]
246255
247256
====
248257

258+
`mapping_addition`::
259+
(Optional, <<mapping,mapping object>>)
260+
Definition of a mapping that will be merged into the index's mapping for validation during the course of this request.
261+
249262
[[simulate-ingest-api-example]]
250263
==== {api-examples-title}
251264

‎qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml‎

Lines changed: 355 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1216,3 +1216,358 @@ setup:
12161216
- match: { docs.0.doc._source.foo: "FOO" }
12171217
- match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] }
12181218
- not_exists: docs.0.doc.error
1219+
1220+
---
1221+
"Test ingest simulate with mapping addition for data streams":
1222+
# In this test, we make sure that when the index template is a data stream template, simulate ingest works the same whether the data
1223+
# stream has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings.
1224+
1225+
- skip:
1226+
features:
1227+
- headers
1228+
- allowed_warnings
1229+
1230+
- requires:
1231+
cluster_features: ["simulate.mapping.addition"]
1232+
reason: "ingest simulate mapping addition added in 8.16"
1233+
1234+
- do:
1235+
headers:
1236+
Content-Type: application/json
1237+
ingest.put_pipeline:
1238+
id: "foo-pipeline"
1239+
body: >
1240+
{
1241+
"processors": [
1242+
{
1243+
"set": {
1244+
"field": "foo",
1245+
"value": true
1246+
}
1247+
}
1248+
]
1249+
}
1250+
- match: { acknowledged: true }
1251+
1252+
- do:
1253+
cluster.put_component_template:
1254+
name: mappings_template
1255+
body:
1256+
template:
1257+
mappings:
1258+
dynamic: strict
1259+
properties:
1260+
foo:
1261+
type: boolean
1262+
1263+
- do:
1264+
cluster.put_component_template:
1265+
name: settings_template
1266+
body:
1267+
template:
1268+
settings:
1269+
index:
1270+
default_pipeline: "foo-pipeline"
1271+
1272+
- do:
1273+
allowed_warnings:
1274+
- "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation"
1275+
indices.put_index_template:
1276+
name: test-composable-1
1277+
body:
1278+
index_patterns:
1279+
- foo*
1280+
composed_of:
1281+
- mappings_template
1282+
- settings_template
1283+
1284+
- do:
1285+
allowed_warnings:
1286+
- "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation"
1287+
indices.put_index_template:
1288+
name: my-template-1
1289+
body:
1290+
index_patterns: [simple-data-stream1]
1291+
composed_of:
1292+
- mappings_template
1293+
- settings_template
1294+
data_stream: {}
1295+
1296+
# Here we replace my-template-1 with a substitute version that uses the settings_template_2 and mappings_template_2 templates defined in
1297+
# this request, and foo-pipeline-2 defined in this request.
1298+
- do:
1299+
headers:
1300+
Content-Type: application/json
1301+
simulate.ingest:
1302+
index: simple-data-stream1
1303+
body: >
1304+
{
1305+
"docs": [
1306+
{
1307+
"_id": "asdf",
1308+
"_source": {
1309+
"@timestamp": 1234,
1310+
"foo": false
1311+
}
1312+
}
1313+
],
1314+
"pipeline_substitutions": {
1315+
"foo-pipeline-2": {
1316+
"processors": [
1317+
{
1318+
"set": {
1319+
"field": "foo",
1320+
"value": "FOO"
1321+
}
1322+
}
1323+
]
1324+
}
1325+
},
1326+
"component_template_substitutions": {
1327+
"settings_template_2": {
1328+
"template": {
1329+
"settings": {
1330+
"index": {
1331+
"default_pipeline": "foo-pipeline-2"
1332+
}
1333+
}
1334+
}
1335+
},
1336+
"mappings_template_2": {
1337+
"template": {
1338+
"mappings": {
1339+
"dynamic": "strict",
1340+
"properties": {
1341+
"foo": {
1342+
"type": "integer"
1343+
}
1344+
}
1345+
}
1346+
}
1347+
}
1348+
},
1349+
"index_template_substitutions": {
1350+
"my-template-1": {
1351+
"index_patterns": ["simple-data-stream1"],
1352+
"composed_of": ["settings_template_2", "mappings_template_2"],
1353+
"data_stream": {}
1354+
}
1355+
},
1356+
"mapping_addition": {
1357+
"dynamic": "strict",
1358+
"properties": {
1359+
"foo": {
1360+
"type": "keyword"
1361+
}
1362+
}
1363+
}
1364+
}
1365+
- length: { docs: 1 }
1366+
- match: { docs.0.doc._index: "simple-data-stream1" }
1367+
- match: { docs.0.doc._source.foo: "FOO" }
1368+
- match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] }
1369+
- not_exists: docs.0.doc.error
1370+
1371+
- do:
1372+
indices.create_data_stream:
1373+
name: simple-data-stream1
1374+
- is_true: acknowledged
1375+
1376+
- do:
1377+
cluster.health:
1378+
wait_for_status: yellow
1379+
1380+
# Now that we have created a data stream, run the exact same simulate ingeset request to make sure we still get the same result, and that
1381+
# the substitutions and additions from the simulate ingest request are used instead of information from the data stream or its backing
1382+
# index.
1383+
- do:
1384+
headers:
1385+
Content-Type: application/json
1386+
simulate.ingest:
1387+
index: simple-data-stream1
1388+
body: >
1389+
{
1390+
"docs": [
1391+
{
1392+
"_id": "asdf",
1393+
"_source": {
1394+
"@timestamp": 1234,
1395+
"foo": false
1396+
}
1397+
}
1398+
],
1399+
"pipeline_substitutions": {
1400+
"foo-pipeline-2": {
1401+
"processors": [
1402+
{
1403+
"set": {
1404+
"field": "foo",
1405+
"value": "FOO"
1406+
}
1407+
}
1408+
]
1409+
}
1410+
},
1411+
"component_template_substitutions": {
1412+
"settings_template_2": {
1413+
"template": {
1414+
"settings": {
1415+
"index": {
1416+
"default_pipeline": "foo-pipeline-2"
1417+
}
1418+
}
1419+
}
1420+
},
1421+
"mappings_template_2": {
1422+
"template": {
1423+
"mappings": {
1424+
"dynamic": "strict",
1425+
"properties": {
1426+
"foo": {
1427+
"type": "integer"
1428+
}
1429+
}
1430+
}
1431+
}
1432+
}
1433+
},
1434+
"index_template_substitutions": {
1435+
"my-template-1": {
1436+
"index_patterns": ["simple-data-stream1"],
1437+
"composed_of": ["settings_template_2", "mappings_template_2"],
1438+
"data_stream": {}
1439+
}
1440+
},
1441+
"mapping_addition": {
1442+
"dynamic": "strict",
1443+
"properties": {
1444+
"foo": {
1445+
"type": "keyword"
1446+
}
1447+
}
1448+
}
1449+
}
1450+
- length: { docs: 1 }
1451+
- match: { docs.0.doc._index: "simple-data-stream1" }
1452+
- match: { docs.0.doc._source.foo: "FOO" }
1453+
- match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] }
1454+
- not_exists: docs.0.doc.error
1455+
1456+
---
1457+
"Test mapping addition works with legacy templates":
1458+
# In this test, we make sure that when the index template is a data stream template, simulate ingest works the same whether the data
1459+
# stream has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings.
1460+
1461+
- skip:
1462+
features:
1463+
- headers
1464+
- allowed_warnings
1465+
1466+
- requires:
1467+
cluster_features: ["simulate.mapping.addition"]
1468+
reason: "ingest simulate mapping addition added in 8.16"
1469+
1470+
- do:
1471+
indices.put_template:
1472+
name: my-legacy-template
1473+
body:
1474+
index_patterns: foo-*
1475+
settings:
1476+
number_of_replicas: 0
1477+
mappings:
1478+
dynamic: strict
1479+
properties:
1480+
foo:
1481+
type: integer
1482+
bar:
1483+
type: boolean
1484+
1485+
- do:
1486+
headers:
1487+
Content-Type: application/json
1488+
simulate.ingest:
1489+
index: foo-1
1490+
body: >
1491+
{
1492+
"docs": [
1493+
{
1494+
"_id": "asdf",
1495+
"_source": {
1496+
"foo": 3,
1497+
"bar": "not a boolean"
1498+
}
1499+
}
1500+
]
1501+
}
1502+
- length: { docs: 1 }
1503+
- match: { docs.0.doc._index: "foo-1" }
1504+
- match: { docs.0.doc._source.foo: 3 }
1505+
- match: { docs.0.doc._source.bar: "not a boolean" }
1506+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1507+
1508+
- do:
1509+
headers:
1510+
Content-Type: application/json
1511+
simulate.ingest:
1512+
index: foo-1
1513+
body: >
1514+
{
1515+
"docs": [
1516+
{
1517+
"_id": "asdf",
1518+
"_source": {
1519+
"foo": 3,
1520+
"bar": "not a boolean"
1521+
}
1522+
}
1523+
],
1524+
"mapping_addition": {
1525+
"dynamic": "strict",
1526+
"properties": {
1527+
"bar": {
1528+
"type": "keyword"
1529+
}
1530+
}
1531+
}
1532+
}
1533+
- length: { docs: 1 }
1534+
- match: { docs.0.doc._index: "foo-1" }
1535+
- match: { docs.0.doc._source.foo: 3 }
1536+
- match: { docs.0.doc._source.bar: "not a boolean" }
1537+
- not_exists: docs.0.doc.error
1538+
1539+
- do:
1540+
indices.create:
1541+
index: foo-1
1542+
- match: { acknowledged: true }
1543+
1544+
- do:
1545+
headers:
1546+
Content-Type: application/json
1547+
simulate.ingest:
1548+
index: foo-1
1549+
body: >
1550+
{
1551+
"docs": [
1552+
{
1553+
"_id": "asdf",
1554+
"_source": {
1555+
"foo": 3,
1556+
"bar": "not a boolean"
1557+
}
1558+
}
1559+
],
1560+
"mapping_addition": {
1561+
"dynamic": "strict",
1562+
"properties": {
1563+
"bar": {
1564+
"type": "keyword"
1565+
}
1566+
}
1567+
}
1568+
}
1569+
- length: { docs: 1 }
1570+
- match: { docs.0.doc._index: "foo-1" }
1571+
- match: { docs.0.doc._source.foo: 3 }
1572+
- match: { docs.0.doc._source.bar: "not a boolean" }
1573+
- not_exists: docs.0.doc.error

0 commit comments

Comments
 (0)