Skip to content

Commit f09294c

Browse files
masseykemasseyke
authored
Simulate ingest API uses existing index mapping when mapping_addition is given (#132101) (#132204)
1 parent 06062cd commit f09294c

File tree

3 files changed

+217
-36
lines changed

3 files changed

+217
-36
lines changed

‎docs/changelog/132101.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132101
2+
summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
3+
given
4+
area: Ingest Node
5+
type: bug
6+
issues: []

‎qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1740,3 +1740,194 @@ setup:
17401740
- match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" }
17411741
- match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] }
17421742
- not_exists: docs.0.doc.error
1743+
1744+
---
1745+
"Test mapping addition correctly respects mapping of indices without templates":
1746+
# In this test, we make sure that when we have an index that has mapping but was not built with a template, that the
1747+
# additional_mapping respects the existing mapping for validation.
1748+
1749+
- skip:
1750+
features:
1751+
- headers
1752+
- allowed_warnings
1753+
1754+
# A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
1755+
# because this test is making sure we get correct behavior when an index matches *no* template:
1756+
- do:
1757+
indices.delete_template:
1758+
name: '*'
1759+
ignore: 404
1760+
1761+
# We create the index no-template-index with an implicit mapping that has a foo field with type long:
1762+
- do:
1763+
bulk:
1764+
refresh: true
1765+
body:
1766+
- '{"index": {"_index": "no-template-index"}}'
1767+
- '{"foo": 3}'
1768+
1769+
# Now we make sure that the existing mapping is taken into account when we simulate with a mapping_addition. Since
1770+
# the pre-existing mapping has foo mapped as a long, this ought to fail with a document_parsing_exception because
1771+
# we are attempting to write a boolean foo.
1772+
- do:
1773+
headers:
1774+
Content-Type: application/json
1775+
simulate.ingest:
1776+
index: no-template-index
1777+
body: >
1778+
{
1779+
"docs": [
1780+
{
1781+
"_id": "test-id",
1782+
"_index": "no-template-index",
1783+
"_source": {
1784+
"@timestamp": "2025-07-25T09:06:06.929Z",
1785+
"is_valid": true,
1786+
"foo": true
1787+
}
1788+
}
1789+
],
1790+
"mapping_addition": {
1791+
"properties": {
1792+
"is_valid": {
1793+
"type": "boolean"
1794+
}
1795+
}
1796+
}
1797+
}
1798+
- length: { docs: 1 }
1799+
- match: { docs.0.doc._index: "no-template-index" }
1800+
- match: { docs.0.doc._source.foo: true }
1801+
- match: { docs.0.doc._source.is_valid: true }
1802+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1803+
1804+
# Now we add a template for this index.
1805+
- do:
1806+
indices.put_template:
1807+
name: my-template-1
1808+
body:
1809+
index_patterns: no-template-index
1810+
mappings:
1811+
properties:
1812+
foo:
1813+
type: boolean
1814+
1815+
# And we still expect the index's mapping to be used rather than the template:
1816+
- do:
1817+
headers:
1818+
Content-Type: application/json
1819+
simulate.ingest:
1820+
index: no-template-index
1821+
body: >
1822+
{
1823+
"docs": [
1824+
{
1825+
"_id": "test-id",
1826+
"_index": "no-template-index",
1827+
"_source": {
1828+
"@timestamp": "2025-07-25T09:06:06.929Z",
1829+
"is_valid": true,
1830+
"foo": true
1831+
}
1832+
}
1833+
],
1834+
"mapping_addition": {
1835+
"properties": {
1836+
"is_valid": {
1837+
"type": "boolean"
1838+
}
1839+
}
1840+
}
1841+
}
1842+
- length: { docs: 1 }
1843+
- match: { docs.0.doc._index: "no-template-index" }
1844+
- match: { docs.0.doc._source.foo: true }
1845+
- match: { docs.0.doc._source.is_valid: true }
1846+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1847+
1848+
---
1849+
"Test ingest simulate with mapping addition for data streams when write index has different mapping":
1850+
# In this test, we make sure that when a data stream's write index has a mapping that is different from the mapping
1851+
# in its template, and a mapping_override is given, then the mapping_override is applied to the mapping of the write
1852+
# index rather than the mapping of the template.
1853+
1854+
- skip:
1855+
features:
1856+
- headers
1857+
- allowed_warnings
1858+
1859+
- do:
1860+
cluster.put_component_template:
1861+
name: mappings_template
1862+
body:
1863+
template:
1864+
mappings:
1865+
dynamic: strict
1866+
properties:
1867+
foo:
1868+
type: boolean
1869+
bar:
1870+
type: boolean
1871+
1872+
- do:
1873+
allowed_warnings:
1874+
- "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation"
1875+
indices.put_index_template:
1876+
name: my-template-1
1877+
body:
1878+
index_patterns: [simple-data-stream1]
1879+
composed_of:
1880+
- mappings_template
1881+
data_stream: {}
1882+
1883+
- do:
1884+
indices.create_data_stream:
1885+
name: simple-data-stream1
1886+
- is_true: acknowledged
1887+
1888+
- do:
1889+
cluster.health:
1890+
wait_for_status: yellow
1891+
1892+
# Now that the data stream exists, we change the template to remove the mapping for bar. The write index still has the
1893+
# old mapping.
1894+
- do:
1895+
cluster.put_component_template:
1896+
name: mappings_template
1897+
body:
1898+
template:
1899+
mappings:
1900+
properties:
1901+
foo:
1902+
type: boolean
1903+
1904+
# We expect the mapping_addition to be added to the mapping of the write index, which has a boolean bar field. So this
1905+
# simulate ingest ought to fail.
1906+
- do:
1907+
headers:
1908+
Content-Type: application/json
1909+
simulate.ingest:
1910+
index: simple-data-stream1
1911+
body: >
1912+
{
1913+
"docs": [
1914+
{
1915+
"_id": "asdf",
1916+
"_source": {
1917+
"@timestamp": 1234,
1918+
"bar": "baz"
1919+
}
1920+
}
1921+
],
1922+
"mapping_addition": {
1923+
"properties": {
1924+
"baz": {
1925+
"type": "keyword"
1926+
}
1927+
}
1928+
}
1929+
}
1930+
- length: { docs: 1 }
1931+
- match: { docs.0.doc._index: "simple-data-stream1" }
1932+
- match: { docs.0.doc._source.bar: "baz" }
1933+
- match: { docs.0.doc.error.type: "document_parsing_exception" }

‎server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java

Lines changed: 20 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import java.util.HashMap;
7070
import java.util.List;
7171
import java.util.Map;
72+
import java.util.Optional;
7273
import java.util.Set;
7374
import java.util.concurrent.Executor;
7475

@@ -198,32 +199,15 @@ private Tuple<Collection<String>, Exception> validateMappings(
198199
Collection<String> ignoredFields = List.of();
199200
IndexAbstraction indexAbstraction = project.getIndicesLookup().get(request.index());
200201
try {
201-
if (indexAbstraction != null
202-
&& componentTemplateSubstitutions.isEmpty()
203-
&& indexTemplateSubstitutions.isEmpty()
204-
&& mappingAddition.isEmpty()) {
202+
if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) {
205203
/*
206-
* In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService
207-
* to do the mapping validation, using all the existing logic for validation.
204+
* In this case the index exists and we don't have any template overrides. So we can just merge the mappingAddition (which
205+
* might not exist) into the existing index mapping.
208206
*/
209207
IndexMetadata imd = project.getIndexSafe(indexAbstraction.getWriteIndex(request, project));
210-
indicesService.withTempIndexService(imd, indexService -> {
211-
indexService.mapperService().updateMapping(null, imd);
212-
return IndexShard.prepareIndex(
213-
indexService.mapperService(),
214-
sourceToParse,
215-
SequenceNumbers.UNASSIGNED_SEQ_NO,
216-
-1,
217-
-1,
218-
VersionType.INTERNAL,
219-
Engine.Operation.Origin.PRIMARY,
220-
Long.MIN_VALUE,
221-
false,
222-
request.ifSeqNo(),
223-
request.ifPrimaryTerm(),
224-
0
225-
);
226-
});
208+
CompressedXContent mappings = Optional.ofNullable(imd.mapping()).map(MappingMetadata::source).orElse(null);
209+
CompressedXContent mergedMappings = mappingAddition == null ? null : mergeMappings(mappings, mappingAddition);
210+
ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, mergedMappings, request, sourceToParse);
227211
} else {
228212
/*
229213
* The index did not exist, or we have component template substitutions, so we put together the mappings from existing
@@ -296,15 +280,6 @@ private Tuple<Collection<String>, Exception> validateMappings(
296280
);
297281
final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
298282
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
299-
} else if (indexAbstraction != null && mappingAddition.isEmpty() == false) {
300-
/*
301-
* The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we
302-
* merge in the mapping addition if it exists, and validate.
303-
*/
304-
MappingMetadata mappingFromIndex = project.index(indexAbstraction.getName()).mapping();
305-
CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source();
306-
CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition);
307-
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
308283
} else {
309284
/*
310285
* The index matched no templates and had no mapping of its own. If there were component template substitutions
@@ -332,9 +307,6 @@ private Collection<String> validateUpdatedMappings(
332307
IndexRequest request,
333308
SourceToParse sourceToParse
334309
) throws IOException {
335-
if (updatedMappings == null) {
336-
return List.of(); // no validation to do
337-
}
338310
Settings dummySettings = Settings.builder()
339311
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
340312
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
@@ -346,8 +318,20 @@ private Collection<String> validateUpdatedMappings(
346318
originalIndexMetadataBuilder.putMapping(new MappingMetadata(originalMappings));
347319
}
348320
final IndexMetadata originalIndexMetadata = originalIndexMetadataBuilder.build();
321+
return validateUpdatedMappingsFromIndexMetadata(originalIndexMetadata, updatedMappings, request, sourceToParse);
322+
}
323+
324+
private Collection<String> validateUpdatedMappingsFromIndexMetadata(
325+
IndexMetadata originalIndexMetadata,
326+
@Nullable CompressedXContent updatedMappings,
327+
IndexRequest request,
328+
SourceToParse sourceToParse
329+
) throws IOException {
330+
if (updatedMappings == null) {
331+
return List.of(); // no validation to do
332+
}
349333
final IndexMetadata updatedIndexMetadata = IndexMetadata.builder(request.index())
350-
.settings(dummySettings)
334+
.settings(originalIndexMetadata.getSettings())
351335
.putMapping(new MappingMetadata(updatedMappings))
352336
.build();
353337
Engine.Index result = indicesService.withTempIndexService(originalIndexMetadata, indexService -> {

0 commit comments

Comments
 (0)