Skip to content

Commit 832bd4c

Browse files
masseykemasseyke
committed
Simulate ingest API uses existing index mapping when mapping_addition is given (#132101)
(cherry picked from commit 593f48f) # Conflicts: # server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
1 parent b2fd38d commit 832bd4c

File tree

3 files changed

+217
-36
lines changed

3 files changed

+217
-36
lines changed

‎docs/changelog/132101.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132101
2+
summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
3+
given
4+
area: Ingest Node
5+
type: bug
6+
issues: []

‎qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1740,3 +1740,194 @@ setup:
17401740
- match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" }
17411741
- match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] }
17421742
- not_exists: docs.0.doc.error
1743+
1744+
---
1745+
"Test mapping addition correctly respects mapping of indices without templates":
1746+
# In this test, we make sure that when we have an index that has mapping but was not built with a template, that the
1747+
# additional_mapping respects the existing mapping for validation.
1748+
1749+
- skip:
1750+
features:
1751+
- headers
1752+
- allowed_warnings
1753+
1754+
# A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
1755+
# because this test is making sure we get correct behavior when an index matches *no* template:
1756+
- do:
1757+
indices.delete_template:
1758+
name: '*'
1759+
ignore: 404
1760+
1761+
# We create the index no-template-index with an implicit mapping that has a foo field with type long:
1762+
- do:
1763+
bulk:
1764+
refresh: true
1765+
body:
1766+
- '{"index": {"_index": "no-template-index"}}'
1767+
- '{"foo": 3}'
1768+
1769+
# Now we make sure that the existing mapping is taken into account when we simulate with a mapping_addition. Since
1770+
# the pre-existing mapping has foo mapped as a long, this ought to fail with a document_parsing_exception because
1771+
# we are attempting to write a boolean foo.
1772+
- do:
1773+
headers:
1774+
Content-Type: application/json
1775+
simulate.ingest:
1776+
index: no-template-index
1777+
body: >
1778+
{
1779+
"docs": [
1780+
{
1781+
"_id": "test-id",
1782+
"_index": "no-template-index",
1783+
"_source": {
1784+
"@timestamp": "2025-07-25T09:06:06.929Z",
1785+
"is_valid": true,
1786+
"foo": true
1787+
}
1788+
}
1789+
],
1790+
"mapping_addition": {
1791+
"properties": {
1792+
"is_valid": {
1793+
"type": "boolean"
1794+
}
1795+
}
1796+
}
1797+
}
1798+
- length: { docs: 1 }
1799+
- match: { docs.0.doc._index: "no-template-index" }
1800+
- match: { docs.0.doc._source.foo: true }
1801+
- match: { docs.0.doc._source.is_valid: true }
1802+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1803+
1804+
# Now we add a template for this index.
1805+
- do:
1806+
indices.put_template:
1807+
name: my-template-1
1808+
body:
1809+
index_patterns: no-template-index
1810+
mappings:
1811+
properties:
1812+
foo:
1813+
type: boolean
1814+
1815+
# And we still expect the index's mapping to be used rather than the template:
1816+
- do:
1817+
headers:
1818+
Content-Type: application/json
1819+
simulate.ingest:
1820+
index: no-template-index
1821+
body: >
1822+
{
1823+
"docs": [
1824+
{
1825+
"_id": "test-id",
1826+
"_index": "no-template-index",
1827+
"_source": {
1828+
"@timestamp": "2025-07-25T09:06:06.929Z",
1829+
"is_valid": true,
1830+
"foo": true
1831+
}
1832+
}
1833+
],
1834+
"mapping_addition": {
1835+
"properties": {
1836+
"is_valid": {
1837+
"type": "boolean"
1838+
}
1839+
}
1840+
}
1841+
}
1842+
- length: { docs: 1 }
1843+
- match: { docs.0.doc._index: "no-template-index" }
1844+
- match: { docs.0.doc._source.foo: true }
1845+
- match: { docs.0.doc._source.is_valid: true }
1846+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1847+
1848+
---
1849+
"Test ingest simulate with mapping addition for data streams when write index has different mapping":
1850+
# In this test, we make sure that when a data stream's write index has a mapping that is different from the mapping
1851+
# in its template, and a mapping_override is given, then the mapping_override is applied to the mapping of the write
1852+
# index rather than the mapping of the template.
1853+
1854+
- skip:
1855+
features:
1856+
- headers
1857+
- allowed_warnings
1858+
1859+
- do:
1860+
cluster.put_component_template:
1861+
name: mappings_template
1862+
body:
1863+
template:
1864+
mappings:
1865+
dynamic: strict
1866+
properties:
1867+
foo:
1868+
type: boolean
1869+
bar:
1870+
type: boolean
1871+
1872+
- do:
1873+
allowed_warnings:
1874+
- "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation"
1875+
indices.put_index_template:
1876+
name: my-template-1
1877+
body:
1878+
index_patterns: [simple-data-stream1]
1879+
composed_of:
1880+
- mappings_template
1881+
data_stream: {}
1882+
1883+
- do:
1884+
indices.create_data_stream:
1885+
name: simple-data-stream1
1886+
- is_true: acknowledged
1887+
1888+
- do:
1889+
cluster.health:
1890+
wait_for_status: yellow
1891+
1892+
# Now that the data stream exists, we change the template to remove the mapping for bar. The write index still has the
1893+
# old mapping.
1894+
- do:
1895+
cluster.put_component_template:
1896+
name: mappings_template
1897+
body:
1898+
template:
1899+
mappings:
1900+
properties:
1901+
foo:
1902+
type: boolean
1903+
1904+
# We expect the mapping_addition to be added to the mapping of the write index, which has a boolean bar field. So this
1905+
# simulate ingest ought to fail.
1906+
- do:
1907+
headers:
1908+
Content-Type: application/json
1909+
simulate.ingest:
1910+
index: simple-data-stream1
1911+
body: >
1912+
{
1913+
"docs": [
1914+
{
1915+
"_id": "asdf",
1916+
"_source": {
1917+
"@timestamp": 1234,
1918+
"bar": "baz"
1919+
}
1920+
}
1921+
],
1922+
"mapping_addition": {
1923+
"properties": {
1924+
"baz": {
1925+
"type": "keyword"
1926+
}
1927+
}
1928+
}
1929+
}
1930+
- length: { docs: 1 }
1931+
- match: { docs.0.doc._index: "simple-data-stream1" }
1932+
- match: { docs.0.doc._source.bar: "baz" }
1933+
- match: { docs.0.doc.error.type: "document_parsing_exception" }

‎server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java‎

Lines changed: 20 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import java.util.HashMap;
7070
import java.util.List;
7171
import java.util.Map;
72+
import java.util.Optional;
7273
import java.util.Set;
7374
import java.util.concurrent.Executor;
7475

@@ -196,32 +197,15 @@ private Tuple<Collection<String>, Exception> validateMappings(
196197
Collection<String> ignoredFields = List.of();
197198
IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index());
198199
try {
199-
if (indexAbstraction != null
200-
&& componentTemplateSubstitutions.isEmpty()
201-
&& indexTemplateSubstitutions.isEmpty()
202-
&& mappingAddition.isEmpty()) {
200+
if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) {
203201
/*
204-
* In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService
205-
* to do the mapping validation, using all the existing logic for validation.
202+
* In this case the index exists and we don't have any template overrides. So we can just merge the mappingAddition (which
203+
* might not exist) into the existing index mapping.
206204
*/
207205
IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata()));
208-
indicesService.withTempIndexService(imd, indexService -> {
209-
indexService.mapperService().updateMapping(null, imd);
210-
return IndexShard.prepareIndex(
211-
indexService.mapperService(),
212-
sourceToParse,
213-
SequenceNumbers.UNASSIGNED_SEQ_NO,
214-
-1,
215-
-1,
216-
VersionType.INTERNAL,
217-
Engine.Operation.Origin.PRIMARY,
218-
Long.MIN_VALUE,
219-
false,
220-
request.ifSeqNo(),
221-
request.ifPrimaryTerm(),
222-
0
223-
);
224-
});
206+
CompressedXContent mappings = Optional.ofNullable(imd.mapping()).map(MappingMetadata::source).orElse(null);
207+
CompressedXContent mergedMappings = mappingAddition == null ? null : mergeMappings(mappings, mappingAddition);
208+
ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, mergedMappings, request, sourceToParse);
225209
} else {
226210
/*
227211
* The index did not exist, or we have component template substitutions, so we put together the mappings from existing
@@ -295,15 +279,6 @@ private Tuple<Collection<String>, Exception> validateMappings(
295279
);
296280
final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
297281
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
298-
} else if (indexAbstraction != null && mappingAddition.isEmpty() == false) {
299-
/*
300-
* The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we
301-
* merge in the mapping addition if it exists, and validate.
302-
*/
303-
MappingMetadata mappingFromIndex = clusterService.state().metadata().index(indexAbstraction.getName()).mapping();
304-
CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source();
305-
CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition);
306-
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
307282
} else {
308283
/*
309284
* The index matched no templates and had no mapping of its own. If there were component template substitutions
@@ -331,9 +306,6 @@ private Collection<String> validateUpdatedMappings(
331306
IndexRequest request,
332307
SourceToParse sourceToParse
333308
) throws IOException {
334-
if (updatedMappings == null) {
335-
return List.of(); // no validation to do
336-
}
337309
Settings dummySettings = Settings.builder()
338310
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
339311
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
@@ -345,8 +317,20 @@ private Collection<String> validateUpdatedMappings(
345317
originalIndexMetadataBuilder.putMapping(new MappingMetadata(originalMappings));
346318
}
347319
final IndexMetadata originalIndexMetadata = originalIndexMetadataBuilder.build();
320+
return validateUpdatedMappingsFromIndexMetadata(originalIndexMetadata, updatedMappings, request, sourceToParse);
321+
}
322+
323+
private Collection<String> validateUpdatedMappingsFromIndexMetadata(
324+
IndexMetadata originalIndexMetadata,
325+
@Nullable CompressedXContent updatedMappings,
326+
IndexRequest request,
327+
SourceToParse sourceToParse
328+
) throws IOException {
329+
if (updatedMappings == null) {
330+
return List.of(); // no validation to do
331+
}
348332
final IndexMetadata updatedIndexMetadata = IndexMetadata.builder(request.index())
349-
.settings(dummySettings)
333+
.settings(originalIndexMetadata.getSettings())
350334
.putMapping(new MappingMetadata(updatedMappings))
351335
.build();
352336
Engine.Index result = indicesService.withTempIndexService(originalIndexMetadata, indexService -> {

0 commit comments

Comments
 (0)