Skip to content

Commit 8656cc6

Browse files
masseykemasseyke
authored
Simulate ingest API uses existing index mapping when mapping_addition is given (#132101) (#132217)
(cherry picked from commit 593f48f) # Conflicts: # server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
1 parent 5cfc279 commit 8656cc6

File tree

3 files changed

+217
-36
lines changed

3 files changed

+217
-36
lines changed

‎docs/changelog/132101.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132101
2+
summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
3+
given
4+
area: Ingest Node
5+
type: bug
6+
issues: []

‎qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1776,3 +1776,194 @@ setup:
17761776
- match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" }
17771777
- match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] }
17781778
- not_exists: docs.0.doc.error
1779+
1780+
---
1781+
"Test mapping addition correctly respects mapping of indices without templates":
1782+
# In this test, we make sure that when we have an index that has mapping but was not built with a template, that the
1783+
# additional_mapping respects the existing mapping for validation.
1784+
1785+
- skip:
1786+
features:
1787+
- headers
1788+
- allowed_warnings
1789+
1790+
# A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
1791+
# because this test is making sure we get correct behavior when an index matches *no* template:
1792+
- do:
1793+
indices.delete_template:
1794+
name: '*'
1795+
ignore: 404
1796+
1797+
# We create the index no-template-index with an implicit mapping that has a foo field with type long:
1798+
- do:
1799+
bulk:
1800+
refresh: true
1801+
body:
1802+
- '{"index": {"_index": "no-template-index"}}'
1803+
- '{"foo": 3}'
1804+
1805+
# Now we make sure that the existing mapping is taken into account when we simulate with a mapping_addition. Since
1806+
# the pre-existing mapping has foo mapped as a long, this ought to fail with a document_parsing_exception because
1807+
# we are attempting to write a boolean foo.
1808+
- do:
1809+
headers:
1810+
Content-Type: application/json
1811+
simulate.ingest:
1812+
index: no-template-index
1813+
body: >
1814+
{
1815+
"docs": [
1816+
{
1817+
"_id": "test-id",
1818+
"_index": "no-template-index",
1819+
"_source": {
1820+
"@timestamp": "2025-07-25T09:06:06.929Z",
1821+
"is_valid": true,
1822+
"foo": true
1823+
}
1824+
}
1825+
],
1826+
"mapping_addition": {
1827+
"properties": {
1828+
"is_valid": {
1829+
"type": "boolean"
1830+
}
1831+
}
1832+
}
1833+
}
1834+
- length: { docs: 1 }
1835+
- match: { docs.0.doc._index: "no-template-index" }
1836+
- match: { docs.0.doc._source.foo: true }
1837+
- match: { docs.0.doc._source.is_valid: true }
1838+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1839+
1840+
# Now we add a template for this index.
1841+
- do:
1842+
indices.put_template:
1843+
name: my-template-1
1844+
body:
1845+
index_patterns: no-template-index
1846+
mappings:
1847+
properties:
1848+
foo:
1849+
type: boolean
1850+
1851+
# And we still expect the index's mapping to be used rather than the template:
1852+
- do:
1853+
headers:
1854+
Content-Type: application/json
1855+
simulate.ingest:
1856+
index: no-template-index
1857+
body: >
1858+
{
1859+
"docs": [
1860+
{
1861+
"_id": "test-id",
1862+
"_index": "no-template-index",
1863+
"_source": {
1864+
"@timestamp": "2025-07-25T09:06:06.929Z",
1865+
"is_valid": true,
1866+
"foo": true
1867+
}
1868+
}
1869+
],
1870+
"mapping_addition": {
1871+
"properties": {
1872+
"is_valid": {
1873+
"type": "boolean"
1874+
}
1875+
}
1876+
}
1877+
}
1878+
- length: { docs: 1 }
1879+
- match: { docs.0.doc._index: "no-template-index" }
1880+
- match: { docs.0.doc._source.foo: true }
1881+
- match: { docs.0.doc._source.is_valid: true }
1882+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1883+
1884+
---
1885+
"Test ingest simulate with mapping addition for data streams when write index has different mapping":
1886+
# In this test, we make sure that when a data stream's write index has a mapping that is different from the mapping
1887+
# in its template, and a mapping_override is given, then the mapping_override is applied to the mapping of the write
1888+
# index rather than the mapping of the template.
1889+
1890+
- skip:
1891+
features:
1892+
- headers
1893+
- allowed_warnings
1894+
1895+
- do:
1896+
cluster.put_component_template:
1897+
name: mappings_template
1898+
body:
1899+
template:
1900+
mappings:
1901+
dynamic: strict
1902+
properties:
1903+
foo:
1904+
type: boolean
1905+
bar:
1906+
type: boolean
1907+
1908+
- do:
1909+
allowed_warnings:
1910+
- "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation"
1911+
indices.put_index_template:
1912+
name: my-template-1
1913+
body:
1914+
index_patterns: [simple-data-stream1]
1915+
composed_of:
1916+
- mappings_template
1917+
data_stream: {}
1918+
1919+
- do:
1920+
indices.create_data_stream:
1921+
name: simple-data-stream1
1922+
- is_true: acknowledged
1923+
1924+
- do:
1925+
cluster.health:
1926+
wait_for_status: yellow
1927+
1928+
# Now that the data stream exists, we change the template to remove the mapping for bar. The write index still has the
1929+
# old mapping.
1930+
- do:
1931+
cluster.put_component_template:
1932+
name: mappings_template
1933+
body:
1934+
template:
1935+
mappings:
1936+
properties:
1937+
foo:
1938+
type: boolean
1939+
1940+
# We expect the mapping_addition to be added to the mapping of the write index, which has a boolean bar field. So this
1941+
# simulate ingest ought to fail.
1942+
- do:
1943+
headers:
1944+
Content-Type: application/json
1945+
simulate.ingest:
1946+
index: simple-data-stream1
1947+
body: >
1948+
{
1949+
"docs": [
1950+
{
1951+
"_id": "asdf",
1952+
"_source": {
1953+
"@timestamp": 1234,
1954+
"bar": "baz"
1955+
}
1956+
}
1957+
],
1958+
"mapping_addition": {
1959+
"properties": {
1960+
"baz": {
1961+
"type": "keyword"
1962+
}
1963+
}
1964+
}
1965+
}
1966+
- length: { docs: 1 }
1967+
- match: { docs.0.doc._index: "simple-data-stream1" }
1968+
- match: { docs.0.doc._source.bar: "baz" }
1969+
- match: { docs.0.doc.error.type: "document_parsing_exception" }

‎server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java

Lines changed: 20 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import java.util.HashMap;
7070
import java.util.List;
7171
import java.util.Map;
72+
import java.util.Optional;
7273
import java.util.Set;
7374
import java.util.concurrent.Executor;
7475

@@ -205,32 +206,15 @@ private Tuple<Collection<String>, Exception> validateMappings(
205206
Collection<String> ignoredFields = List.of();
206207
IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index());
207208
try {
208-
if (indexAbstraction != null
209-
&& componentTemplateSubstitutions.isEmpty()
210-
&& indexTemplateSubstitutions.isEmpty()
211-
&& mappingAddition.isEmpty()) {
209+
if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) {
212210
/*
213-
* In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService
214-
* to do the mapping validation, using all the existing logic for validation.
211+
* In this case the index exists and we don't have any template overrides. So we can just merge the mappingAddition (which
212+
* might not exist) into the existing index mapping.
215213
*/
216214
IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata()));
217-
indicesService.withTempIndexService(imd, indexService -> {
218-
indexService.mapperService().updateMapping(null, imd);
219-
return IndexShard.prepareIndex(
220-
indexService.mapperService(),
221-
sourceToParse,
222-
SequenceNumbers.UNASSIGNED_SEQ_NO,
223-
-1,
224-
-1,
225-
VersionType.INTERNAL,
226-
Engine.Operation.Origin.PRIMARY,
227-
Long.MIN_VALUE,
228-
false,
229-
request.ifSeqNo(),
230-
request.ifPrimaryTerm(),
231-
0
232-
);
233-
});
215+
CompressedXContent mappings = Optional.ofNullable(imd.mapping()).map(MappingMetadata::source).orElse(null);
216+
CompressedXContent mergedMappings = mappingAddition == null ? null : mergeMappings(mappings, mappingAddition);
217+
ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, mergedMappings, request, sourceToParse);
234218
} else {
235219
/*
236220
* The index did not exist, or we have component template substitutions, so we put together the mappings from existing
@@ -304,15 +288,6 @@ private Tuple<Collection<String>, Exception> validateMappings(
304288
);
305289
final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
306290
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
307-
} else if (indexAbstraction != null && mappingAddition.isEmpty() == false) {
308-
/*
309-
* The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we
310-
* merge in the mapping addition if it exists, and validate.
311-
*/
312-
MappingMetadata mappingFromIndex = clusterService.state().metadata().index(indexAbstraction.getName()).mapping();
313-
CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source();
314-
CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition);
315-
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
316291
} else {
317292
/*
318293
* The index matched no templates and had no mapping of its own. If there were component template substitutions
@@ -340,9 +315,6 @@ private Collection<String> validateUpdatedMappings(
340315
IndexRequest request,
341316
SourceToParse sourceToParse
342317
) throws IOException {
343-
if (updatedMappings == null) {
344-
return List.of(); // no validation to do
345-
}
346318
Settings dummySettings = Settings.builder()
347319
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
348320
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
@@ -354,8 +326,20 @@ private Collection<String> validateUpdatedMappings(
354326
originalIndexMetadataBuilder.putMapping(new MappingMetadata(originalMappings));
355327
}
356328
final IndexMetadata originalIndexMetadata = originalIndexMetadataBuilder.build();
329+
return validateUpdatedMappingsFromIndexMetadata(originalIndexMetadata, updatedMappings, request, sourceToParse);
330+
}
331+
332+
private Collection<String> validateUpdatedMappingsFromIndexMetadata(
333+
IndexMetadata originalIndexMetadata,
334+
@Nullable CompressedXContent updatedMappings,
335+
IndexRequest request,
336+
SourceToParse sourceToParse
337+
) throws IOException {
338+
if (updatedMappings == null) {
339+
return List.of(); // no validation to do
340+
}
357341
final IndexMetadata updatedIndexMetadata = IndexMetadata.builder(request.index())
358-
.settings(dummySettings)
342+
.settings(originalIndexMetadata.getSettings())
359343
.putMapping(new MappingMetadata(updatedMappings))
360344
.build();
361345
Engine.Index result = indicesService.withTempIndexService(originalIndexMetadata, indexService -> {

0 commit comments

Comments
 (0)