Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/changelog/132101.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 132101
summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
given
area: Ingest Node
type: bug
issues: []
Original file line number Diff line number Diff line change
Expand Up @@ -1740,3 +1740,194 @@ setup:
- match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" }
- match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] }
- not_exists: docs.0.doc.error

---
"Test mapping addition correctly respects mapping of indices without templates":
# In this test, we make sure that when we have an index that has mapping but was not built with a template, that the
# additional_mapping respects the existing mapping for validation.

- skip:
features:
- headers
- allowed_warnings

# A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
# because this test is making sure we get correct behavior when an index matches *no* template:
- do:
indices.delete_template:
name: '*'
ignore: 404

# We create the index no-template-index with an implicit mapping that has a foo field with type long:
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "no-template-index"}}'
- '{"foo": 3}'

# Now we make sure that the existing mapping is taken into account when we simulate with a mapping_addition. Since
# the pre-existing mapping has foo mapped as a long, this ought to fail with a document_parsing_exception because
# we are attempting to write a boolean foo.
- do:
headers:
Content-Type: application/json
simulate.ingest:
index: no-template-index
body: >
{
"docs": [
{
"_id": "test-id",
"_index": "no-template-index",
"_source": {
"@timestamp": "2025-07-25T09:06:06.929Z",
"is_valid": true,
"foo": true
}
}
],
"mapping_addition": {
"properties": {
"is_valid": {
"type": "boolean"
}
}
}
}
- length: { docs: 1 }
- match: { docs.0.doc._index: "no-template-index" }
- match: { docs.0.doc._source.foo: true }
- match: { docs.0.doc._source.is_valid: true }
- match: { docs.0.doc.error.type: "document_parsing_exception" }

# Now we add a template for this index.
- do:
indices.put_template:
name: my-template-1
body:
index_patterns: no-template-index
mappings:
properties:
foo:
type: boolean

# And we still expect the index's mapping to be used rather than the template:
- do:
headers:
Content-Type: application/json
simulate.ingest:
index: no-template-index
body: >
{
"docs": [
{
"_id": "test-id",
"_index": "no-template-index",
"_source": {
"@timestamp": "2025-07-25T09:06:06.929Z",
"is_valid": true,
"foo": true
}
}
],
"mapping_addition": {
"properties": {
"is_valid": {
"type": "boolean"
}
}
}
}
- length: { docs: 1 }
- match: { docs.0.doc._index: "no-template-index" }
- match: { docs.0.doc._source.foo: true }
- match: { docs.0.doc._source.is_valid: true }
- match: { docs.0.doc.error.type: "document_parsing_exception" }

---
"Test ingest simulate with mapping addition for data streams when write index has different mapping":
# In this test, we make sure that when a data stream's write index has a mapping that is different from the mapping
# in its template, and a mapping_override is given, then the mapping_override is applied to the mapping of the write
# index rather than the mapping of the template.

- skip:
features:
- headers
- allowed_warnings

- do:
cluster.put_component_template:
name: mappings_template
body:
template:
mappings:
dynamic: strict
properties:
foo:
type: boolean
bar:
type: boolean

- do:
allowed_warnings:
- "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation"
indices.put_index_template:
name: my-template-1
body:
index_patterns: [simple-data-stream1]
composed_of:
- mappings_template
data_stream: {}

- do:
indices.create_data_stream:
name: simple-data-stream1
- is_true: acknowledged

- do:
cluster.health:
wait_for_status: yellow

# Now that the data stream exists, we change the template to remove the mapping for bar. The write index still has the
# old mapping.
- do:
cluster.put_component_template:
name: mappings_template
body:
template:
mappings:
properties:
foo:
type: boolean

# We expect the mapping_addition to be added to the mapping of the write index, which has a boolean bar field. So this
# simulate ingest ought to fail.
- do:
headers:
Content-Type: application/json
simulate.ingest:
index: simple-data-stream1
body: >
{
"docs": [
{
"_id": "asdf",
"_source": {
"@timestamp": 1234,
"bar": "baz"
}
}
],
"mapping_addition": {
"properties": {
"baz": {
"type": "keyword"
}
}
}
}
- length: { docs: 1 }
- match: { docs.0.doc._index: "simple-data-stream1" }
- match: { docs.0.doc._source.bar: "baz" }
- match: { docs.0.doc.error.type: "document_parsing_exception" }
Original file line number Diff line number Diff line change
Expand Up @@ -198,32 +198,15 @@ private Tuple<Collection<String>, Exception> validateMappings(
Collection<String> ignoredFields = List.of();
IndexAbstraction indexAbstraction = project.getIndicesLookup().get(request.index());
try {
if (indexAbstraction != null
&& componentTemplateSubstitutions.isEmpty()
&& indexTemplateSubstitutions.isEmpty()
&& mappingAddition.isEmpty()) {
if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) {
/*
* In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService
* to do the mapping validation, using all the existing logic for validation.
* In this case the index exists and we don't have any template overrides. So we can just merge the mappingAddition (which
* might not exist) into the existing index mapping.
*/
IndexMetadata imd = project.getIndexSafe(indexAbstraction.getWriteIndex(request, project));
indicesService.withTempIndexService(imd, indexService -> {
indexService.mapperService().updateMapping(null, imd);
return IndexShard.prepareIndex(
indexService.mapperService(),
sourceToParse,
SequenceNumbers.UNASSIGNED_SEQ_NO,
-1,
-1,
VersionType.INTERNAL,
Engine.Operation.Origin.PRIMARY,
Long.MIN_VALUE,
false,
request.ifSeqNo(),
request.ifPrimaryTerm(),
0
);
});
CompressedXContent mappings = imd.mapping() == null ? null : imd.mapping().source();
CompressedXContent mergedMappings = mappingAddition == null ? null : mergeMappings(mappings, mappingAddition);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These … == null ? null : … are calling out for Optional.map() :D

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ha yeah I thought about doing that but then didn't (for reasons I don't remember). I'll do that now.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh right, because mergeMappings throws a checked exception, and dealing with that checked exception makes the code far more complicated than just the null checks. I'll change the first line though.

ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, mergedMappings, request, sourceToParse);
} else {
/*
* The index did not exist, or we have component template substitutions, so we put together the mappings from existing
Expand Down Expand Up @@ -296,15 +279,6 @@ private Tuple<Collection<String>, Exception> validateMappings(
);
final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
} else if (indexAbstraction != null && mappingAddition.isEmpty() == false) {
/*
* The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we
* merge in the mapping addition if it exists, and validate.
*/
MappingMetadata mappingFromIndex = project.index(indexAbstraction.getName()).mapping();
CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source();
CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition);
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
} else {
/*
* The index matched no templates and had no mapping of its own. If there were component template substitutions
Expand Down Expand Up @@ -332,9 +306,6 @@ private Collection<String> validateUpdatedMappings(
IndexRequest request,
SourceToParse sourceToParse
) throws IOException {
if (updatedMappings == null) {
return List.of(); // no validation to do
}
Settings dummySettings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
Expand All @@ -346,8 +317,20 @@ private Collection<String> validateUpdatedMappings(
originalIndexMetadataBuilder.putMapping(new MappingMetadata(originalMappings));
}
final IndexMetadata originalIndexMetadata = originalIndexMetadataBuilder.build();
return validateUpdatedMappingsFromIndexMetadata(originalIndexMetadata, updatedMappings, request, sourceToParse);
}

private Collection<String> validateUpdatedMappingsFromIndexMetadata(
IndexMetadata originalIndexMetadata,
@Nullable CompressedXContent updatedMappings,
IndexRequest request,
SourceToParse sourceToParse
) throws IOException {
if (updatedMappings == null) {
return List.of(); // no validation to do
}
final IndexMetadata updatedIndexMetadata = IndexMetadata.builder(request.index())
.settings(dummySettings)
.settings(originalIndexMetadata.getSettings())
.putMapping(new MappingMetadata(updatedMappings))
.build();
Engine.Index result = indicesService.withTempIndexService(originalIndexMetadata, indexService -> {
Expand Down