Skip to content

Commit c4fc521

Browse files
masseykemasseyke
committed
merging main
2 parents 1971fa8 + a231726 commit c4fc521

File tree

18 files changed

+322
-186
lines changed

18 files changed

+322
-186
lines changed

‎distribution/docker/src/docker/iron_bank/hardening_manifest.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ tags:
1111
# Build args passed to Dockerfile ARGs
1212
args:
1313
BASE_IMAGE: "redhat/ubi/ubi9"
14-
BASE_TAG: "9.5"
14+
BASE_TAG: "9.6"
1515
# Docker image labels
1616
labels:
1717
org.opencontainers.image.title: "elasticsearch"

‎docs/changelog/132101.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132101
2+
summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
3+
given
4+
area: Ingest Node
5+
type: bug
6+
issues: []

‎muted-tests.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -614,7 +614,9 @@ tests:
614614
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
615615
method: "testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_disk #2}"
616616
issue: https://github.com/elastic/elasticsearch/issues/132189
617-
617+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
618+
method: testDenseVectorMappingUpdate {initialType=int4_hnsw updateType=bbq_hnsw}
619+
issue: https://github.com/elastic/elasticsearch/issues/132211
618620

619621
# Examples:
620622
#

‎qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1741,6 +1741,197 @@ setup:
17411741
- match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] }
17421742
- not_exists: docs.0.doc.error
17431743

1744+
---
1745+
"Test mapping addition correctly respects mapping of indices without templates":
1746+
# In this test, we make sure that when we have an index that has mapping but was not built with a template, that the
1747+
# additional_mapping respects the existing mapping for validation.
1748+
1749+
- skip:
1750+
features:
1751+
- headers
1752+
- allowed_warnings
1753+
1754+
# A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
1755+
# because this test is making sure we get correct behavior when an index matches *no* template:
1756+
- do:
1757+
indices.delete_template:
1758+
name: '*'
1759+
ignore: 404
1760+
1761+
# We create the index no-template-index with an implicit mapping that has a foo field with type long:
1762+
- do:
1763+
bulk:
1764+
refresh: true
1765+
body:
1766+
- '{"index": {"_index": "no-template-index"}}'
1767+
- '{"foo": 3}'
1768+
1769+
# Now we make sure that the existing mapping is taken into account when we simulate with a mapping_addition. Since
1770+
# the pre-existing mapping has foo mapped as a long, this ought to fail with a document_parsing_exception because
1771+
# we are attempting to write a boolean foo.
1772+
- do:
1773+
headers:
1774+
Content-Type: application/json
1775+
simulate.ingest:
1776+
index: no-template-index
1777+
body: >
1778+
{
1779+
"docs": [
1780+
{
1781+
"_id": "test-id",
1782+
"_index": "no-template-index",
1783+
"_source": {
1784+
"@timestamp": "2025-07-25T09:06:06.929Z",
1785+
"is_valid": true,
1786+
"foo": true
1787+
}
1788+
}
1789+
],
1790+
"mapping_addition": {
1791+
"properties": {
1792+
"is_valid": {
1793+
"type": "boolean"
1794+
}
1795+
}
1796+
}
1797+
}
1798+
- length: { docs: 1 }
1799+
- match: { docs.0.doc._index: "no-template-index" }
1800+
- match: { docs.0.doc._source.foo: true }
1801+
- match: { docs.0.doc._source.is_valid: true }
1802+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1803+
1804+
# Now we add a template for this index.
1805+
- do:
1806+
indices.put_template:
1807+
name: my-template-1
1808+
body:
1809+
index_patterns: no-template-index
1810+
mappings:
1811+
properties:
1812+
foo:
1813+
type: boolean
1814+
1815+
# And we still expect the index's mapping to be used rather than the template:
1816+
- do:
1817+
headers:
1818+
Content-Type: application/json
1819+
simulate.ingest:
1820+
index: no-template-index
1821+
body: >
1822+
{
1823+
"docs": [
1824+
{
1825+
"_id": "test-id",
1826+
"_index": "no-template-index",
1827+
"_source": {
1828+
"@timestamp": "2025-07-25T09:06:06.929Z",
1829+
"is_valid": true,
1830+
"foo": true
1831+
}
1832+
}
1833+
],
1834+
"mapping_addition": {
1835+
"properties": {
1836+
"is_valid": {
1837+
"type": "boolean"
1838+
}
1839+
}
1840+
}
1841+
}
1842+
- length: { docs: 1 }
1843+
- match: { docs.0.doc._index: "no-template-index" }
1844+
- match: { docs.0.doc._source.foo: true }
1845+
- match: { docs.0.doc._source.is_valid: true }
1846+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1847+
1848+
---
1849+
"Test ingest simulate with mapping addition for data streams when write index has different mapping":
1850+
# In this test, we make sure that when a data stream's write index has a mapping that is different from the mapping
1851+
# in its template, and a mapping_override is given, then the mapping_override is applied to the mapping of the write
1852+
# index rather than the mapping of the template.
1853+
1854+
- skip:
1855+
features:
1856+
- headers
1857+
- allowed_warnings
1858+
1859+
- do:
1860+
cluster.put_component_template:
1861+
name: mappings_template
1862+
body:
1863+
template:
1864+
mappings:
1865+
dynamic: strict
1866+
properties:
1867+
foo:
1868+
type: boolean
1869+
bar:
1870+
type: boolean
1871+
1872+
- do:
1873+
allowed_warnings:
1874+
- "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation"
1875+
indices.put_index_template:
1876+
name: my-template-1
1877+
body:
1878+
index_patterns: [simple-data-stream1]
1879+
composed_of:
1880+
- mappings_template
1881+
data_stream: {}
1882+
1883+
- do:
1884+
indices.create_data_stream:
1885+
name: simple-data-stream1
1886+
- is_true: acknowledged
1887+
1888+
- do:
1889+
cluster.health:
1890+
wait_for_status: yellow
1891+
1892+
# Now that the data stream exists, we change the template to remove the mapping for bar. The write index still has the
1893+
# old mapping.
1894+
- do:
1895+
cluster.put_component_template:
1896+
name: mappings_template
1897+
body:
1898+
template:
1899+
mappings:
1900+
properties:
1901+
foo:
1902+
type: boolean
1903+
1904+
# We expect the mapping_addition to be added to the mapping of the write index, which has a boolean bar field. So this
1905+
# simulate ingest ought to fail.
1906+
- do:
1907+
headers:
1908+
Content-Type: application/json
1909+
simulate.ingest:
1910+
index: simple-data-stream1
1911+
body: >
1912+
{
1913+
"docs": [
1914+
{
1915+
"_id": "asdf",
1916+
"_source": {
1917+
"@timestamp": 1234,
1918+
"bar": "baz"
1919+
}
1920+
}
1921+
],
1922+
"mapping_addition": {
1923+
"properties": {
1924+
"baz": {
1925+
"type": "keyword"
1926+
}
1927+
}
1928+
}
1929+
}
1930+
- length: { docs: 1 }
1931+
- match: { docs.0.doc._index: "simple-data-stream1" }
1932+
- match: { docs.0.doc._source.bar: "baz" }
1933+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1934+
17441935
---
17451936
"Simulate ingest with data stream with mapping overrides":
17461937
- skip:

‎server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java‎

Lines changed: 20 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import java.util.HashMap;
7070
import java.util.List;
7171
import java.util.Map;
72+
import java.util.Optional;
7273
import java.util.Set;
7374
import java.util.concurrent.Executor;
7475

@@ -198,32 +199,15 @@ private Tuple<Collection<String>, Exception> validateMappings(
198199
Collection<String> ignoredFields = List.of();
199200
IndexAbstraction indexAbstraction = project.getIndicesLookup().get(request.index());
200201
try {
201-
if (indexAbstraction != null
202-
&& componentTemplateSubstitutions.isEmpty()
203-
&& indexTemplateSubstitutions.isEmpty()
204-
&& mappingAddition.isEmpty()) {
202+
if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) {
205203
/*
206-
* In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService
207-
* to do the mapping validation, using all the existing logic for validation.
204+
* In this case the index exists and we don't have any template overrides. So we can just merge the mappingAddition (which
205+
* might not exist) into the existing index mapping.
208206
*/
209207
IndexMetadata imd = project.getIndexSafe(indexAbstraction.getWriteIndex(request, project));
210-
indicesService.withTempIndexService(imd, indexService -> {
211-
indexService.mapperService().updateMapping(null, imd);
212-
return IndexShard.prepareIndex(
213-
indexService.mapperService(),
214-
sourceToParse,
215-
SequenceNumbers.UNASSIGNED_SEQ_NO,
216-
-1,
217-
-1,
218-
VersionType.INTERNAL,
219-
Engine.Operation.Origin.PRIMARY,
220-
Long.MIN_VALUE,
221-
false,
222-
request.ifSeqNo(),
223-
request.ifPrimaryTerm(),
224-
0
225-
);
226-
});
208+
CompressedXContent mappings = Optional.ofNullable(imd.mapping()).map(MappingMetadata::source).orElse(null);
209+
CompressedXContent mergedMappings = mappingAddition == null ? null : mergeMappings(mappings, mappingAddition);
210+
ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, mergedMappings, request, sourceToParse);
227211
} else {
228212
/*
229213
* The index did not exist, or we have component template substitutions, so we put together the mappings from existing
@@ -297,15 +281,6 @@ private Tuple<Collection<String>, Exception> validateMappings(
297281
);
298282
final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
299283
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
300-
} else if (indexAbstraction != null && mappingAddition.isEmpty() == false) {
301-
/*
302-
* The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we
303-
* merge in the mapping addition if it exists, and validate.
304-
*/
305-
MappingMetadata mappingFromIndex = project.index(indexAbstraction.getName()).mapping();
306-
CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source();
307-
CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition);
308-
ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
309284
} else {
310285
/*
311286
* The index matched no templates and had no mapping of its own. If there were component template substitutions
@@ -333,9 +308,6 @@ private Collection<String> validateUpdatedMappings(
333308
IndexRequest request,
334309
SourceToParse sourceToParse
335310
) throws IOException {
336-
if (updatedMappings == null) {
337-
return List.of(); // no validation to do
338-
}
339311
Settings dummySettings = Settings.builder()
340312
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
341313
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
@@ -347,8 +319,20 @@ private Collection<String> validateUpdatedMappings(
347319
originalIndexMetadataBuilder.putMapping(new MappingMetadata(originalMappings));
348320
}
349321
final IndexMetadata originalIndexMetadata = originalIndexMetadataBuilder.build();
322+
return validateUpdatedMappingsFromIndexMetadata(originalIndexMetadata, updatedMappings, request, sourceToParse);
323+
}
324+
325+
private Collection<String> validateUpdatedMappingsFromIndexMetadata(
326+
IndexMetadata originalIndexMetadata,
327+
@Nullable CompressedXContent updatedMappings,
328+
IndexRequest request,
329+
SourceToParse sourceToParse
330+
) throws IOException {
331+
if (updatedMappings == null) {
332+
return List.of(); // no validation to do
333+
}
350334
final IndexMetadata updatedIndexMetadata = IndexMetadata.builder(request.index())
351-
.settings(dummySettings)
335+
.settings(originalIndexMetadata.getSettings())
352336
.putMapping(new MappingMetadata(updatedMappings))
353337
.build();
354338
Engine.Index result = indicesService.withTempIndexService(originalIndexMetadata, indexService -> {

‎x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.elasticsearch.xcontent.XContentBuilder;
2828
import org.elasticsearch.xcontent.json.JsonXContent;
2929
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
30-
import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase;
3130
import org.junit.Before;
3231
import org.junit.ClassRule;
3332

@@ -40,6 +39,7 @@
4039
import static org.elasticsearch.test.ListMatcher.matchesList;
4140
import static org.elasticsearch.test.MapMatcher.assertMap;
4241
import static org.elasticsearch.test.MapMatcher.matchesMap;
42+
import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.hasCapabilities;
4343
import static org.hamcrest.Matchers.containsString;
4444
import static org.hamcrest.Matchers.equalTo;
4545
import static org.hamcrest.Matchers.is;
@@ -594,7 +594,7 @@ record Listen(long timestamp, String songId, double duration) {
594594
public void testLookupJoinIndexAllowed() throws Exception {
595595
assumeTrue(
596596
"Requires LOOKUP JOIN capability",
597-
EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
597+
hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
598598
);
599599

600600
Response resp = runESQLCommand(
@@ -685,7 +685,7 @@ public void testLookupJoinIndexAllowed() throws Exception {
685685
public void testLookupJoinDocLevelSecurity() throws Exception {
686686
assumeTrue(
687687
"Requires LOOKUP JOIN capability",
688-
EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
688+
hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
689689
);
690690

691691
Response resp = runESQLCommand("dls_user", "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-user2 ON value | KEEP x, org");
@@ -734,7 +734,7 @@ public void testLookupJoinDocLevelSecurity() throws Exception {
734734
public void testLookupJoinFieldLevelSecurity() throws Exception {
735735
assumeTrue(
736736
"Requires LOOKUP JOIN capability",
737-
EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
737+
hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
738738
);
739739

740740
Response resp = runESQLCommand("fls_user2", "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-user2 ON value");
@@ -792,7 +792,7 @@ public void testLookupJoinFieldLevelSecurity() throws Exception {
792792
public void testLookupJoinFieldLevelSecurityOnAlias() throws Exception {
793793
assumeTrue(
794794
"Requires LOOKUP JOIN capability",
795-
EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
795+
hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
796796
);
797797

798798
Response resp = runESQLCommand("fls_user2_alias", "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-second-alias ON value");
@@ -850,7 +850,7 @@ public void testLookupJoinFieldLevelSecurityOnAlias() throws Exception {
850850
public void testLookupJoinIndexForbidden() throws Exception {
851851
assumeTrue(
852852
"Requires LOOKUP JOIN capability",
853-
EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
853+
hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
854854
);
855855

856856
var resp = expectThrows(

0 commit comments

Comments
 (0)