Merge branch 'main' into es-11576-multi-project-aware

Author: JeremyDahlgren

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -11,7 +11,7 @@ tags:
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.5"
+  BASE_TAG: "9.6"
 # Docker image labels
 labels:
   org.opencontainers.image.title: "elasticsearch"

docs/changelog/132101.yaml

@@ -0,0 +1,6 @@
+pr: 132101
+summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
+  given
+area: Ingest Node
+type: bug
+issues: []

muted-tests.yml

@@ -614,7 +614,39 @@ tests:
 - class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
   method: "testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_disk #2}"
   issue: https://github.com/elastic/elasticsearch/issues/132189
-
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int4_hnsw updateType=bbq_hnsw}
+  issue: https://github.com/elastic/elasticsearch/issues/132211
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: "testDenseVectorMappingUpdate {initialType=int8_hnsw updateType=bbq_disk #2}"
+  issue: https://github.com/elastic/elasticsearch/issues/132213
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int8_hnsw updateType=bbq_disk}
+  issue: https://github.com/elastic/elasticsearch/issues/132214
+- class: org.elasticsearch.xpack.security.authz.store.CompositeRolesStoreTests
+  method: testCacheCleanupOnProjectDeletion
+  issue: https://github.com/elastic/elasticsearch/issues/132218
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_hnsw}
+  issue: https://github.com/elastic/elasticsearch/issues/132219
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=hnsw updateType=bbq_disk}
+  issue: https://github.com/elastic/elasticsearch/issues/132220
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_disk}
+  issue: https://github.com/elastic/elasticsearch/issues/132221
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=flat updateType=bbq_disk}
+  issue: https://github.com/elastic/elasticsearch/issues/132222
+- class: org.elasticsearch.test.rest.yaml.MDPYamlTestSuiteIT
+  method: test {yaml=mdp/10_basic/Index using shared data path}
+  issue: https://github.com/elastic/elasticsearch/issues/132223
+- class: org.elasticsearch.xpack.logsdb.qa.StandardVersusStandardReindexedIntoLogsDbChallengeRestIT
+  method: testTermsQuery
+  issue: https://github.com/elastic/elasticsearch/issues/132225
+- class: org.elasticsearch.xpack.logsdb.qa.StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT
+  method: testTermsQuery
+  issue: https://github.com/elastic/elasticsearch/issues/132226
 
 # Examples:
 #

qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml

@@ -1740,3 +1740,194 @@ setup:
   - match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" }
   - match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] }
   - not_exists: docs.0.doc.error
+
+---
+"Test mapping addition correctly respects mapping of indices without templates":
+  # In this test, we make sure that when we have an index that has mapping but was not built with a template, that the
+  # additional_mapping respects the existing mapping for validation.
+
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+
+  # A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
+  # because this test is making sure we get correct behavior when an index matches *no* template:
+  - do:
+      indices.delete_template:
+        name:   '*'
+        ignore: 404
+
+  # We create the index no-template-index with an implicit mapping that has a foo field with type long:
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "no-template-index"}}'
+          - '{"foo": 3}'
+
+  # Now we make sure that the existing mapping is taken into account when we simulate with a mapping_addition. Since
+  # the pre-existing mapping has foo mapped as a long, this ought to fail with a document_parsing_exception because
+  # we are attempting to write a boolean foo.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: no-template-index
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "test-id",
+                "_index": "no-template-index",
+                "_source": {
+                  "@timestamp": "2025-07-25T09:06:06.929Z",
+                  "is_valid": true,
+                  "foo": true
+                }
+              }
+            ],
+            "mapping_addition": {
+              "properties": {
+                "is_valid": {
+                  "type": "boolean"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "no-template-index" }
+  - match: { docs.0.doc._source.foo: true }
+  - match: { docs.0.doc._source.is_valid: true }
+  - match: { docs.0.doc.error.type: "document_parsing_exception" }
+
+  # Now we add a template for this index.
+  - do:
+      indices.put_template:
+        name: my-template-1
+        body:
+          index_patterns: no-template-index
+          mappings:
+            properties:
+              foo:
+                type: boolean
+
+  # And we still expect the index's mapping to be used rather than the template:
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: no-template-index
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "test-id",
+                "_index": "no-template-index",
+                "_source": {
+                  "@timestamp": "2025-07-25T09:06:06.929Z",
+                  "is_valid": true,
+                  "foo": true
+                }
+              }
+            ],
+            "mapping_addition": {
+              "properties": {
+                "is_valid": {
+                  "type": "boolean"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "no-template-index" }
+  - match: { docs.0.doc._source.foo: true }
+  - match: { docs.0.doc._source.is_valid: true }
+  - match: { docs.0.doc.error.type: "document_parsing_exception" }
+
+---
+"Test ingest simulate with mapping addition for data streams when write index has different mapping":
+  # In this test, we make sure that when a data stream's write index has a mapping that is different from the mapping
+  # in its template, and a mapping_override is given, then the mapping_override is applied to the mapping of the write
+  # index rather than the mapping of the template.
+
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+
+  - do:
+      cluster.put_component_template:
+        name: mappings_template
+        body:
+          template:
+            mappings:
+              dynamic: strict
+              properties:
+                foo:
+                  type: boolean
+                bar:
+                  type: boolean
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template-1
+        body:
+          index_patterns: [simple-data-stream1]
+          composed_of:
+            - mappings_template
+          data_stream: {}
+
+  - do:
+      indices.create_data_stream:
+        name: simple-data-stream1
+  - is_true: acknowledged
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  # Now that the data stream exists, we change the template to remove the mapping for bar. The write index still has the
+  # old mapping.
+  - do:
+      cluster.put_component_template:
+        name: mappings_template
+        body:
+          template:
+            mappings:
+              properties:
+                foo:
+                  type: boolean
+
+  # We expect the mapping_addition to be added to the mapping of the write index, which has a boolean bar field. So this
+  # simulate ingest ought to fail.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: simple-data-stream1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "@timestamp": 1234,
+                  "bar": "baz"
+                }
+              }
+            ],
+            "mapping_addition": {
+              "properties": {
+                "baz": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "simple-data-stream1" }
+  - match: { docs.0.doc._source.bar: "baz" }
+  - match: { docs.0.doc.error.type: "document_parsing_exception" }