Skip to content

Commit 62956af

Browse files
julian-elasticjulian-elastic
committed
Merge branch 'main' into lookupJoin
2 parents 806933f + f7e3fe1 commit 62956af

File tree

37 files changed

+874
-338
lines changed

37 files changed

+874
-338
lines changed

‎distribution/docker/src/docker/iron_bank/hardening_manifest.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ tags:
1111
# Build args passed to Dockerfile ARGs
1212
args:
1313
BASE_IMAGE: "redhat/ubi/ubi9"
14-
BASE_TAG: "9.5"
14+
BASE_TAG: "9.6"
1515
# Docker image labels
1616
labels:
1717
org.opencontainers.image.title: "elasticsearch"

‎docs/changelog/131937.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 131937
2+
summary: Fix race condition in `RemoteClusterService.collectNodes()`
3+
area: Distributed
4+
type: bug
5+
issues: []

‎docs/changelog/132101.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132101
2+
summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
3+
given
4+
area: Ingest Node
5+
type: bug
6+
issues: []

‎gradle/verification-metadata.xml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2150,24 +2150,24 @@
21502150
<sha256 value="5c8551990307a032336d98ddaed549a39a689f07d4d4c6b950601bf22b3d6a1b" origin="Generated by Gradle"/>
21512151
</artifact>
21522152
</component>
2153-
<component group="org.apache.arrow" name="arrow-format" version="16.1.0">
2154-
<artifact name="arrow-format-16.1.0.jar">
2155-
<sha256 value="ad97e0fc72e193b1de3cbce4818d1ff16e81673fd523d001e8d2774bde40ee6c" origin="Generated by Gradle"/>
2153+
<component group="org.apache.arrow" name="arrow-format" version="18.3.0">
2154+
<artifact name="arrow-format-18.3.0.jar">
2155+
<sha256 value="728297d7757d192053ce071cc59e76c8a03ff4f5e430177fa97e96b29dd2de1b" origin="Generated by Gradle"/>
21562156
</artifact>
21572157
</component>
2158-
<component group="org.apache.arrow" name="arrow-memory-core" version="16.1.0">
2159-
<artifact name="arrow-memory-core-16.1.0.jar">
2160-
<sha256 value="da7af1a1a899bd5a1b6c71284243b9f3c0e1098f0cb10cd7be4b8b455ced79dd" origin="Generated by Gradle"/>
2158+
<component group="org.apache.arrow" name="arrow-memory-core" version="18.3.0">
2159+
<artifact name="arrow-memory-core-18.3.0.jar">
2160+
<sha256 value="32ed0719bf2ba42becc3a88c95722851cf1bb88b37d06c95654a1152bed6ef2e" origin="Generated by Gradle"/>
21612161
</artifact>
21622162
</component>
2163-
<component group="org.apache.arrow" name="arrow-memory-unsafe" version="16.1.0">
2164-
<artifact name="arrow-memory-unsafe-16.1.0.jar">
2165-
<sha256 value="6534eded25f2c30593416a294c1047f0b017baa9906d98f6f3270737b076c745" origin="Generated by Gradle"/>
2163+
<component group="org.apache.arrow" name="arrow-memory-unsafe" version="18.3.0">
2164+
<artifact name="arrow-memory-unsafe-18.3.0.jar">
2165+
<sha256 value="e9652f322d7be306c400ea0d775b3114a11b48d2afe23e8b8566f13c469d5483" origin="Generated by Gradle"/>
21662166
</artifact>
21672167
</component>
2168-
<component group="org.apache.arrow" name="arrow-vector" version="16.1.0">
2169-
<artifact name="arrow-vector-16.1.0.jar">
2170-
<sha256 value="c5837b3aa24dfd93759f57bc5759b9a8fbb5bf3912d55994d70cabb904436aab" origin="Generated by Gradle"/>
2168+
<component group="org.apache.arrow" name="arrow-vector" version="18.3.0">
2169+
<artifact name="arrow-vector-18.3.0.jar">
2170+
<sha256 value="b37eda92daccaffc12abf5ed425db1d0bdb3edea150a6ca856f1cf4292442299" origin="Generated by Gradle"/>
21712171
</artifact>
21722172
</component>
21732173
<component group="org.apache.avro" name="avro" version="1.7.4">

‎muted-tests.yml‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -500,9 +500,6 @@ tests:
500500
- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
501501
method: test {p0=search/600_flattened_ignore_above/flattened ignore_above multi-value field}
502502
issue: https://github.com/elastic/elasticsearch/issues/131967
503-
- class: org.elasticsearch.search.routing.SearchReplicaSelectionIT
504-
method: testNodeSelection
505-
issue: https://github.com/elastic/elasticsearch/issues/132017
506503
- class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS1EnrichUnavailableRemotesIT
507504
method: testEsqlEnrichWithSkipUnavailable
508505
issue: https://github.com/elastic/elasticsearch/issues/132078
@@ -617,6 +614,42 @@ tests:
617614
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
618615
method: "testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_disk #2}"
619616
issue: https://github.com/elastic/elasticsearch/issues/132189
617+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
618+
method: testDenseVectorMappingUpdate {initialType=int4_hnsw updateType=bbq_hnsw}
619+
issue: https://github.com/elastic/elasticsearch/issues/132211
620+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
621+
method: "testDenseVectorMappingUpdate {initialType=int8_hnsw updateType=bbq_disk #2}"
622+
issue: https://github.com/elastic/elasticsearch/issues/132213
623+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
624+
method: testDenseVectorMappingUpdate {initialType=int8_hnsw updateType=bbq_disk}
625+
issue: https://github.com/elastic/elasticsearch/issues/132214
626+
- class: org.elasticsearch.xpack.security.authz.store.CompositeRolesStoreTests
627+
method: testCacheCleanupOnProjectDeletion
628+
issue: https://github.com/elastic/elasticsearch/issues/132218
629+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
630+
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_hnsw}
631+
issue: https://github.com/elastic/elasticsearch/issues/132219
632+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
633+
method: testDenseVectorMappingUpdate {initialType=hnsw updateType=bbq_disk}
634+
issue: https://github.com/elastic/elasticsearch/issues/132220
635+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
636+
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_disk}
637+
issue: https://github.com/elastic/elasticsearch/issues/132221
638+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
639+
method: testDenseVectorMappingUpdate {initialType=flat updateType=bbq_disk}
640+
issue: https://github.com/elastic/elasticsearch/issues/132222
641+
- class: org.elasticsearch.test.rest.yaml.MDPYamlTestSuiteIT
642+
method: test {yaml=mdp/10_basic/Index using shared data path}
643+
issue: https://github.com/elastic/elasticsearch/issues/132223
644+
- class: org.elasticsearch.xpack.logsdb.qa.StandardVersusStandardReindexedIntoLogsDbChallengeRestIT
645+
method: testTermsQuery
646+
issue: https://github.com/elastic/elasticsearch/issues/132225
647+
- class: org.elasticsearch.xpack.logsdb.qa.StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT
648+
method: testTermsQuery
649+
issue: https://github.com/elastic/elasticsearch/issues/132226
650+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
651+
method: "testDenseVectorMappingUpdate {initialType=int4_hnsw updateType=bbq_disk #2}"
652+
issue: https://github.com/elastic/elasticsearch/issues/132228
620653

621654
# Examples:
622655
#

‎qa/multi-data-path/build.gradle‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apply plugin: 'elasticsearch.internal-yaml-rest-test'
2+
3+
// This subproject verifies MDP continues to work with entitlements.
4+
5+
restResources {
6+
restApi {
7+
include '_common', 'capabilities', 'index', 'indices', 'indices.create'
8+
}
9+
}
10+

‎qa/multi-data-path/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/MDPYamlTestSuiteIT.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.test.rest.yaml;
11+
12+
import com.carrotsearch.randomizedtesting.annotations.Name;
13+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
14+
15+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
16+
import org.junit.ClassRule;
17+
18+
import java.io.IOException;
19+
import java.nio.file.Files;
20+
21+
public class MDPYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
22+
23+
@ClassRule
24+
public static ElasticsearchCluster cluster = ElasticsearchCluster.local().setting("path.shared_data", tempSharedDataPath()).build();
25+
26+
public MDPYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
27+
super(testCandidate);
28+
}
29+
30+
@ParametersFactory
31+
public static Iterable<Object[]> parameters() throws Exception {
32+
return createParameters();
33+
}
34+
35+
@Override
36+
protected String getTestRestCluster() {
37+
return cluster.getHttpAddresses();
38+
}
39+
40+
private static String tempSharedDataPath() {
41+
try {
42+
return Files.createTempDirectory("shared_data").toString();
43+
} catch (IOException e) {
44+
throw new AssertionError(e);
45+
}
46+
}
47+
}

‎qa/multi-data-path/src/yamlRestTest/resources/rest-api-spec/test/mdp/10_basic.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
"Index using shared data path":
3+
4+
- requires:
5+
test_runner_features: ["warnings"]
6+
7+
- do:
8+
warnings:
9+
- "[index.data_path] setting was deprecated in Elasticsearch and will be removed in a future release. See the deprecation documentation for the next major version."
10+
indices.create:
11+
index: test_index
12+
body:
13+
settings:
14+
data_path: "test_index_data_path"
15+
16+
- do:
17+
index:
18+
index: test_index
19+
id: "1"
20+
body: { foo: bar }
21+
22+
- match: { result: created }
23+
24+
- do:
25+
index:
26+
index: test_index
27+
id: "1"
28+
body: { foo: bar }
29+
op_type: index
30+
31+
- match: { result: updated }

‎qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1740,3 +1740,194 @@ setup:
17401740
- match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" }
17411741
- match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] }
17421742
- not_exists: docs.0.doc.error
1743+
1744+
---
1745+
"Test mapping addition correctly respects mapping of indices without templates":
1746+
# In this test, we make sure that when we have an index that has mapping but was not built with a template, that the
1747+
# additional_mapping respects the existing mapping for validation.
1748+
1749+
- skip:
1750+
features:
1751+
- headers
1752+
- allowed_warnings
1753+
1754+
# A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
1755+
# because this test is making sure we get correct behavior when an index matches *no* template:
1756+
- do:
1757+
indices.delete_template:
1758+
name: '*'
1759+
ignore: 404
1760+
1761+
# We create the index no-template-index with an implicit mapping that has a foo field with type long:
1762+
- do:
1763+
bulk:
1764+
refresh: true
1765+
body:
1766+
- '{"index": {"_index": "no-template-index"}}'
1767+
- '{"foo": 3}'
1768+
1769+
# Now we make sure that the existing mapping is taken into account when we simulate with a mapping_addition. Since
1770+
# the pre-existing mapping has foo mapped as a long, this ought to fail with a document_parsing_exception because
1771+
# we are attempting to write a boolean foo.
1772+
- do:
1773+
headers:
1774+
Content-Type: application/json
1775+
simulate.ingest:
1776+
index: no-template-index
1777+
body: >
1778+
{
1779+
"docs": [
1780+
{
1781+
"_id": "test-id",
1782+
"_index": "no-template-index",
1783+
"_source": {
1784+
"@timestamp": "2025-07-25T09:06:06.929Z",
1785+
"is_valid": true,
1786+
"foo": true
1787+
}
1788+
}
1789+
],
1790+
"mapping_addition": {
1791+
"properties": {
1792+
"is_valid": {
1793+
"type": "boolean"
1794+
}
1795+
}
1796+
}
1797+
}
1798+
- length: { docs: 1 }
1799+
- match: { docs.0.doc._index: "no-template-index" }
1800+
- match: { docs.0.doc._source.foo: true }
1801+
- match: { docs.0.doc._source.is_valid: true }
1802+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1803+
1804+
# Now we add a template for this index.
1805+
- do:
1806+
indices.put_template:
1807+
name: my-template-1
1808+
body:
1809+
index_patterns: no-template-index
1810+
mappings:
1811+
properties:
1812+
foo:
1813+
type: boolean
1814+
1815+
# And we still expect the index's mapping to be used rather than the template:
1816+
- do:
1817+
headers:
1818+
Content-Type: application/json
1819+
simulate.ingest:
1820+
index: no-template-index
1821+
body: >
1822+
{
1823+
"docs": [
1824+
{
1825+
"_id": "test-id",
1826+
"_index": "no-template-index",
1827+
"_source": {
1828+
"@timestamp": "2025-07-25T09:06:06.929Z",
1829+
"is_valid": true,
1830+
"foo": true
1831+
}
1832+
}
1833+
],
1834+
"mapping_addition": {
1835+
"properties": {
1836+
"is_valid": {
1837+
"type": "boolean"
1838+
}
1839+
}
1840+
}
1841+
}
1842+
- length: { docs: 1 }
1843+
- match: { docs.0.doc._index: "no-template-index" }
1844+
- match: { docs.0.doc._source.foo: true }
1845+
- match: { docs.0.doc._source.is_valid: true }
1846+
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1847+
1848+
---
1849+
"Test ingest simulate with mapping addition for data streams when write index has different mapping":
1850+
# In this test, we make sure that when a data stream's write index has a mapping that is different from the mapping
1851+
# in its template, and a mapping_override is given, then the mapping_override is applied to the mapping of the write
1852+
# index rather than the mapping of the template.
1853+
1854+
- skip:
1855+
features:
1856+
- headers
1857+
- allowed_warnings
1858+
1859+
- do:
1860+
cluster.put_component_template:
1861+
name: mappings_template
1862+
body:
1863+
template:
1864+
mappings:
1865+
dynamic: strict
1866+
properties:
1867+
foo:
1868+
type: boolean
1869+
bar:
1870+
type: boolean
1871+
1872+
- do:
1873+
allowed_warnings:
1874+
- "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation"
1875+
indices.put_index_template:
1876+
name: my-template-1
1877+
body:
1878+
index_patterns: [simple-data-stream1]
1879+
composed_of:
1880+
- mappings_template
1881+
data_stream: {}
1882+
1883+
- do:
1884+
indices.create_data_stream:
1885+
name: simple-data-stream1
1886+
- is_true: acknowledged
1887+
1888+
- do:
1889+
cluster.health:
1890+
wait_for_status: yellow
1891+
1892+
# Now that the data stream exists, we change the template to remove the mapping for bar. The write index still has the
1893+
# old mapping.
1894+
- do:
1895+
cluster.put_component_template:
1896+
name: mappings_template
1897+
body:
1898+
template:
1899+
mappings:
1900+
properties:
1901+
foo:
1902+
type: boolean
1903+
1904+
# We expect the mapping_addition to be added to the mapping of the write index, which has a boolean bar field. So this
1905+
# simulate ingest ought to fail.
1906+
- do:
1907+
headers:
1908+
Content-Type: application/json
1909+
simulate.ingest:
1910+
index: simple-data-stream1
1911+
body: >
1912+
{
1913+
"docs": [
1914+
{
1915+
"_id": "asdf",
1916+
"_source": {
1917+
"@timestamp": 1234,
1918+
"bar": "baz"
1919+
}
1920+
}
1921+
],
1922+
"mapping_addition": {
1923+
"properties": {
1924+
"baz": {
1925+
"type": "keyword"
1926+
}
1927+
}
1928+
}
1929+
}
1930+
- length: { docs: 1 }
1931+
- match: { docs.0.doc._index: "simple-data-stream1" }
1932+
- match: { docs.0.doc._source.bar: "baz" }
1933+
- match: { docs.0.doc.error.type: "document_parsing_exception" }

0 commit comments

Comments
 (0)