Skip to content

Commit ffc15a2

Browse files
prdoyleprdoyle
authored
Merge branch 'main' into file-prefixes
2 parents c8299cd + 954db37 commit ffc15a2

File tree

40 files changed

+1481
-1490
lines changed

40 files changed

+1481
-1490
lines changed

‎docs/changelog/120807.yaml‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎muted-tests.yml‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -209,8 +209,6 @@ tests:
209209
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncEnrichStopIT
210210
method: testEnrichAfterStop
211211
issue: https://github.com/elastic/elasticsearch/issues/120757
212-
- class: org.elasticsearch.search.fieldcaps.FieldCapabilitiesIT
213-
issue: https://github.com/elastic/elasticsearch/issues/120772
214212
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
215213
method: test {p0=ml/3rd_party_deployment/Test start deployment fails while model download in progress}
216214
issue: https://github.com/elastic/elasticsearch/issues/120810
@@ -270,9 +268,6 @@ tests:
270268
- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
271269
method: test {p0=nodes.stats/11_indices_metrics/indices mappings exact count test for indices level}
272270
issue: https://github.com/elastic/elasticsearch/issues/120950
273-
- class: org.elasticsearch.xpack.shutdown.AllocationFailuresResetOnShutdownIT
274-
method: testResetAllocationFailuresOnNodeShutdown
275-
issue: https://github.com/elastic/elasticsearch/issues/121129
276271
- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
277272
method: testActivateProfileForJWT
278273
issue: https://github.com/elastic/elasticsearch/issues/120983
@@ -431,6 +426,21 @@ tests:
431426
- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryUnavailableRemotesIT
432427
method: testRemoteOnlyCCSAgainstDisconnectedRemoteWithSkipUnavailableTrue
433428
issue: https://github.com/elastic/elasticsearch/issues/121578
429+
- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
430+
method: testTasks
431+
issue: https://github.com/elastic/elasticsearch/issues/121626
432+
- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
433+
method: testCloseSkipUnavailable
434+
issue: https://github.com/elastic/elasticsearch/issues/121627
435+
- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
436+
method: testCancel
437+
issue: https://github.com/elastic/elasticsearch/issues/121632
438+
- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
439+
method: testCancelSkipUnavailable
440+
issue: https://github.com/elastic/elasticsearch/issues/121631
441+
- class: org.elasticsearch.upgrades.UpgradeClusterClientYamlTestSuiteIT
442+
method: test {p0=mixed_cluster/110_enrich/Enrich stats query smoke test for mixed cluster}
443+
issue: https://github.com/elastic/elasticsearch/issues/121642
434444

435445
# Examples:
436446
#

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
import org.apache.http.entity.ContentType;
1313
import org.apache.http.entity.InputStreamEntity;
1414
import org.elasticsearch.client.Request;
15+
import org.elasticsearch.client.RequestOptions;
1516
import org.elasticsearch.client.ResponseException;
17+
import org.elasticsearch.client.WarningsHandler;
1618
import org.elasticsearch.cluster.block.ClusterBlock;
1719
import org.elasticsearch.cluster.metadata.IndexMetadata;
1820
import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
@@ -27,6 +29,7 @@
2729
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
2830
import org.elasticsearch.test.cluster.util.Version;
2931
import org.elasticsearch.test.rest.ESRestTestCase;
32+
import org.elasticsearch.test.rest.ObjectPath;
3033
import org.elasticsearch.xcontent.XContentType;
3134
import org.hamcrest.Matcher;
3235
import org.junit.After;
@@ -161,8 +164,21 @@ protected static boolean isFullyUpgradedTo(Version version) throws Exception {
161164
}
162165

163166
protected static Version indexVersion(String indexName) throws Exception {
164-
var response = assertOK(client().performRequest(new Request("GET", "/" + indexName + "/_settings")));
165-
int id = Integer.parseInt(createFromResponse(response).evaluate(indexName + ".settings.index.version.created"));
167+
return indexVersion(indexName, false);
168+
}
169+
170+
protected static Version indexVersion(String indexName, boolean ignoreWarnings) throws Exception {
171+
Request request = new Request("GET", "/" + indexName + "/_settings");
172+
request.addParameter("flat_settings", "true");
173+
if (ignoreWarnings) {
174+
RequestOptions.Builder options = request.getOptions().toBuilder();
175+
options.setWarningsHandler(WarningsHandler.PERMISSIVE);
176+
request.setOptions(options);
177+
}
178+
var response = assertOK(client().performRequest(request));
179+
ObjectPath fromResponse = createFromResponse(response);
180+
Map<String, Object> settings = fromResponse.evaluateExact(indexName, "settings");
181+
int id = Integer.parseInt((String) settings.get("index.version.created"));
166182
return new Version((byte) ((id / 1000000) % 100), (byte) ((id / 10000) % 100), (byte) ((id / 100) % 100));
167183
}
168184

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSystemIndexCompatibilityIT.java‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.lucene;
11+
12+
import org.elasticsearch.client.Request;
13+
import org.elasticsearch.client.RequestOptions;
14+
import org.elasticsearch.client.Response;
15+
import org.elasticsearch.client.RestClient;
16+
import org.elasticsearch.client.WarningsHandler;
17+
import org.elasticsearch.cluster.metadata.IndexMetadata;
18+
import org.elasticsearch.common.settings.Settings;
19+
import org.elasticsearch.test.cluster.util.Version;
20+
import org.elasticsearch.test.rest.ObjectPath;
21+
22+
import java.io.IOException;
23+
import java.util.HashMap;
24+
import java.util.Map;
25+
26+
import static org.hamcrest.Matchers.equalTo;
27+
28+
public class FullClusterRestartSystemIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase {
29+
30+
static {
31+
clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
32+
}
33+
34+
public FullClusterRestartSystemIndexCompatibilityIT(Version version) {
35+
super(version);
36+
}
37+
38+
// we need a place to store async_search ids across cluster restarts
39+
private static Map<String, String> async_search_ids = new HashMap<>(3);
40+
41+
/**
42+
* 1. creates an index on N-2 and performs async_search on it that is kept in system index
43+
* 2. After update to N-1 (latest) perform a system index migration step, also write block the index
44+
* 3. on N, check that async search results are still retrievable and we can write to the system index
45+
*/
46+
public void testAsyncSearchIndexMigration() throws Exception {
47+
final String index = suffix("index");
48+
final String asyncSearchIndex = ".async-search";
49+
final int numDocs = 2431;
50+
51+
final Request asyncSearchRequest = new Request("POST", "/" + index + "/_async_search?size=100&keep_on_completion=true");
52+
53+
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
54+
createIndex(
55+
client(),
56+
index,
57+
Settings.builder()
58+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
59+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
60+
.build()
61+
);
62+
indexDocs(index, numDocs);
63+
ensureGreen(index);
64+
65+
assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
66+
String asyncId = searchAsyncAndStoreId(asyncSearchRequest, "n-2_id");
67+
ensureGreen(asyncSearchIndex);
68+
69+
assertAsyncSearchHitCount(asyncId, numDocs);
70+
assertBusy(() -> assertDocCountNoWarnings(client(), asyncSearchIndex, 1));
71+
assertThat(indexVersion(asyncSearchIndex, true), equalTo(VERSION_MINUS_2));
72+
return;
73+
}
74+
75+
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
76+
// check .async-search index is readable
77+
assertThat(indexVersion(asyncSearchIndex, true), equalTo(VERSION_MINUS_2));
78+
assertAsyncSearchHitCount(async_search_ids.get("n-2_id"), numDocs);
79+
80+
// migrate system indices
81+
Request migrateRequest = new Request("POST", "/_migration/system_features");
82+
assertThat(
83+
ObjectPath.createFromResponse(client().performRequest(migrateRequest)).evaluate("features.0.feature_name"),
84+
equalTo("async_search")
85+
);
86+
assertBusy(() -> {
87+
Request checkMigrateProgress = new Request("GET", "/_migration/system_features");
88+
Response resp = null;
89+
try {
90+
assertFalse(
91+
ObjectPath.createFromResponse(client().performRequest(checkMigrateProgress))
92+
.evaluate("migration_status")
93+
.equals("IN_PROGRESS")
94+
);
95+
} catch (IOException e) {
96+
throw new AssertionError("System feature migration failed", e);
97+
}
98+
});
99+
100+
// check search results from n-2 search are still readable
101+
assertAsyncSearchHitCount(async_search_ids.get("n-2_id"), numDocs);
102+
103+
// perform new async search and check its readable
104+
String asyncId = searchAsyncAndStoreId(asyncSearchRequest, "n-1_id");
105+
assertAsyncSearchHitCount(asyncId, numDocs);
106+
assertBusy(() -> assertDocCountNoWarnings(client(), asyncSearchIndex, 2));
107+
108+
// in order to move to current version we need write block for n-2 index
109+
addIndexBlock(index, IndexMetadata.APIBlock.WRITE);
110+
}
111+
112+
if (isFullyUpgradedTo(VERSION_CURRENT)) {
113+
assertThat(indexVersion(index, true), equalTo(VERSION_MINUS_2));
114+
assertAsyncSearchHitCount(async_search_ids.get("n-2_id"), numDocs);
115+
assertAsyncSearchHitCount(async_search_ids.get("n-1_id"), numDocs);
116+
117+
// check system index is still writeable
118+
String asyncId = searchAsyncAndStoreId(asyncSearchRequest, "n_id");
119+
assertAsyncSearchHitCount(asyncId, numDocs);
120+
assertBusy(() -> assertDocCountNoWarnings(client(), asyncSearchIndex, 3));
121+
}
122+
123+
}
124+
125+
private static String searchAsyncAndStoreId(Request asyncSearchRequest, String asyncIdName) throws IOException {
126+
ObjectPath resp = ObjectPath.createFromResponse(client().performRequest(asyncSearchRequest));
127+
String asyncId = resp.evaluate("id");
128+
assertNotNull(asyncId);
129+
async_search_ids.put(asyncIdName, asyncId);
130+
return asyncId;
131+
}
132+
133+
private static void assertAsyncSearchHitCount(String asyncId, int numDocs) throws IOException {
134+
var asyncGet = new Request("GET", "/_async_search/" + asyncId);
135+
ObjectPath resp = ObjectPath.createFromResponse(client().performRequest(asyncGet));
136+
assertEquals(Integer.valueOf(numDocs), resp.evaluate("response.hits.total.value"));
137+
}
138+
139+
/**
140+
* Assert that the index in question has the given number of documents present
141+
*/
142+
private static void assertDocCountNoWarnings(RestClient client, String indexName, long docCount) throws IOException {
143+
Request countReq = new Request("GET", "/" + indexName + "/_count");
144+
RequestOptions.Builder options = countReq.getOptions().toBuilder();
145+
options.setWarningsHandler(WarningsHandler.PERMISSIVE);
146+
countReq.setOptions(options);
147+
ObjectPath resp = ObjectPath.createFromResponse(client.performRequest(countReq));
148+
assertEquals(
149+
"expected " + docCount + " documents but it was a different number",
150+
docCount,
151+
Long.parseLong(resp.evaluate("count").toString())
152+
);
153+
}
154+
}

‎qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
import java.util.List;
1919
import java.util.Map;
2020

21-
import static org.hamcrest.Matchers.containsString;
2221
import static org.hamcrest.Matchers.equalTo;
2322

2423
public class SourceModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
@@ -83,20 +82,10 @@ public void testConfigureStoredSourceWhenIndexIsCreatedLegacy() throws IOExcepti
8382
private void assertDeprecationWarningForTemplate(String templateName) throws IOException {
8483
var request = new Request("GET", "/_migration/deprecations");
8584
var response = entityAsMap(client().performRequest(request));
86-
if (response.containsKey("templates")) {
87-
// Check the newer version of the deprecation API that contains the templates section
88-
Map<?, ?> issuesByTemplate = (Map<?, ?>) response.get("templates");
89-
assertThat(issuesByTemplate.containsKey(templateName), equalTo(true));
90-
var templateIssues = (List<?>) issuesByTemplate.get(templateName);
91-
assertThat(((Map<?, ?>) templateIssues.getFirst()).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
92-
} else {
93-
// Bwc version with 8.18 until https://github.com/elastic/elasticsearch/pull/120505/ gets backported, clean up after backport
94-
var nodeSettings = (Map<?, ?>) ((List<?>) response.get("node_settings")).getFirst();
95-
assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
96-
assertThat(
97-
(String) nodeSettings.get("details"),
98-
containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]")
99-
);
100-
}
85+
assertThat(response.containsKey("templates"), equalTo(true));
86+
Map<?, ?> issuesByTemplate = (Map<?, ?>) response.get("templates");
87+
assertThat(issuesByTemplate.containsKey(templateName), equalTo(true));
88+
var templateIssues = (List<?>) issuesByTemplate.get(templateName);
89+
assertThat(((Map<?, ?>) templateIssues.getFirst()).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
10190
}
10291
}

‎server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.client.Cancellable;
2929
import org.elasticsearch.client.Request;
3030
import org.elasticsearch.client.Response;
31+
import org.elasticsearch.cluster.ClusterState;
3132
import org.elasticsearch.cluster.metadata.IndexMetadata;
3233
import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
3334
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
@@ -73,6 +74,7 @@
7374
import java.util.Collection;
7475
import java.util.Collections;
7576
import java.util.HashMap;
77+
import java.util.HashSet;
7678
import java.util.List;
7779
import java.util.Map;
7880
import java.util.concurrent.CancellationException;
@@ -591,21 +593,31 @@ public void testNoActiveCopy() throws Exception {
591593

592594
private void moveOrCloseShardsOnNodes(String nodeName) throws Exception {
593595
final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeName);
596+
final ClusterState clusterState = clusterService().state();
594597
for (IndexService indexService : indicesService) {
595598
for (IndexShard indexShard : indexService) {
596599
if (randomBoolean()) {
597600
closeShardNoCheck(indexShard, randomBoolean());
598601
} else if (randomBoolean()) {
599602
final ShardId shardId = indexShard.shardId();
600-
603+
final var assignedNodes = new HashSet<>();
604+
clusterState.routingTable().shardRoutingTable(shardId).allShards().forEach(shr -> {
605+
if (shr.currentNodeId() != null) {
606+
assignedNodes.add(shr.currentNodeId());
607+
}
608+
if (shr.relocatingNodeId() != null) {
609+
assignedNodes.add(shr.relocatingNodeId());
610+
}
611+
});
601612
final var targetNodes = new ArrayList<String>();
602613
for (final var targetIndicesService : internalCluster().getInstances(IndicesService.class)) {
603614
final var targetNode = targetIndicesService.clusterService().localNode();
604-
if (targetNode.canContainData() && targetIndicesService.getShardOrNull(shardId) == null) {
615+
if (targetNode.canContainData()
616+
&& targetIndicesService.getShardOrNull(shardId) == null
617+
&& assignedNodes.contains(targetNode.getId()) == false) {
605618
targetNodes.add(targetNode.getId());
606619
}
607620
}
608-
609621
if (targetNodes.isEmpty()) {
610622
continue;
611623
}

0 commit comments

Comments
 (0)