Skip to content

Remove test dependencies on cluster state API master waiting #129118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 16, 2025
Merged

Remove test dependencies on cluster state API master waiting #129118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f6b9116
Remove test dependencies on cluster state API master waiting
nielsbauman Jun 7, 2025
60ff9e8
Merge branch 'main' into cs-master-dep
nielsbauman Jun 13, 2025
1b2345d
Readd roundtrip for ML tests
nielsbauman Jun 13, 2025
c702b51
Fix test
nielsbauman Jun 13, 2025
2efacee
Merge branch 'main' into cs-master-dep
nielsbauman Jun 16, 2025
9a223c6
Add comment
nielsbauman Jun 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 2 additions & 21 deletions .../internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Discovery;
import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Management;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.test.ESIntegTestCase;

import static org.hamcrest.Matchers.containsString;
Expand All @@ -28,16 +27,7 @@ public void testOneNodeShouldRunUsingPrivateIp() {

final String node1 = internalCluster().startNode(settings);
registerAzureNode(node1);
assertNotNull(
client().admin()
.cluster()
.prepareState(TEST_REQUEST_TIMEOUT)
.setMasterNodeTimeout(TimeValue.timeValueSeconds(1))
.get()
.getState()
.nodes()
.getMasterNodeId()
);
awaitMasterNode();

// We expect having 1 node as part of the cluster, let's test that
assertNumberOfNodes(1);
Expand All @@ -50,16 +40,7 @@ public void testOneNodeShouldRunUsingPublicIp() {

final String node1 = internalCluster().startNode(settings);
registerAzureNode(node1);
assertNotNull(
client().admin()
.cluster()
.prepareState(TEST_REQUEST_TIMEOUT)
.setMasterNodeTimeout(TimeValue.timeValueSeconds(1))
.get()
.getState()
.nodes()
.getMasterNodeId()
);
awaitMasterNode();

// We expect having 1 node as part of the cluster, let's test that
assertNumberOfNodes(1);
Expand Down
23 changes: 2 additions & 21 deletions ...ClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Discovery;
import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Management;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.test.ESIntegTestCase;

@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
Expand All @@ -31,30 +30,12 @@ public void testTwoNodesShouldRunUsingPrivateOrPublicIp() {
logger.info("--> start first node");
final String node1 = internalCluster().startNode(settings);
registerAzureNode(node1);
assertNotNull(
client().admin()
.cluster()
.prepareState(TEST_REQUEST_TIMEOUT)
.setMasterNodeTimeout(TimeValue.timeValueSeconds(1))
.get()
.getState()
.nodes()
.getMasterNodeId()
);
awaitMasterNode();

logger.info("--> start another node");
final String node2 = internalCluster().startNode(settings);
registerAzureNode(node2);
assertNotNull(
client().admin()
.cluster()
.prepareState(TEST_REQUEST_TIMEOUT)
.setMasterNodeTimeout(TimeValue.timeValueSeconds(1))
.get()
.getState()
.nodes()
.getMasterNodeId()
);
awaitMasterNode();

// We expect having 2 nodes as part of the cluster, let's test that
assertNumberOfNodes(2);
Expand Down
1 change: 1 addition & 0 deletions server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ public void testElectMasterWithLatestVersion() throws Exception {

isolateAllNodes.stopDisrupting();

awaitMasterNode();
final ClusterState state = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
if (state.metadata().getProject().hasIndex("test") == false) {
fail("index 'test' was lost. current cluster state: " + state);
Expand Down
46 changes: 14 additions & 32 deletions server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
*/
package org.elasticsearch.readiness;

import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
Expand All @@ -18,9 +17,7 @@
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.Strings;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.discovery.MasterNotDiscoveredException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.reservedstate.service.FileSettingsService;
import org.elasticsearch.test.ESIntegTestCase;
Expand Down Expand Up @@ -48,7 +45,6 @@
import static org.elasticsearch.test.NodeRoles.nonDataNode;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.notNullValue;

Expand Down Expand Up @@ -110,44 +106,30 @@ protected Collection<Class<? extends Plugin>> getMockPlugins() {
return Collections.unmodifiableList(plugins);
}

private void assertMasterNode(Client client, String node) {
assertThat(
client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName(),
equalTo(node)
);
}

private void expectMasterNotFound() {
expectThrows(
MasterNotDiscoveredException.class,
clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).setMasterNodeTimeout(TimeValue.timeValueMillis(100))
);
}

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108613")
public void testReadinessDuringRestarts() throws Exception {
internalCluster().setBootstrapMasterNodeIndex(0);
writeFileSettings(testJSON);
logger.info("--> start data node / non master node");
String dataNode = internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s"));

expectMasterNotFound();
awaitMasterNotFound();
assertFalse(internalCluster().getInstance(ReadinessService.class, dataNode).ready());

logger.info("--> start master node");
final String masterNode = internalCluster().startMasterOnlyNode();

assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);
tcpReadinessProbeTrue(internalCluster().getInstance(ReadinessService.class, dataNode));
tcpReadinessProbeTrue(internalCluster().getInstance(ReadinessService.class, masterNode));

final var masterReadinessService = internalCluster().getInstance(ReadinessService.class, masterNode);
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);

logger.info("--> stop master node");
Settings masterDataPathSettings = internalCluster().dataPathSettings(internalCluster().getMasterName());
internalCluster().stopCurrentMasterNode();
expectMasterNotFound();
awaitMasterNotFound();

tcpReadinessProbeFalse(masterReadinessService);

Expand All @@ -156,8 +138,8 @@ public void testReadinessDuringRestarts() throws Exception {
Settings.builder().put(nonDataNode(masterNode())).put(masterDataPathSettings)
);

assertMasterNode(internalCluster().nonMasterClient(), nextMasterEligibleNodeName);
assertMasterNode(internalCluster().masterClient(), nextMasterEligibleNodeName);
awaitMasterNode(internalCluster().getNonMasterNodeName(), nextMasterEligibleNodeName);
awaitMasterNode(internalCluster().getMasterName(), nextMasterEligibleNodeName);
tcpReadinessProbeTrue(internalCluster().getInstance(ReadinessService.class, nextMasterEligibleNodeName));
}

Expand All @@ -168,7 +150,7 @@ public void testReadinessDuringRestartsNormalOrder() throws Exception {
String masterNode = internalCluster().startMasterOnlyNode();
internalCluster().validateClusterFormed();

assertMasterNode(internalCluster().masterClient(), masterNode);
awaitMasterNode(internalCluster().getMasterName(), masterNode);

logger.info("--> start 2 data nodes");
List<String> dataNodes = internalCluster().startDataOnlyNodes(2);
Expand Down Expand Up @@ -196,7 +178,7 @@ public Settings onNodeStopped(String nodeName) throws Exception {
internalCluster().restartNode(masterNode, new InternalTestCluster.RestartCallback() {
@Override
public Settings onNodeStopped(String nodeName) throws Exception {
expectMasterNotFound();
awaitMasterNotFound();

logger.info("--> master node [{}] stopped", nodeName);

Expand Down Expand Up @@ -271,7 +253,7 @@ public void testNotReadyOnBadFileSettings() throws Exception {
final String masterNode = internalCluster().startMasterOnlyNode(
Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
);
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);
var savedClusterState = setupClusterStateListenerForError(masterNode);

FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
Expand All @@ -294,22 +276,22 @@ public void testReadyAfterRestartWithBadFileSettings() throws Exception {
String dataNode = internalCluster().startNode(Settings.builder().put(dataOnlyNode()).put("discovery.initial_state_timeout", "1s"));
String masterNode = internalCluster().startMasterOnlyNode();

assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);
assertBusy(() -> assertTrue("master node ready", internalCluster().getInstance(ReadinessService.class, masterNode).ready()));
assertBusy(() -> assertTrue("data node ready", internalCluster().getInstance(ReadinessService.class, dataNode).ready()));

logger.info("--> stop master node");
Settings masterDataPathSettings = internalCluster().dataPathSettings(internalCluster().getMasterName());
internalCluster().stopCurrentMasterNode();
expectMasterNotFound();
awaitMasterNotFound();

logger.info("--> write bad file settings before restarting master node");
writeFileSettings(testErrorJSON);

logger.info("--> restart master node");
String nextMasterNode = internalCluster().startNode(Settings.builder().put(nonDataNode(masterNode())).put(masterDataPathSettings));

assertMasterNode(internalCluster().nonMasterClient(), nextMasterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), nextMasterNode);

var savedClusterState = setupClusterStateListenerForError(nextMasterNode);
assertTrue(savedClusterState.await(20, TimeUnit.SECONDS));
Expand All @@ -324,7 +306,7 @@ public void testReadyWhenMissingFileSettings() throws Exception {
final String masterNode = internalCluster().startMasterOnlyNode(
Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
);
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);
var savedClusterState = setupClusterStateListener(masterNode);

// we need this after we setup the listener above, in case the node started and processed
Expand Down Expand Up @@ -382,7 +364,7 @@ public void testReadyAfterCorrectFileSettings() throws Exception {

logger.info("--> start master node");
final String masterNode = internalCluster().startMasterOnlyNode();
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);
var readinessProbeListening = setupReadinessProbeListener(masterNode);

FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
Expand Down
14 changes: 2 additions & 12 deletions ...erTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
import org.elasticsearch.action.admin.indices.template.reservedstate.ReservedComposableIndexTemplateAction;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.metadata.ComponentTemplate;
Expand All @@ -35,7 +34,6 @@
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

Expand All @@ -46,7 +44,6 @@
import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.notNullValue;

Expand Down Expand Up @@ -355,13 +352,6 @@ public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase {
}
}""";

private void assertMasterNode(Client client, String node) throws ExecutionException, InterruptedException {
assertThat(
client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).execute().get().getState().nodes().getMasterNode().getName(),
equalTo(node)
);
}

private void writeJSONFile(String node, String json) throws Exception {
FileSettingsServiceIT.writeJSONFile(node, json, logger, versionCounter.incrementAndGet());
}
Expand Down Expand Up @@ -543,7 +533,7 @@ public void testSettingsApplied() throws Exception {

logger.info("--> start master node");
final String masterNode = internalCluster().startMasterOnlyNode();
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);

assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2());

Expand Down Expand Up @@ -611,7 +601,7 @@ public void testErrorSaved() throws Exception {

logger.info("--> start master node");
final String masterNode = internalCluster().startMasterOnlyNode();
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);
var savedClusterState = setupClusterStateListenerForError(masterNode);

writeJSONFile(masterNode, testErrorJSON);
Expand Down
20 changes: 6 additions & 14 deletions ...ternalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata;
Expand Down Expand Up @@ -129,13 +128,6 @@ public void resetVersionCounter() {
}
}""";

private void assertMasterNode(Client client, String node) {
assertThat(
client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName(),
equalTo(node)
);
}

public static void writeJSONFile(String node, String json, Logger logger, Long version) throws Exception {
FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
writeJSONFile(node, json, logger, version, fileSettingsService.watchedFile());
Expand Down Expand Up @@ -254,7 +246,7 @@ public void testSettingsApplied() throws Exception {

logger.info("--> start master node");
final String masterNode = internalCluster().startMasterOnlyNode();
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);
var savedClusterState = setupClusterStateListener(masterNode, versionCounter.incrementAndGet());

FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
Expand All @@ -281,7 +273,7 @@ public void testSettingsAppliedOnStart() throws Exception {

logger.info("--> start master node");
final String masterNode = internalCluster().startMasterOnlyNode();
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);

FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);

Expand All @@ -297,7 +289,7 @@ public void testReservedStatePersistsOnRestart() throws Exception {
final String masterNode = internalCluster().startMasterOnlyNode(
Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
);
assertMasterNode(internalCluster().masterClient(), masterNode);
awaitMasterNode(internalCluster().getMasterName(), masterNode);
var savedClusterState = setupClusterStateListener(masterNode, versionCounter.incrementAndGet());

FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
Expand Down Expand Up @@ -375,7 +367,7 @@ public void testErrorSaved() throws Exception {
final String masterNode = internalCluster().startMasterOnlyNode(
Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
);
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);
var savedClusterState = setupClusterStateListenerForError(masterNode);

FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
Expand All @@ -399,7 +391,7 @@ public void testErrorCanRecoverOnRestart() throws Exception {
final String masterNode = internalCluster().startMasterOnlyNode(
Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
);
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);
var savedClusterState = setupClusterStateListenerForError(masterNode);

FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
Expand Down Expand Up @@ -435,7 +427,7 @@ public void testNewErrorOnRestartReprocessing() throws Exception {
final String masterNode = internalCluster().startMasterOnlyNode(
Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
);
assertMasterNode(internalCluster().nonMasterClient(), masterNode);
awaitMasterNode(internalCluster().getNonMasterNodeName(), masterNode);
var savedClusterState = setupClusterStateListenerForError(masterNode);

FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
Expand Down
Loading