[Test] Wait for stable cluster before assertion

Nodes that are not publish quorum may not have applied the latest update
when awaitMasterNode returns. This PR fixes it by waiting for the
desired number of nodes.

Relates: elastic#129118
Resolves: elastic#130883

Author: ywangd

muted-tests.yml

@@ -564,9 +564,6 @@ tests:
 - class: org.elasticsearch.cluster.ClusterStateSerializationTests
   method: testSerializationPreMultiProject
   issue: https://github.com/elastic/elasticsearch/issues/130872
-- class: org.elasticsearch.cluster.coordination.votingonly.VotingOnlyNodePluginTests
-  method: testPreferFullMasterOverVotingOnlyNodes
-  issue: https://github.com/elastic/elasticsearch/issues/130883
 
 # Examples:
 #

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -965,12 +965,13 @@ public void awaitMasterNode(String viaNode, String masterNodeName) {
     }
 
     /**
-     * Waits for all nodes in the cluster to have a consistent view of which node is currently the master.
+     * Waits for all nodes forming the publish quorum in the cluster to have a consistent view of which node is currently the master.
      */
     public void awaitMasterNode() {
         // The cluster health API always runs on the master node, and the master only completes cluster state publication when all nodes
-        // in the cluster have accepted the new cluster state. By waiting for all events to have finished on the master node, we ensure
-        // that the whole cluster has a consistent view of which node is the master.
+        // forming the publish quorum have accepted the new cluster state. By waiting for all events to have finished on the
+        // master node, we ensure that all nodes that are part of the publish quorum have a consistent view of which node is the master.
+        // But nodes that are _not_ part of the quorum might still not have the same view yet.
         clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT).setTimeout(TEST_REQUEST_TIMEOUT).setWaitForEvents(Priority.LANGUID).get();
     }
 

x-pack/plugin/voting-only-node/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/votingonly/VotingOnlyNodePluginTests.java

@@ -96,7 +96,8 @@ public void testPreferFullMasterOverVotingOnlyNodes() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(0);
         internalCluster().startNodes(2);
         internalCluster().startNode(addRoles(Set.of(DiscoveryNodeRole.VOTING_ONLY_NODE_ROLE)));
-        internalCluster().startDataOnlyNodes(randomInt(2));
+        final int numDataNodes = randomInt(2);
+        internalCluster().startDataOnlyNodes(numDataNodes);
         assertBusy(
             () -> assertThat(
                 clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().getLastCommittedConfiguration().getNodeIds().size(),
@@ -108,6 +109,7 @@ public void testPreferFullMasterOverVotingOnlyNodes() throws Exception {
         internalCluster().stopCurrentMasterNode();
         awaitMasterNode();
         assertNotEquals(originalMaster, internalCluster().getMasterName());
+        ensureStableCluster(2 + numDataNodes); // wait for all nodes to join
         assertThat(
             VotingOnlyNodePlugin.isVotingOnlyNode(
                 clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode()