Fix ClusterInfoServiceIT#testMaxQueueLatenciesInClusterInfo (elastic#136461)

Closes: elastic#134500

Author: nicktindall

muted-tests.yml

@@ -441,9 +441,6 @@ tests:
 - class: org.elasticsearch.action.admin.cluster.state.TransportClusterStateActionTests
   method: testGetClusterStateWithDefaultProjectOnly
   issue: https://github.com/elastic/elasticsearch/issues/134450
-- class: org.elasticsearch.cluster.ClusterInfoServiceIT
-  method: testMaxQueueLatenciesInClusterInfo
-  issue: https://github.com/elastic/elasticsearch/issues/134500
 - class: org.elasticsearch.xpack.esql.expression.function.scalar.score.DecayTests
   method: "testEvaluateBlockWithNulls {TestCase=<date_nanos>, <date_nanos>, <time_duration>, <_source> #11}"
   issue: https://github.com/elastic/elasticsearch/issues/134509

server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java

@@ -514,6 +514,12 @@ public void testMaxQueueLatenciesInClusterInfo() throws Exception {
                 threadsToJoin[i].join();
             }
             Arrays.stream(threadsToJoin).forEach(thread -> assertFalse(thread.isAlive()));
+            // Monitor the write executor on the data node to try and determine when the backlog of tasks
+            // has been fully drained (and
+            // {@link org.elasticsearch.common.util.concurrent.TaskExecutionTimeTrackingEsThreadPoolExecutor.afterExecute()}
+            // has been called). This is probably not foolproof, so worth investigating if we see non-zero utilization numbers
+            // after the next poll. See https://github.com/elastic/elasticsearch/issues/134500
+            assertBusy(() -> assertThat(trackingWriteExecutor.getActiveCount(), equalTo(0)));
 
             assertThat(
                 "Unexpectedly found a task queued for the write thread pool. Write thread pool dump: " + trackingWriteExecutor,