Backport Fix RetrySearchIntegTests (#122919) (#129404)

Don't simultaneously restart multiple nodes in a cluster. It causes data races when
multiple primaries are trying to mark the `[[.snapshot-blob-cache][0]]` shard as stale.
This backports #122919 to the 8.17 branch since we still get occasional test failures 
related to it there.

Closes #127136

Co-authored-by: Artem Prigoda <[email protected]>

Author: cbuescher

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/RetrySearchIntegTests.java

@@ -90,6 +90,7 @@ public void testSearcherId() throws Exception {
         for (String allocatedNode : allocatedNodes) {
             if (randomBoolean()) {
                 internalCluster().restartNode(allocatedNode);
+                ensureGreen(indexName);
             }
         }
         ensureGreen(indexName);
@@ -151,6 +152,7 @@ public void testRetryPointInTime() throws Exception {
             final Set<String> allocatedNodes = internalCluster().nodesInclude(indexName);
             for (String allocatedNode : allocatedNodes) {
                 internalCluster().restartNode(allocatedNode);
+                ensureGreen(indexName);
             }
             ensureGreen(indexName);
             assertNoFailuresAndResponse(