Reduce cluster state persistence check frequency (#84517)

In #84142 we introduced a somewhat-expensive assertion that every
cluster state we write can be read again. In practice we don't need to
check that property every time, so with this commit we usually skip it.

Author: DaveCTurner

server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java

@@ -45,6 +45,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.CheckedBiConsumer;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.CompositeBytesReference;
@@ -1008,7 +1009,7 @@ void commit(long currentTerm, long lastAcceptedVersion, Version oldestIndexVersi
         }
 
         private boolean assertOnCommit() {
-            if (assertOnCommit != null /* TODO && Randomness.get().nextInt(100) == 0 */) {
+            if (assertOnCommit != null && Randomness.get().nextInt(100) == 0) {
                 // only rarely run this assertion since reloading the whole state can be quite expensive
                 for (final var metadataIndexWriter : metadataIndexWriters) {
                     try (var directoryReader = DirectoryReader.open(metadataIndexWriter.indexWriter)) {