RATIS-2261. Intermittent failure in TestRaftSnapshotWithGrpc.testInstallSnapshotDuringBootstrap.

Author: szetszwo

ratis-server/src/test/java/org/apache/ratis/statemachine/RaftSnapshotBaseTest.java

@@ -49,15 +49,12 @@
 import org.apache.ratis.util.JavaUtils;
 import org.apache.ratis.util.LifeCycle;
 import org.apache.ratis.util.Slf4jUtils;
-import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
@@ -67,11 +64,18 @@
 import org.apache.ratis.thirdparty.com.codahale.metrics.Timer;
 import org.slf4j.event.Level;
 
-public abstract class RaftSnapshotBaseTest extends BaseTest {
+public abstract class RaftSnapshotBaseTest<CLUSTER extends MiniRaftCluster>
+    extends BaseTest
+    implements MiniRaftCluster.Factory.Get<CLUSTER> {
   {
     Slf4jUtils.setLogLevel(RaftServer.Division.LOG, Level.DEBUG);
     Slf4jUtils.setLogLevel(RaftLog.LOG, Level.DEBUG);
     Slf4jUtils.setLogLevel(RaftClient.LOG, Level.DEBUG);
+
+    final RaftProperties p = getProperties();
+    p.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, SimpleStateMachine4Testing.class, StateMachine.class);
+    RaftServerConfigKeys.Snapshot.setAutoTriggerThreshold(p, SNAPSHOT_TRIGGER_THRESHOLD);
+    RaftServerConfigKeys.Snapshot.setAutoTriggerEnabled(p, true);
   }
 
   static final Logger LOG = LoggerFactory.getLogger(RaftSnapshotBaseTest.class);
@@ -119,36 +123,18 @@ public static void assertLogContent(RaftServer.Division server, boolean isLeader
     }
   }
 
-  private MiniRaftCluster cluster;
-
-  public abstract MiniRaftCluster.Factory<?> getFactory();
-
-  @BeforeEach
-  public void setup() throws IOException {
-    final RaftProperties prop = new RaftProperties();
-    prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
-        SimpleStateMachine4Testing.class, StateMachine.class);
-    RaftServerConfigKeys.Snapshot.setAutoTriggerThreshold(
-        prop, SNAPSHOT_TRIGGER_THRESHOLD);
-    RaftServerConfigKeys.Snapshot.setAutoTriggerEnabled(prop, true);
-    this.cluster = getFactory().newCluster(1, prop);
-    cluster.start();
-  }
-
-  @AfterEach
-  public void tearDown() {
-    if (cluster != null) {
-      cluster.shutdown();
-    }
-  }
-
   /**
    * Keep generating writing traffic and make sure snapshots are taken.
    * We then restart the whole raft peer and check if it can correctly load
    * snapshots + raft log.
    */
   @Test
   public void testRestartPeer() throws Exception {
+    runWithNewCluster(1, this::runTestRestartPeer);
+
+  }
+
+  void runTestRestartPeer(CLUSTER cluster) throws Exception {
     RaftTestUtil.waitForLeader(cluster);
     final RaftPeerId leaderId = cluster.getLeader().getId();
     int i = 0;
@@ -180,7 +166,7 @@ public void testRestartPeer() throws Exception {
 
   public static boolean exists(File f) {
     if (f.exists()) {
-      LOG.info("File exists: " + f);
+      LOG.info("File exists: {}", f);
       return true;
     }
     return false;
@@ -193,6 +179,10 @@ public static boolean exists(File f) {
    */
   @Test
   public void testBasicInstallSnapshot() throws Exception {
+    runWithNewCluster(1, this::runTestBasicInstallSnapshot);
+  }
+
+  void runTestBasicInstallSnapshot(CLUSTER cluster) throws Exception {
     final List<LogSegmentPath> logs;
     int i = 0;
     try {
@@ -275,6 +265,10 @@ public void testBasicInstallSnapshot() throws Exception {
    */
   @Test
   public void testInstallSnapshotDuringBootstrap() throws Exception {
+    runWithNewCluster(3, this::runTestInstallSnapshotDuringBootstrap);
+  }
+
+  void runTestInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exception {
     int i = 0;
     try {
       RaftTestUtil.waitForLeader(cluster);
@@ -300,7 +294,7 @@ public void testInstallSnapshotDuringBootstrap() throws Exception {
       assertLeaderContent(cluster);
 
       // add two more peers
-      String[] newPeers = new String[]{"s3", "s4"};
+      final String[] newPeers = {"new0", "new1"};
       MiniRaftCluster.PeerChanges change = cluster.addNewPeers(
           newPeers, true, false);
       // trigger setConfiguration

ratis-server/src/test/java/org/apache/ratis/statemachine/impl/SimpleStateMachine4Testing.java

@@ -210,7 +210,7 @@ private void put(LogEntryProto entry) {
   @Override
   public synchronized void initialize(RaftServer server, RaftGroupId raftGroupId,
       RaftStorage raftStorage) throws IOException {
-    LOG.info("Initializing " + this);
+    LOG.info("Initializing {}", this);
     this.groupId = raftGroupId;
     getLifeCycle().startAndTransition(() -> {
       super.initialize(server, raftGroupId, raftStorage);
@@ -233,7 +233,10 @@ public synchronized void pause() {
 
   @Override
   public synchronized void reinitialize() throws IOException {
-    LOG.info("Reinitializing " + this);
+    LOG.info("Reinitializing {}", this);
+    indexMap.clear();
+    dataMap.clear();
+
     loadSnapshot(storage.getLatestSnapshot());
     if (getLifeCycleState() == LifeCycle.State.PAUSED) {
       getLifeCycle().transition(LifeCycle.State.STARTING);
@@ -328,14 +331,14 @@ public CompletableFuture<Message> query(Message request) {
     final String string = request.getContent().toStringUtf8();
     Exception exception;
     try {
-      LOG.info("query " + string);
+      LOG.info("query {}", string);
       final LogEntryProto entry = dataMap.get(string);
       if (entry != null) {
         return CompletableFuture.completedFuture(Message.valueOf(entry.toByteString()));
       }
       exception = new IndexOutOfBoundsException(getId() + ": LogEntry not found for query " + string);
     } catch (Exception e) {
-      LOG.warn("Failed request " + request, e);
+      LOG.warn("Failed request {}", request, e);
       exception = e;
     }
     return JavaUtils.completeExceptionally(new StateMachineException(

ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftSnapshotWithGrpc.java

@@ -20,22 +20,16 @@
 import java.util.Optional;
 
 import org.apache.ratis.metrics.LongCounter;
-import org.apache.ratis.server.impl.MiniRaftCluster;
 import org.apache.ratis.metrics.MetricRegistries;
 import org.apache.ratis.metrics.MetricRegistryInfo;
 import org.apache.ratis.metrics.RatisMetricRegistry;
 import org.apache.ratis.server.RaftServer;
 import org.apache.ratis.statemachine.RaftSnapshotBaseTest;
-import org.apache.ratis.test.tag.Flaky;
 import org.junit.jupiter.api.Assertions;
 
-@Flaky("RATIS-2261")
-public class TestRaftSnapshotWithGrpc extends RaftSnapshotBaseTest {
-  @Override
-  public MiniRaftCluster.Factory<?> getFactory() {
-    return MiniRaftClusterWithGrpc.FACTORY;
-  }
-
+public class TestRaftSnapshotWithGrpc
+    extends RaftSnapshotBaseTest<MiniRaftClusterWithGrpc>
+    implements MiniRaftClusterWithGrpc.FactoryGet {
   @Override
   protected void verifyInstallSnapshotMetric(RaftServer.Division leader) {
     MetricRegistryInfo info = new MetricRegistryInfo(leader.getMemberId().toString(),

ratis-test/src/test/java/org/apache/ratis/netty/TestRaftSnapshotWithNetty.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -17,12 +17,9 @@
  */
 package org.apache.ratis.netty;
 
-import org.apache.ratis.server.impl.MiniRaftCluster;
 import org.apache.ratis.statemachine.RaftSnapshotBaseTest;
 
-public class TestRaftSnapshotWithNetty extends RaftSnapshotBaseTest {
-  @Override
-  public MiniRaftCluster.Factory<?> getFactory() {
-    return MiniRaftClusterWithNetty.FACTORY;
-  }
+public class TestRaftSnapshotWithNetty
+    extends RaftSnapshotBaseTest<MiniRaftClusterWithNetty>
+    implements MiniRaftClusterWithNetty.FactoryGet {
 }

ratis-test/src/test/java/org/apache/ratis/server/simulation/TestRaftSnapshotWithSimulatedRpc.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -17,12 +17,9 @@
  */
 package org.apache.ratis.server.simulation;
 
-import org.apache.ratis.server.impl.MiniRaftCluster;
 import org.apache.ratis.statemachine.RaftSnapshotBaseTest;
 
-public class TestRaftSnapshotWithSimulatedRpc extends RaftSnapshotBaseTest {
-  @Override
-  public MiniRaftCluster.Factory<?> getFactory() {
-    return MiniRaftClusterWithSimulatedRpc.FACTORY;
-  }
+public class TestRaftSnapshotWithSimulatedRpc
+    extends RaftSnapshotBaseTest<MiniRaftClusterWithSimulatedRpc>
+    implements MiniRaftClusterWithSimulatedRpc.FactoryGet {
 }