Add retry for inactivePeer of region migration (#15294) (#15303)

* done

* fix

(cherry picked from commit b1a0dee)

Author: liyuheng55555

iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java

@@ -331,7 +331,7 @@ public void addRemotePeer(ConsensusGroupId groupId, Peer peer) throws ConsensusE
       try {
         // step 1: inactive new Peer to prepare for following steps
         logger.info("[IoTConsensus] inactivate new peer: {}", peer);
-        impl.inactivePeer(peer, false);
+        impl.inactivatePeer(peer, false);
 
         // step 2: notify all the other Peers to build the sync connection to newPeer
         logger.info("[IoTConsensus] notify current peers to build sync log...");
@@ -397,7 +397,7 @@ public void removeRemotePeer(ConsensusGroupId groupId, Peer peer) throws Consens
 
       try {
         // let target peer reject new write
-        impl.inactivePeer(peer, true);
+        impl.inactivatePeer(peer, true);
         KillPoint.setKillPoint(IoTConsensusRemovePeerCoordinatorKillPoints.AFTER_INACTIVE_PEER);
         // wait its SyncLog to complete
         impl.waitTargetPeerUntilSyncLogCompleted(peer);

iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java

@@ -413,26 +413,38 @@ public interface ThrowableFunction<T, R> {
     R apply(T t) throws Exception;
   }
 
-  public void inactivePeer(Peer peer, boolean forDeletionPurpose)
+  public void inactivatePeer(Peer peer, boolean forDeletionPurpose)
       throws ConsensusGroupModifyPeerException {
-    try (SyncIoTConsensusServiceClient client =
-        syncClientManager.borrowClient(peer.getEndpoint())) {
-      try {
-        TInactivatePeerRes res =
-            client.inactivatePeer(
-                new TInactivatePeerReq(peer.getGroupId().convertToTConsensusGroupId())
-                    .setForDeletionPurpose(forDeletionPurpose));
-        if (!isSuccess(res.status)) {
-          throw new ConsensusGroupModifyPeerException(
-              String.format("error when inactivating %s. %s", peer, res.getStatus()));
+    ConsensusGroupModifyPeerException lastException = null;
+    // In region migration, if the target node restarts before the “addRegionPeer” phase within 1
+    // minutes,
+    // the client in the ClientManager will become invalid.
+    // This PR adds 1 retry at this point to ensure that region migration can still proceed
+    // correctly in such cases.
+    for (int i = 0; i < 2; i++) {
+      try (SyncIoTConsensusServiceClient client =
+          syncClientManager.borrowClient(peer.getEndpoint())) {
+        try {
+          TInactivatePeerRes res =
+              client.inactivatePeer(
+                  new TInactivatePeerReq(peer.getGroupId().convertToTConsensusGroupId())
+                      .setForDeletionPurpose(forDeletionPurpose));
+          if (isSuccess(res.status)) {
+            return;
+          }
+          lastException =
+              new ConsensusGroupModifyPeerException(
+                  String.format("error when inactivating %s. %s", peer, res.getStatus()));
+        } catch (Exception e) {
+          lastException =
+              new ConsensusGroupModifyPeerException(
+                  String.format("error when inactivating %s", peer), e);
         }
-      } catch (Exception e) {
-        throw new ConsensusGroupModifyPeerException(
-            String.format("error when inactivating %s", peer), e);
+      } catch (ClientManagerException e) {
+        lastException = new ConsensusGroupModifyPeerException(e);
       }
-    } catch (ClientManagerException e) {
-      throw new ConsensusGroupModifyPeerException(e);
     }
+    throw lastException;
   }
 
   public void triggerSnapshotLoad(Peer peer) throws ConsensusGroupModifyPeerException {