set dirty and add ip to suspect list (#379)

Author: JackShi148

src/main/java/com/alipay/oceanbase/rpc/location/model/RouteTableRefresher.java

@@ -21,10 +21,7 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -33,6 +30,7 @@
 import com.alipay.oceanbase.rpc.exception.ObTableTryLockTimeoutException;
 import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException;
 import com.alipay.oceanbase.rpc.location.LocationUtil;
+import com.alipay.oceanbase.rpc.table.ObTable;
 import org.slf4j.Logger;
 
 import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.getLogger;
@@ -41,6 +39,8 @@ public class RouteTableRefresher {
 
     private static final Logger                                           logger    = getLogger(RouteTableRefresher.class);
 
+    private static final int                                              failureLimit = 3;
+
     private static final String                                           sql = "select 'detect server alive' from dual";
 
     private final ObTableClient                                           tableClient;
@@ -136,7 +136,7 @@ private void doCheckAliveTask() {
                 checkAlive(entry.getKey());
             } catch (Exception e) {
                 // silence resolving
-                logger.warn("RouteTableRefresher::doCheckAliveTask fail, failed server: {}", entry.getKey().toString());
+                logger.warn("RouteTableRefresher::doCheckAliveTask fail, failed server: {}", entry.getKey().toString(), e);
             }
         }
     }
@@ -198,7 +198,7 @@ private void checkAlive(ObServerAddr addr) {
         }
     }
 
-    public static void addIntoSuspectIPs(SuspectObServer server) throws Exception {
+    public static void addIntoSuspectIPs(SuspectObServer server) throws InterruptedException {
         ObServerAddr addr = server.getAddr();
         if (suspectServers.get(addr) != null) {
             // already in the list, directly return
@@ -264,7 +264,16 @@ private void removeFromSuspectIPs(ObServerAddr addr) {
                         throw new ObTableTryLockTimeoutException("try to get suspect server lock timeout, timeout: 1s");
                     }
                     // no need to remove lock
-                    suspectServers.remove(addr);
+                    SuspectObServer server = suspectServers.remove(addr);
+                    if (server != null) {
+                        int failure = server.getFailure();
+                        if (failure < failureLimit) {
+                            ObTable obTable = tableClient.getTableRoute().getTableRoster().getTable(addr);
+                            if (obTable != null && !obTable.isValid()) {
+                                obTable.setValid();
+                            }
+                        }
+                    }
                     logger.debug("removed server from suspect list: {}", addr);
                     break;
                 } catch (ObTableTryLockTimeoutException e) {
@@ -286,7 +295,6 @@ private void removeFromSuspectIPs(ObServerAddr addr) {
     }
 
     private void calcFailureOrClearCache(ObServerAddr addr) {
-        int failureLimit = 3;
         TableRoute tableRoute = tableClient.getTableRoute();
         SuspectObServer server = suspectServers.get(addr);
         server.incrementFailure();

src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java

@@ -26,6 +26,7 @@
 import com.alipay.oceanbase.rpc.exception.*;
 import com.alipay.oceanbase.rpc.filter.ObTableFilter;
 import com.alipay.oceanbase.rpc.location.model.ObServerAddr;
+import com.alipay.oceanbase.rpc.location.model.RouteTableRefresher;
 import com.alipay.oceanbase.rpc.mutation.*;
 import com.alipay.oceanbase.rpc.protocol.payload.ObPayload;
 import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*;
@@ -429,13 +430,13 @@ public ObPayload execute(final ObPayload request) throws RemotingException,
             // cannot connect to ob server, need refresh table location
             // do not set odp ip and port as dirty
             if (!isOdpMode) {
-                setDirty();
+                dealWithReconnectFailForObTableConnection();
             }
             throw new ObTableServerConnectException(ex);
         } catch (ObTableServerConnectException ex) {
             // do not set odp ip and port as dirty
             if (!isOdpMode) {
-                setDirty();
+                dealWithReconnectFailForObTableConnection();
             }
             throw ex;
         } catch (Exception ex) {
@@ -505,13 +506,13 @@ public ObPayload executeWithConnection(final ObPayload request,
             // Cannot connect to ob server, need refresh table location
             // do not set odp ip and port as dirty
             if (!isOdpMode) {
-                setDirty();
+                dealWithReconnectFailForObTableConnection();
             }
             throw new ObTableServerConnectException(ex);
         } catch (ObTableServerConnectException ex) {
             // do not set odp ip and port as dirty
             if (!isOdpMode) {
-                setDirty();
+                dealWithReconnectFailForObTableConnection();
             }
             throw ex;
         } catch (Exception ex) {
@@ -520,6 +521,12 @@ public ObPayload executeWithConnection(final ObPayload request,
         return executeWithReconnect(connection, request);
     }
 
+    private void dealWithReconnectFailForObTableConnection() throws InterruptedException {
+        setDirty();
+        RouteTableRefresher.SuspectObServer suspectAddr = new RouteTableRefresher.SuspectObServer(addr);
+        RouteTableRefresher.addIntoSuspectIPs(suspectAddr);
+    }
+
     private void checkObTableOperationResult(String ip, int port, Object result) {
         if (result == null) {
             throw new ObTableException("client get unexpected NULL result");