fix check expired nullpointer, add login-failed ip into suspect list

Author: JackShi148

src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java

@@ -1184,12 +1184,12 @@ public Row transformToRow(String tableName, Object[] rowkey) throws Exception {
     public void dealWithRpcTimeoutForSingleTablet(ObServerAddr addr, String tableName, long tabletId) throws Exception {
         tableRoute.refreshPartitionLocation(tableName, tabletId, null);
         RouteTableRefresher.SuspectObServer suspectAddr = new RouteTableRefresher.SuspectObServer(addr);
-        tableRoute.addIntoSuspectIPs(suspectAddr);
+        RouteTableRefresher.addIntoSuspectIPs(suspectAddr);
     }
     public void dealWithRpcTimeoutForBatchTablet(ObServerAddr addr, String tableName) throws Exception {
         tableRoute.refreshTabletLocationBatch(tableName);
         RouteTableRefresher.SuspectObServer suspectAddr = new RouteTableRefresher.SuspectObServer(addr);
-        tableRoute.addIntoSuspectIPs(suspectAddr);
+        RouteTableRefresher.addIntoSuspectIPs(suspectAddr);
     }
 
     /**

src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableConnection.java

@@ -20,6 +20,7 @@
 import com.alipay.oceanbase.rpc.ObGlobal;
 import com.alipay.oceanbase.rpc.exception.*;
 import com.alipay.oceanbase.rpc.location.LocationUtil;
+import com.alipay.oceanbase.rpc.location.model.RouteTableRefresher;
 import com.alipay.oceanbase.rpc.protocol.payload.impl.login.ObTableLoginRequest;
 import com.alipay.oceanbase.rpc.protocol.payload.impl.login.ObTableLoginResult;
 import com.alipay.oceanbase.rpc.table.ObTable;
@@ -115,6 +116,8 @@ private boolean connect() throws Exception {
         MONITOR.info(logMessage(null, "CONNECT", endpoint, System.currentTimeMillis() - start));
 
         if (tries >= maxTryTimes) {
+            RouteTableRefresher.SuspectObServer suspectAddr = new RouteTableRefresher.SuspectObServer(obTable.getObServerAddr());
+            RouteTableRefresher.addIntoSuspectIPs(suspectAddr);
             LOGGER.warn("connect failed after max " + maxTryTimes + " tries "
                         + TraceUtil.formatIpPort(obTable));
             throw new ObTableServerConnectException("connect failed after max " + maxTryTimes
@@ -252,7 +255,7 @@ public void reConnectAndLogin(String msg) throws ObTableException {
         try {
             // 1. check the connection is available, force to close it
             if (checkAvailable()) {
-                LOGGER.warn("The connection would be closed and reconnected if: "
+                LOGGER.warn("The connection would be closed and reconnected is: "
                             + connection.getUrl());
                 close();
             }

src/main/java/com/alipay/oceanbase/rpc/location/model/RouteTableRefresher.java

@@ -39,21 +39,21 @@
 
 public class RouteTableRefresher {
 
-    private static final Logger                                    logger    = getLogger(RouteTableRefresher.class);
+    private static final Logger                                           logger    = getLogger(RouteTableRefresher.class);
 
-    private static final String                                    sql = "select 'detect server alive' from dual";
+    private static final String                                           sql = "select 'detect server alive' from dual";
 
-    private final ObTableClient                                    tableClient;
+    private final ObTableClient                                           tableClient;
 
-    private final ObUserAuth                                       sysUA;
+    private final ObUserAuth                                              sysUA;
 
-    private final ScheduledExecutorService                         scheduler = Executors.newScheduledThreadPool(2);
+    private final ScheduledExecutorService                                scheduler = Executors.newScheduledThreadPool(2);
 
-    private final ConcurrentHashMap<ObServerAddr, Lock>            suspectLocks = new ConcurrentHashMap<>(); // ObServer -> access lock
+    private final static ConcurrentHashMap<ObServerAddr, Lock>            suspectLocks = new ConcurrentHashMap<>(); // ObServer -> access lock
 
-    private final ConcurrentHashMap<ObServerAddr, SuspectObServer> suspectServers = new ConcurrentHashMap<>(); // ObServer -> information structure
+    private final static ConcurrentHashMap<ObServerAddr, SuspectObServer> suspectServers = new ConcurrentHashMap<>(); // ObServer -> information structure
 
-    private final HashMap<ObServerAddr, Long>                      serverLastAccessTimestamps = new HashMap<>(); // ObServer -> last access timestamp
+    private final static HashMap<ObServerAddr, Long>                      serverLastAccessTimestamps = new HashMap<>(); // ObServer -> last access timestamp
 
     public RouteTableRefresher(ObTableClient tableClient, ObUserAuth sysUA) {
         this.tableClient = tableClient;
@@ -149,24 +149,21 @@ private void checkAlive(ObServerAddr addr) {
         Statement statement = null;
         ResultSet rs = null;
         try {
-            logger.debug("[background keep alive] check alive, server: {}", addr);
             connection = LocationUtil.getMetaRefreshConnection(url, sysUA);
             statement = connection.createStatement();
             rs = statement.executeQuery(sql);
             boolean alive = false;
             while (rs.next()) {
                 String res = rs.getString("detect server alive");
-                logger.debug("[background keep alive] result: {}", res);
                 alive = res.equalsIgnoreCase("detect server alive");
             }
             if (alive) {
-                logger.debug("[background keep alive] alive, remove server: {}", addr);
                 removeFromSuspectIPs(addr);
             } else {
                 calcFailureOrClearCache(addr);
             }
         } catch (Throwable t) {
-            logger.debug("check alive failed, server: {}", addr, t);
+            logger.debug("check alive failed, server: {}", addr.toString(), t);
             if (t instanceof SQLException) {
                 // occurred during query
                 calcFailureOrClearCache(addr);
@@ -201,8 +198,7 @@ private void checkAlive(ObServerAddr addr) {
         }
     }
 
-    public void addIntoSuspectIPs(SuspectObServer server) throws Exception {
-        logger.debug("[background keep alive] enter addInto");
+    public static void addIntoSuspectIPs(SuspectObServer server) throws Exception {
         ObServerAddr addr = server.getAddr();
         if (suspectServers.get(addr) != null) {
             // already in the list, directly return
@@ -233,7 +229,7 @@ public void addIntoSuspectIPs(SuspectObServer server) throws Exception {
                             break;
                         }
                     }
-                    logger.debug("[background keep alive] add into ips, server: {}", addr);
+                    logger.debug("add into suspect list, server: {}", addr);
                     suspectServers.put(addr, server);
                     serverLastAccessTimestamps.put(addr, server.getAccessTimestamp());
                     break;
@@ -252,7 +248,6 @@ public void addIntoSuspectIPs(SuspectObServer server) throws Exception {
     }
 
     private void removeFromSuspectIPs(ObServerAddr addr) {
-        logger.debug("[background keep alive] remove server, server:{}", addr);
         Lock lock = suspectLocks.get(addr);
         if (lock == null) {
             // lock must have been added before remove
@@ -270,7 +265,7 @@ private void removeFromSuspectIPs(ObServerAddr addr) {
                     }
                     // no need to remove lock
                     suspectServers.remove(addr);
-                    logger.debug("[background keep alive] removed server: {}", addr);
+                    logger.debug("removed server from suspect list: {}", addr);
                     break;
                 } catch (ObTableTryLockTimeoutException e) {
                     // if try lock timeout, need to retry

src/main/java/com/alipay/oceanbase/rpc/location/model/TableRoute.java

@@ -368,10 +368,6 @@ public void removeObServer(ObServerAddr addr) {
         servers.remove(addr);
     }
 
-    public void addIntoSuspectIPs(RouteTableRefresher.SuspectObServer addr) throws Exception {
-        routeRefresher.addIntoSuspectIPs(addr);
-    }
-
     /**
      * refresh all ob server synchronized, it will not refresh if last refresh time is 1 min ago
      * @param newRsList new root servers

src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java

@@ -977,7 +977,7 @@ private void checkAndReconnect() {
                     }
                     connections[idx].reConnectAndLogin("expired");
                 } catch (Exception e) {
-                    log.warn("ObTableConnectionPool::checkAndReconnect reconnect fail {}. {}", connections[idx].getConnection().getUrl(), e.getMessage());
+                    log.warn("ObTableConnectionPool::checkAndReconnect reconnect fail {}:{}. {}", obTable.getIp(), obTable.getPort(), e.getMessage());
                 } finally {
                     connections[idx].setExpired(false);
                 }