Capture exceptions in the refreshIndexInfo and do syncRefreshMetadata

Author: maochongxin

src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java

@@ -1144,16 +1144,35 @@ public ObIndexInfo getOrRefreshIndexInfo(final String indexTableName, boolean fo
                 int refreshTryTimes = tableEntryRefreshTryTimes > serverSize ? serverSize
                     : tableEntryRefreshTryTimes;
                 for (int i = 0; i < refreshTryTimes; i++) {
-                    ObServerAddr serverAddr = serverRoster.getServer(serverAddressPriorityTimeout,
-                        serverAddressCachingTimeout);
-                    indexInfo = getIndexInfoFromRemote(serverAddr, sysUA,
-                        tableEntryAcquireConnectTimeout, tableEntryAcquireSocketTimeout,
-                        indexTableName);
-                    if (indexInfo != null) {
-                        indexinfos.put(indexTableName, indexInfo);
-                    } else {
-                        RUNTIME.error("get index info from remote is null, indexTableName: {}",
-                            indexTableName);
+                    try {
+                        ObServerAddr serverAddr = serverRoster.getServer(serverAddressPriorityTimeout,
+                                serverAddressCachingTimeout);
+                        indexInfo = getIndexInfoFromRemote(serverAddr, sysUA,
+                                tableEntryAcquireConnectTimeout, tableEntryAcquireSocketTimeout,
+                                indexTableName);
+                        if (indexInfo != null) {
+                            indexinfos.put(indexTableName, indexInfo);
+                        } else {
+                            RUNTIME.error("get index info from remote is null, indexTableName: {}",
+                                    indexTableName);
+                        }
+                    } catch (ObTableServerCacheExpiredException e) {
+                        RUNTIME.error("get index info from remote meet exception", e);
+                        syncRefreshMetadata(false);
+                    } catch (ObTableEntryRefreshException e) {
+                        RUNTIME.error("get index info from remote meet exception", e);
+                        if (tableEntryRefreshContinuousFailureCount.incrementAndGet() > tableEntryRefreshContinuousFailureCeiling) {
+                            logger.error(LCD.convert("01-00019"),
+                                    tableEntryRefreshContinuousFailureCeiling);
+                            syncRefreshMetadata(false);
+                            tableEntryRefreshContinuousFailureCount.set(0);
+                        } else if (e.isConnectInactive()) {
+                            syncRefreshMetadata(false);
+                            tableEntryRefreshContinuousFailureCount.set(0);
+                        }
+                    } catch (Throwable t) {
+                        RUNTIME.error("getOrRefreshTableEntry meet exception", t);
+                        throw t;
                     }
                 }
                 return indexInfo;

src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java

@@ -1050,6 +1050,10 @@ public static ObIndexInfo getIndexInfoFromRemote(ObServerAddr obServerAddr, ObUs
             } else {
                 throw new ObTableEntryRefreshException("index is not exist");
             }
+        } catch (SQLException e) {
+            // cannot execute sql, maybe some of the observers have been killed
+            RUNTIME.error(LCD.convert("01-00010"), indexTableName, e.getMessage());
+            throw new ObTableEntryRefreshException("fail to get index info from remote", e, true);
         } catch (Exception e) {
             if (e instanceof ObTableEntryRefreshException) {
                 throw new ObTableEntryRefreshException(format(