fix(database): database sync failed after updated an invalid datasource to a valid datasource (#2382)

* fix dead lock

* response to cr comment

* update submodule

* forma the code

---------

Co-authored-by: yh263208 <yh263208@oceanbase.com>

Author: MarkPotato777

client

@@ -30,7 +30,13 @@
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.locks.Lock;
 import java.util.stream.Collectors;
 
@@ -433,7 +439,12 @@ private void syncTeamDataSources(ConnectionConfig connection) {
         Long currentProjectId = connection.getProjectId();
         List<String> blockedDatabaseNames = listBlockedDatabaseNames(connection.getDialectType());
         DataSource teamDataSource = new OBConsoleDataSourceFactory(connection, true, false).getDataSource();
-        try (Connection conn = teamDataSource.getConnection()) {
+        ExecutorService executorService = Executors.newFixedThreadPool(1);
+        Future<Connection> connectionFuture = executorService.submit(
+                (Callable<Connection>) teamDataSource::getConnection);
+        Connection conn = null;
+        try {
+            conn = connectionFuture.get(5, TimeUnit.SECONDS);
             List<DatabaseEntity> latestDatabases = dbSchemaService.listDatabases(connection.getDialectType(), conn)
                     .stream().map(database -> {
                         DatabaseEntity entity = new DatabaseEntity();
@@ -512,16 +523,32 @@ private void syncTeamDataSources(ConnectionConfig connection) {
                         "update connect_database set table_count=?, collation_name=?, charset_name=?, project_id=? where id = ?";
                 jdbcTemplate.batchUpdate(update, toUpdate);
             }
-        } catch (SQLException e) {
-            deleteDatabaseIfClusterNotExists(e, connection.getId(),
-                    "update connect_database set is_existed = 0 where connection_id=?");
-            throw new IllegalStateException(e);
+        } catch (ExecutionException | InterruptedException | TimeoutException e) {
+            log.warn("Failed to obtain the connection, errorMessage={}", e.getMessage());
+            Throwable rootCause = e.getCause();
+            if (rootCause instanceof SQLException) {
+                deleteDatabaseIfClusterNotExists((SQLException) rootCause,
+                        connection.getId(), "update connect_database set is_existed = 0 where connection_id=?");
+                throw new IllegalStateException(rootCause);
+            }
         } finally {
+            try {
+                executorService.shutdownNow();
+            } catch (Exception e) {
+                // eat the exception
+            }
+            if (conn != null) {
+                try {
+                    conn.close();
+                } catch (Exception e) {
+                    log.warn("Close connection failed, errorMessage={}", e.getMessage());
+                }
+            }
             if (teamDataSource instanceof AutoCloseable) {
                 try {
                     ((AutoCloseable) teamDataSource).close();
                 } catch (Exception e) {
-                    log.warn("Failed to close datasource", e);
+                    log.warn("Failed to close datasource, errorMessgae={}", e.getMessage());
                 }
             }
         }
@@ -542,7 +569,12 @@ private Long getProjectId(DatabaseEntity database, Long currentProjectId, List<S
 
     private void syncIndividualDataSources(ConnectionConfig connection) {
         DataSource individualDataSource = new OBConsoleDataSourceFactory(connection, true, false).getDataSource();
-        try (Connection conn = individualDataSource.getConnection()) {
+        ExecutorService executorService = Executors.newFixedThreadPool(1);
+        Future<Connection> connectionFuture = executorService.submit(
+                (Callable<Connection>) individualDataSource::getConnection);
+        Connection conn = null;
+        try {
+            conn = connectionFuture.get(5, TimeUnit.SECONDS);
             Set<String> latestDatabaseNames = dbSchemaService.showDatabases(connection.getDialectType(), conn);
             List<DatabaseEntity> existedDatabasesInDb =
                     databaseRepository.findByConnectionId(connection.getId()).stream()
@@ -578,16 +610,32 @@ private void syncIndividualDataSources(ConnectionConfig connection) {
             if (!CollectionUtils.isEmpty(toDelete)) {
                 jdbcTemplate.batchUpdate("delete from connect_database where id = ?", toDelete);
             }
-        } catch (SQLException e) {
-            deleteDatabaseIfClusterNotExists(e, connection.getId(),
-                    "delete from connect_database where connection_id=?");
-            throw new IllegalStateException(e);
+        } catch (ExecutionException | InterruptedException | TimeoutException e) {
+            log.warn("Failed to obtain the connection, errorMessage={}", e.getMessage());
+            Throwable rootCause = e.getCause();
+            if (rootCause instanceof SQLException) {
+                deleteDatabaseIfClusterNotExists((SQLException) rootCause,
+                        connection.getId(), "delete from connect_database where connection_id=?");
+                throw new IllegalStateException(rootCause);
+            }
         } finally {
+            try {
+                executorService.shutdownNow();
+            } catch (Exception e) {
+                // eat the exception
+            }
+            if (conn != null) {
+                try {
+                    conn.close();
+                } catch (Exception e) {
+                    log.warn("Close connection failed, errorMessage={}", e.getMessage());
+                }
+            }
             if (individualDataSource instanceof AutoCloseable) {
                 try {
                     ((AutoCloseable) individualDataSource).close();
                 } catch (Exception e) {
-                    log.warn("Failed to close datasource", e);
+                    log.warn("Failed to close datasource, errorMessgae={}", e.getMessage());
                 }
             }
         }