diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index 8c3caef563d963..9420ca95d8aac7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -33,6 +33,7 @@ import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TStorageMedium; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -54,6 +55,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -68,6 +71,7 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { private Map idToTable; private Map idToPartition; private Map idToRecycleTime; + private ReadWriteLock lock = new ReentrantReadWriteLock(); // Caches below to avoid calculate meta with same name every demon run cycle. // When the meta is updated, these caches should be updated too. No need to @@ -85,13 +89,14 @@ public class CatalogRecycleBin extends MasterDaemon implements Writable { public CatalogRecycleBin() { super("recycle bin", FeConstants.runningUnitTest ? 10L : DEFAULT_INTERVAL_SECONDS * 1000L); - idToDatabase = Maps.newHashMap(); - idToTable = Maps.newHashMap(); - idToPartition = Maps.newHashMap(); - idToRecycleTime = Maps.newHashMap(); + idToDatabase = Maps.newConcurrentMap(); + idToTable = Maps.newConcurrentMap(); + idToPartition = Maps.newConcurrentMap(); + idToRecycleTime = Maps.newConcurrentMap(); } - public synchronized boolean allTabletsInRecycledStatus(List backendTabletIds) { + @VisibleForTesting + public boolean allTabletsInRecycledStatus(List backendTabletIds) { Set recycledTabletSet = Sets.newHashSet(); Iterator> iterator = idToPartition.entrySet().iterator(); @@ -141,117 +146,132 @@ private void addRecycledTabletsForPartition(Set recycledTabletSet, Partiti } } - public synchronized boolean recycleDatabase(Database db, Set tableNames, Set tableIds, + public boolean recycleDatabase(Database db, Set tableNames, Set tableIds, boolean isReplay, boolean isForceDrop, long replayRecycleTime) { - long recycleTime = 0; - if (idToDatabase.containsKey(db.getId())) { - LOG.error("db[{}] already in recycle bin.", db.getId()); - return false; - } - - // db should be empty. all tables are recycled before - if (!db.getTableIds().isEmpty()) { - throw new IllegalStateException("Database " + db.getFullName() + " is not empty. Contains tables: " - + db.getTableIds().stream().collect(Collectors.toSet())); - } - - // recycle db - RecycleDatabaseInfo databaseInfo = new RecycleDatabaseInfo(db, tableNames, tableIds); - idToDatabase.put(db.getId(), databaseInfo); - if (isForceDrop) { - // The 'force drop' database should be recycle immediately. - recycleTime = 0; - } else if (!isReplay || replayRecycleTime == 0) { - recycleTime = System.currentTimeMillis(); - } else { - recycleTime = replayRecycleTime; - } - idToRecycleTime.put(db.getId(), recycleTime); - dbNameToIds.computeIfAbsent(db.getFullName(), k -> ConcurrentHashMap.newKeySet()).add(db.getId()); - LOG.info("recycle db[{}-{}], is force drop: {}", db.getId(), db.getFullName(), isForceDrop); - return true; + lock.writeLock().lock(); + try { + long recycleTime = 0; + if (idToDatabase.containsKey(db.getId())) { + LOG.error("db[{}] already in recycle bin.", db.getId()); + return false; + } + + // db should be empty. all tables are recycled before + if (!db.getTableIds().isEmpty()) { + throw new IllegalStateException("Database " + db.getFullName() + " is not empty. Contains tables: " + + db.getTableIds().stream().collect(Collectors.toSet())); + } + + // recycle db + RecycleDatabaseInfo databaseInfo = new RecycleDatabaseInfo(db, tableNames, tableIds); + idToDatabase.put(db.getId(), databaseInfo); + if (isForceDrop) { + // The 'force drop' database should be recycle immediately. + recycleTime = 0; + } else if (!isReplay || replayRecycleTime == 0) { + recycleTime = System.currentTimeMillis(); + } else { + recycleTime = replayRecycleTime; + } + idToRecycleTime.put(db.getId(), recycleTime); + dbNameToIds.computeIfAbsent(db.getFullName(), k -> ConcurrentHashMap.newKeySet()).add(db.getId()); + LOG.info("recycle db[{}-{}], is force drop: {}", db.getId(), db.getFullName(), isForceDrop); + return true; + } finally { + lock.writeLock().unlock(); + } } - public synchronized boolean recycleTable(long dbId, Table table, boolean isReplay, + public boolean recycleTable(long dbId, Table table, boolean isReplay, boolean isForceDrop, long replayRecycleTime) { - long recycleTime = 0; - if (idToTable.containsKey(table.getId())) { - LOG.error("table[{}] already in recycle bin.", table.getId()); - return false; - } - - // recycle table - RecycleTableInfo tableInfo = new RecycleTableInfo(dbId, table); - if (isForceDrop) { - // The 'force drop' table should be recycle immediately. - recycleTime = 0; - } else if (!isReplay || replayRecycleTime == 0) { - recycleTime = System.currentTimeMillis(); - } else { - recycleTime = replayRecycleTime; - } - idToRecycleTime.put(table.getId(), recycleTime); - idToTable.put(table.getId(), tableInfo); - dbIdTableNameToIds.computeIfAbsent(Pair.of(dbId, table.getName()), - k -> ConcurrentHashMap.newKeySet()).add(table.getId()); - LOG.info("recycle table[{}-{}], is force drop: {}", table.getId(), table.getName(), isForceDrop); - return true; + lock.writeLock().lock(); + try { + long recycleTime = 0; + if (idToTable.containsKey(table.getId())) { + LOG.error("table[{}] already in recycle bin.", table.getId()); + return false; + } + + // recycle table + RecycleTableInfo tableInfo = new RecycleTableInfo(dbId, table); + if (isForceDrop) { + // The 'force drop' table should be recycle immediately. + recycleTime = 0; + } else if (!isReplay || replayRecycleTime == 0) { + recycleTime = System.currentTimeMillis(); + } else { + recycleTime = replayRecycleTime; + } + idToRecycleTime.put(table.getId(), recycleTime); + idToTable.put(table.getId(), tableInfo); + dbIdTableNameToIds.computeIfAbsent(Pair.of(dbId, table.getName()), + k -> ConcurrentHashMap.newKeySet()).add(table.getId()); + LOG.info("recycle table[{}-{}], is force drop: {}", table.getId(), table.getName(), isForceDrop); + return true; + } finally { + lock.writeLock().unlock(); + } } - public synchronized boolean recyclePartition(long dbId, long tableId, String tableName, Partition partition, + public boolean recyclePartition(long dbId, long tableId, String tableName, Partition partition, Range range, PartitionItem listPartitionItem, DataProperty dataProperty, ReplicaAllocation replicaAlloc, boolean isInMemory, boolean isMutable) { - if (idToPartition.containsKey(partition.getId())) { - LOG.error("partition[{}] already in recycle bin.", partition.getId()); - return false; - } - - // recycle partition - RecyclePartitionInfo partitionInfo = new RecyclePartitionInfo(dbId, tableId, partition, - range, listPartitionItem, dataProperty, replicaAlloc, isInMemory, isMutable); - idToRecycleTime.put(partition.getId(), System.currentTimeMillis()); - idToPartition.put(partition.getId(), partitionInfo); - dbTblIdPartitionNameToIds.computeIfAbsent(Pair.of(dbId, tableId), k -> new ConcurrentHashMap<>()) - .computeIfAbsent(partition.getName(), k -> ConcurrentHashMap.newKeySet()).add(partition.getId()); - LOG.info("recycle partition[{}-{}] of table [{}-{}]", partition.getId(), partition.getName(), - tableId, tableName); - return true; + lock.writeLock().lock(); + try { + if (idToPartition.containsKey(partition.getId())) { + LOG.error("partition[{}] already in recycle bin.", partition.getId()); + return false; + } + + // recycle partition + RecyclePartitionInfo partitionInfo = new RecyclePartitionInfo(dbId, tableId, partition, + range, listPartitionItem, dataProperty, replicaAlloc, isInMemory, isMutable); + idToRecycleTime.put(partition.getId(), System.currentTimeMillis()); + idToPartition.put(partition.getId(), partitionInfo); + dbTblIdPartitionNameToIds.computeIfAbsent(Pair.of(dbId, tableId), k -> new ConcurrentHashMap<>()) + .computeIfAbsent(partition.getName(), k -> ConcurrentHashMap.newKeySet()).add(partition.getId()); + LOG.info("recycle partition[{}-{}] of table [{}-{}]", partition.getId(), partition.getName(), + tableId, tableName); + return true; + } finally { + lock.writeLock().unlock(); + } } - public synchronized Long getRecycleTimeById(long id) { + public Long getRecycleTimeById(long id) { return idToRecycleTime.get(id); } - public synchronized void setRecycleTimeByIdForReplay(long id, Long recycleTime) { + public void setRecycleTimeByIdForReplay(long id, Long recycleTime) { idToRecycleTime.put(id, recycleTime); } - public synchronized boolean isRecycleDatabase(long dbId) { + public boolean isRecycleDatabase(long dbId) { return idToDatabase.containsKey(dbId); } - public synchronized boolean isRecycleTable(long dbId, long tableId) { + public boolean isRecycleTable(long dbId, long tableId) { return isRecycleDatabase(dbId) || idToTable.containsKey(tableId); } - public synchronized boolean isRecyclePartition(long dbId, long tableId, long partitionId) { + public boolean isRecyclePartition(long dbId, long tableId, long partitionId) { return isRecycleTable(dbId, tableId) || idToPartition.containsKey(partitionId); } - public synchronized void getRecycleIds(Set dbIds, Set tableIds, Set partitionIds) { + public void getRecycleIds(Set dbIds, Set tableIds, Set partitionIds) { dbIds.addAll(idToDatabase.keySet()); tableIds.addAll(idToTable.keySet()); partitionIds.addAll(idToPartition.keySet()); } - private synchronized boolean isExpire(long id, long currentTimeMs) { + private boolean isExpire(long id, long currentTimeMs) { long latency = currentTimeMs - idToRecycleTime.get(id); return (Config.catalog_trash_ignore_min_erase_latency || latency > minEraseLatency) && latency > Config.catalog_trash_expire_second * 1000L; } - private synchronized void eraseDatabase(long currentTimeMs, int keepNum) { + private void eraseDatabase(long currentTimeMs, int keepNum) { int eraseNum = 0; StopWatch watch = StopWatch.createStarted(); try { @@ -262,18 +282,23 @@ private synchronized void eraseDatabase(long currentTimeMs, int keepNum) { RecycleDatabaseInfo dbInfo = entry.getValue(); Database db = dbInfo.getDb(); if (isExpire(db.getId(), currentTimeMs)) { - // erase db - dbIter.remove(); - idToRecycleTime.remove(entry.getKey()); - - dbNameToIds.computeIfPresent(db.getFullName(), (k, v) -> { - v.remove(db.getId()); - return v.isEmpty() ? null : v; - }); + lock.writeLock().lock(); + try { + // erase db + dbIter.remove(); + idToRecycleTime.remove(entry.getKey()); + + dbNameToIds.computeIfPresent(db.getFullName(), (k, v) -> { + v.remove(db.getId()); + return v.isEmpty() ? null : v; + }); - Env.getCurrentEnv().eraseDatabase(db.getId(), true); - LOG.info("erase db[{}]", db.getId()); - eraseNum++; + Env.getCurrentEnv().eraseDatabase(db.getId(), true); + LOG.info("erase db[{}]", db.getId()); + eraseNum++; + } finally { + lock.writeLock().unlock(); + } } } // 2. erase exceed number @@ -290,29 +315,35 @@ private synchronized void eraseDatabase(long currentTimeMs, int keepNum) { } } - private synchronized void eraseDatabaseWithSameName(String dbName, long currentTimeMs, + private void eraseDatabaseWithSameName(String dbName, long currentTimeMs, int maxSameNameTrashNum, List sameNameDbIdList) { List dbIdToErase = getIdListToEraseByRecycleTime(sameNameDbIdList, maxSameNameTrashNum); for (Long dbId : dbIdToErase) { - RecycleDatabaseInfo dbInfo = idToDatabase.get(dbId); - if (!isExpireMinLatency(dbId, currentTimeMs)) { - continue; - } - eraseAllTables(dbInfo); - idToDatabase.remove(dbId); - idToRecycleTime.remove(dbId); + RecycleDatabaseInfo dbInfo; + lock.writeLock().lock(); + try { + dbInfo = idToDatabase.get(dbId); + if (!isExpireMinLatency(dbId, currentTimeMs)) { + continue; + } + idToDatabase.remove(dbId); + idToRecycleTime.remove(dbId); - dbNameToIds.computeIfPresent(dbName, (k, v) -> { - v.remove(dbId); - return v.isEmpty() ? null : v; - }); + dbNameToIds.computeIfPresent(dbName, (k, v) -> { + v.remove(dbId); + return v.isEmpty() ? null : v; + }); - Env.getCurrentEnv().eraseDatabase(dbId, true); - LOG.info("erase database[{}] name: {}", dbId, dbName); + Env.getCurrentEnv().eraseDatabase(dbId, true); + LOG.info("erase database[{}] name: {}", dbId, dbName); + } finally { + lock.writeLock().unlock(); + } + eraseAllTables(dbInfo); } } - private synchronized boolean isExpireMinLatency(long id, long currentTimeMs) { + private boolean isExpireMinLatency(long id, long currentTimeMs) { return (currentTimeMs - idToRecycleTime.get(id)) > minEraseLatency || FeConstants.runningUnitTest; } @@ -334,21 +365,26 @@ private void eraseAllTables(RecycleDatabaseInfo dbInfo) { if (table.isManagedTable()) { Env.getCurrentEnv().onEraseOlapTable(dbId, (OlapTable) table, false); } - iterator.remove(); - idToRecycleTime.remove(table.getId()); - tableNames.remove(table.getName()); + lock.writeLock().lock(); + try { + iterator.remove(); + idToRecycleTime.remove(table.getId()); + tableNames.remove(table.getName()); - dbIdTableNameToIds.computeIfPresent(Pair.of(tableInfo.getDbId(), table.getName()), (k, v) -> { - v.remove(table.getId()); - return v.isEmpty() ? null : v; - }); + dbIdTableNameToIds.computeIfPresent(Pair.of(tableInfo.getDbId(), table.getName()), (k, v) -> { + v.remove(table.getId()); + return v.isEmpty() ? null : v; + }); - Env.getCurrentEnv().getEditLog().logEraseTable(table.getId()); - LOG.info("erase db[{}] with table[{}]: {}", dbId, table.getId(), table.getName()); + Env.getCurrentEnv().getEditLog().logEraseTable(table.getId()); + LOG.info("erase db[{}] with table[{}]: {}", dbId, table.getId(), table.getName()); + } finally { + lock.writeLock().unlock(); + } } } - public synchronized void replayEraseDatabase(long dbId) { + public void replayEraseDatabase(long dbId) { RecycleDatabaseInfo dbInfo = idToDatabase.remove(dbId); idToRecycleTime.remove(dbId); @@ -363,7 +399,7 @@ public synchronized void replayEraseDatabase(long dbId) { LOG.info("replay erase db[{}]", dbId); } - private synchronized void eraseTable(long currentTimeMs, int keepNum) { + private void eraseTable(long currentTimeMs, int keepNum) { int eraseNum = 0; StopWatch watch = StopWatch.createStarted(); try { @@ -380,20 +416,25 @@ private synchronized void eraseTable(long currentTimeMs, int keepNum) { Env.getCurrentEnv().onEraseOlapTable(tableInfo.dbId, (OlapTable) table, false); } - // erase table - tableIter.remove(); - idToRecycleTime.remove(tableId); + lock.writeLock().lock(); + try { + // erase table + tableIter.remove(); + idToRecycleTime.remove(tableId); - dbIdTableNameToIds.computeIfPresent(Pair.of(tableInfo.getDbId(), table.getName()), - (k, v) -> { - v.remove(tableId); - return v.isEmpty() ? null : v; - }); + dbIdTableNameToIds.computeIfPresent(Pair.of(tableInfo.getDbId(), table.getName()), + (k, v) -> { + v.remove(tableId); + return v.isEmpty() ? null : v; + }); - // log - Env.getCurrentEnv().getEditLog().logEraseTable(tableId); - LOG.info("erase table[{}]", tableId); - eraseNum++; + // log + Env.getCurrentEnv().getEditLog().logEraseTable(tableId); + LOG.info("erase table[{}]", tableId); + eraseNum++; + } finally { + lock.writeLock().unlock(); + } } } // end for tables @@ -411,7 +452,7 @@ private synchronized void eraseTable(long currentTimeMs, int keepNum) { } } - private synchronized void eraseTableWithSameName(long dbId, String tableName, long currentTimeMs, + private void eraseTableWithSameName(long dbId, String tableName, long currentTimeMs, int maxSameNameTrashNum, List sameNameTableIdList) { List tableIdToErase = getIdListToEraseByRecycleTime(sameNameTableIdList, maxSameNameTrashNum); for (Long tableId : tableIdToErase) { @@ -424,20 +465,25 @@ private synchronized void eraseTableWithSameName(long dbId, String tableName, lo Env.getCurrentEnv().onEraseOlapTable(dbId, (OlapTable) table, false); } - idToTable.remove(tableId); - idToRecycleTime.remove(tableId); + lock.writeLock().lock(); + try { + idToTable.remove(tableId); + idToRecycleTime.remove(tableId); - dbIdTableNameToIds.computeIfPresent(Pair.of(dbId, tableName), (k, v) -> { - v.remove(tableId); - return v.isEmpty() ? null : v; - }); + dbIdTableNameToIds.computeIfPresent(Pair.of(dbId, tableName), (k, v) -> { + v.remove(tableId); + return v.isEmpty() ? null : v; + }); - Env.getCurrentEnv().getEditLog().logEraseTable(tableId); - LOG.info("erase table[{}] name: {} from db[{}]", tableId, tableName, dbId); + Env.getCurrentEnv().getEditLog().logEraseTable(tableId); + LOG.info("erase table[{}] name: {} from db[{}]", tableId, tableName, dbId); + } finally { + lock.writeLock().unlock(); + } } } - public synchronized void replayEraseTable(long tableId) { + public void replayEraseTable(long tableId) { LOG.info("before replay erase table[{}]", tableId); RecycleTableInfo tableInfo = idToTable.remove(tableId); idToRecycleTime.remove(tableId); @@ -460,7 +506,7 @@ public synchronized void replayEraseTable(long tableId) { LOG.info("replay erase table[{}]", tableId); } - private synchronized void erasePartition(long currentTimeMs, int keepNum) { + private void erasePartition(long currentTimeMs, int keepNum) { int eraseNum = 0; StopWatch watch = StopWatch.createStarted(); try { @@ -473,23 +519,28 @@ private synchronized void erasePartition(long currentTimeMs, int keepNum) { long partitionId = entry.getKey(); if (isExpire(partitionId, currentTimeMs)) { - Env.getCurrentEnv().onErasePartition(partition); - // erase partition - iterator.remove(); - idToRecycleTime.remove(partitionId); - - dbTblIdPartitionNameToIds.computeIfPresent( - Pair.of(partitionInfo.getDbId(), partitionInfo.getTableId()), (pair, partitionMap) -> { - partitionMap.computeIfPresent(partition.getName(), (name, idSet) -> { - idSet.remove(partitionId); - return idSet.isEmpty() ? null : idSet; + lock.writeLock().lock(); + try { + Env.getCurrentEnv().onErasePartition(partition); + // erase partition + iterator.remove(); + idToRecycleTime.remove(partitionId); + + dbTblIdPartitionNameToIds.computeIfPresent( + Pair.of(partitionInfo.getDbId(), partitionInfo.getTableId()), (pair, partitionMap) -> { + partitionMap.computeIfPresent(partition.getName(), (name, idSet) -> { + idSet.remove(partitionId); + return idSet.isEmpty() ? null : idSet; + }); + return partitionMap.isEmpty() ? null : partitionMap; }); - return partitionMap.isEmpty() ? null : partitionMap; - }); - // log - Env.getCurrentEnv().getEditLog().logErasePartition(partitionId); - LOG.info("erase partition[{}]. reason: expired", partitionId); - eraseNum++; + // log + Env.getCurrentEnv().getEditLog().logErasePartition(partitionId); + LOG.info("erase partition[{}]. reason: expired", partitionId); + eraseNum++; + } finally { + lock.writeLock().unlock(); + } } } // end for partitions @@ -511,36 +562,40 @@ private synchronized void erasePartition(long currentTimeMs, int keepNum) { } } - private synchronized void erasePartitionWithSameName(long dbId, long tableId, String partitionName, + private void erasePartitionWithSameName(long dbId, long tableId, String partitionName, long currentTimeMs, int maxSameNameTrashNum, List sameNamePartitionIdList) { List partitionIdToErase = getIdListToEraseByRecycleTime(sameNamePartitionIdList, maxSameNameTrashNum); for (Long partitionId : partitionIdToErase) { - RecyclePartitionInfo partitionInfo = idToPartition.get(partitionId); - if (!isExpireMinLatency(partitionId, currentTimeMs)) { - continue; - } - Partition partition = partitionInfo.getPartition(); - - Env.getCurrentEnv().onErasePartition(partition); - idToPartition.remove(partitionId); - idToRecycleTime.remove(partitionId); + lock.writeLock().lock(); + try { + RecyclePartitionInfo partitionInfo = idToPartition.get(partitionId); + if (!isExpireMinLatency(partitionId, currentTimeMs)) { + continue; + } + Partition partition = partitionInfo.getPartition(); + Env.getCurrentEnv().onErasePartition(partition); + idToPartition.remove(partitionId); + idToRecycleTime.remove(partitionId); - dbTblIdPartitionNameToIds.computeIfPresent(Pair.of(dbId, tableId), (pair, partitionMap) -> { - partitionMap.computeIfPresent(partitionName, (name, idSet) -> { - idSet.remove(partitionId); - return idSet.isEmpty() ? null : idSet; + dbTblIdPartitionNameToIds.computeIfPresent(Pair.of(dbId, tableId), (pair, partitionMap) -> { + partitionMap.computeIfPresent(partitionName, (name, idSet) -> { + idSet.remove(partitionId); + return idSet.isEmpty() ? null : idSet; + }); + return partitionMap.isEmpty() ? null : partitionMap; }); - return partitionMap.isEmpty() ? null : partitionMap; - }); - Env.getCurrentEnv().getEditLog().logErasePartition(partitionId); + Env.getCurrentEnv().getEditLog().logErasePartition(partitionId); + } finally { + lock.writeLock().unlock(); + } LOG.info("erase partition[{}] name: {} from table[{}] from db[{}]", partitionId, partitionName, tableId, dbId); } } - public synchronized void replayErasePartition(long partitionId) { + public void replayErasePartition(long partitionId) { RecyclePartitionInfo partitionInfo = idToPartition.remove(partitionId); idToRecycleTime.remove(partitionId); @@ -564,7 +619,7 @@ public synchronized void replayErasePartition(long partitionId) { LOG.info("replay erase partition[{}]", partitionId); } - private synchronized List getIdListToEraseByRecycleTime(List ids, int maxTrashNum) { + private List getIdListToEraseByRecycleTime(List ids, int maxTrashNum) { List idToErase = Lists.newArrayList(); if (ids.size() <= maxTrashNum) { return idToErase; @@ -578,48 +633,53 @@ private synchronized List getIdListToEraseByRecycleTime(List ids, in return idToErase; } - public synchronized Database recoverDatabase(String dbName, long dbId) throws DdlException { - RecycleDatabaseInfo dbInfo = null; - // The recycle time of the force dropped tables and databases will be set to zero, use 1 here to - // skip these databases and tables. - long recycleTime = 1; - Iterator> iterator = idToDatabase.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - if (dbName.equals(entry.getValue().getDb().getFullName())) { - if (dbId == -1) { - if (recycleTime <= idToRecycleTime.get(entry.getKey())) { - recycleTime = idToRecycleTime.get(entry.getKey()); + public Database recoverDatabase(String dbName, long dbId) throws DdlException { + lock.writeLock().lock(); + try { + RecycleDatabaseInfo dbInfo = null; + // The recycle time of the force dropped tables and databases will be set to zero, use 1 here to + // skip these databases and tables. + long recycleTime = 1; + Iterator> iterator = idToDatabase.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (dbName.equals(entry.getValue().getDb().getFullName())) { + if (dbId == -1) { + if (recycleTime <= idToRecycleTime.get(entry.getKey())) { + recycleTime = idToRecycleTime.get(entry.getKey()); + dbInfo = entry.getValue(); + } + } else if (entry.getKey() == dbId) { dbInfo = entry.getValue(); + break; } - } else if (entry.getKey() == dbId) { - dbInfo = entry.getValue(); - break; } } - } - if (dbInfo == null) { - throw new DdlException("Unknown database '" + dbName + "' or database id '" + dbId + "'"); - } + if (dbInfo == null) { + throw new DdlException("Unknown database '" + dbName + "' or database id '" + dbId + "'"); + } - // 1. recover all tables in this db - recoverAllTables(dbInfo); + // 1. recover all tables in this db + recoverAllTables(dbInfo); - Database db = dbInfo.getDb(); - // 2. remove db from idToDatabase and idToRecycleTime - idToDatabase.remove(db.getId()); - idToRecycleTime.remove(db.getId()); + Database db = dbInfo.getDb(); + // 2. remove db from idToDatabase and idToRecycleTime + idToDatabase.remove(db.getId()); + idToRecycleTime.remove(db.getId()); - dbNameToIds.computeIfPresent(dbInfo.getDb().getFullName(), (k, v) -> { - v.remove(dbId); - return v.isEmpty() ? null : v; - }); + dbNameToIds.computeIfPresent(dbInfo.getDb().getFullName(), (k, v) -> { + v.remove(dbId); + return v.isEmpty() ? null : v; + }); - return db; + return db; + } finally { + lock.writeLock().unlock(); + } } - public synchronized Database replayRecoverDatabase(long dbId) { + public Database replayRecoverDatabase(long dbId) { RecycleDatabaseInfo dbInfo = idToDatabase.get(dbId); try { @@ -641,88 +701,98 @@ public synchronized Database replayRecoverDatabase(long dbId) { } private void recoverAllTables(RecycleDatabaseInfo dbInfo) throws DdlException { - Database db = dbInfo.getDb(); - Set tableNames = Sets.newHashSet(dbInfo.getTableNames()); - Set tableIds = Sets.newHashSet(dbInfo.getTableIds()); - long dbId = db.getId(); - Iterator> iterator = idToTable.entrySet().iterator(); - while (iterator.hasNext() && !tableNames.isEmpty()) { - Map.Entry entry = iterator.next(); - RecycleTableInfo tableInfo = entry.getValue(); - if (tableInfo.getDbId() != dbId || !tableNames.contains(tableInfo.getTable().getName()) - || !tableIds.contains(tableInfo.getTable().getId())) { - continue; - } + lock.writeLock().lock(); + try { + Database db = dbInfo.getDb(); + Set tableNames = Sets.newHashSet(dbInfo.getTableNames()); + Set tableIds = Sets.newHashSet(dbInfo.getTableIds()); + long dbId = db.getId(); + Iterator> iterator = idToTable.entrySet().iterator(); + while (iterator.hasNext() && !tableNames.isEmpty()) { + Map.Entry entry = iterator.next(); + RecycleTableInfo tableInfo = entry.getValue(); + if (tableInfo.getDbId() != dbId || !tableNames.contains(tableInfo.getTable().getName()) + || !tableIds.contains(tableInfo.getTable().getId())) { + continue; + } - Table table = tableInfo.getTable(); - if (table.getType() == TableType.OLAP) { - db.registerTable(table); - LOG.info("recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName()); - } else { - LOG.info("ignore recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName()); - } - iterator.remove(); - idToRecycleTime.remove(table.getId()); - tableNames.remove(table.getName()); + Table table = tableInfo.getTable(); + if (table.getType() == TableType.OLAP) { + db.registerTable(table); + LOG.info("recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName()); + } else { + LOG.info("ignore recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName()); + } + iterator.remove(); + idToRecycleTime.remove(table.getId()); + tableNames.remove(table.getName()); - dbIdTableNameToIds.computeIfPresent(Pair.of(dbId, table.getName()), (k, v) -> { - v.remove(table.getId()); - return v.isEmpty() ? null : v; - }); - } + dbIdTableNameToIds.computeIfPresent(Pair.of(dbId, table.getName()), (k, v) -> { + v.remove(table.getId()); + return v.isEmpty() ? null : v; + }); + } - if (!tableNames.isEmpty()) { - throw new DdlException("Tables[" + tableNames + "] is missing. Can not recover db"); + if (!tableNames.isEmpty()) { + throw new DdlException("Tables[" + tableNames + "] is missing. Can not recover db"); + } + } finally { + lock.writeLock().unlock(); } } - public synchronized boolean recoverTable(Database db, String tableName, long tableId, + public boolean recoverTable(Database db, String tableName, long tableId, String newTableName) throws DdlException { - // make sure to get db lock - Table table = null; - // The recycle time of the force dropped tables and databases will be set to zero, use 1 here to - // skip these databases and tables. - long recycleTime = 1; - long dbId = db.getId(); - Iterator> iterator = idToTable.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - RecycleTableInfo tableInfo = entry.getValue(); - if (tableInfo.getDbId() != dbId) { - continue; - } + lock.writeLock().lock(); + try { + // make sure to get db lock + Table table = null; + // The recycle time of the force dropped tables and databases will be set to zero, use 1 here to + // skip these databases and tables. + long recycleTime = 1; + long dbId = db.getId(); + Iterator> iterator = idToTable.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + RecycleTableInfo tableInfo = entry.getValue(); + if (tableInfo.getDbId() != dbId) { + continue; + } - if (!tableInfo.getTable().getName().equals(tableName)) { - continue; - } + if (!tableInfo.getTable().getName().equals(tableName)) { + continue; + } - if (tableId == -1) { - if (recycleTime <= idToRecycleTime.get(entry.getKey())) { - recycleTime = idToRecycleTime.get(entry.getKey()); + if (tableId == -1) { + if (recycleTime <= idToRecycleTime.get(entry.getKey())) { + recycleTime = idToRecycleTime.get(entry.getKey()); + table = tableInfo.getTable(); + } + } else if (entry.getKey() == tableId) { table = tableInfo.getTable(); + break; } - } else if (entry.getKey() == tableId) { - table = tableInfo.getTable(); - break; } - } - if (table == null) { - throw new DdlException("Unknown table '" + tableName + "' or table id '" + tableId + "' in " - + db.getFullName()); - } + if (table == null) { + throw new DdlException("Unknown table '" + tableName + "' or table id '" + tableId + "' in " + + db.getFullName()); + } - if (table.getType() == TableType.MATERIALIZED_VIEW) { - throw new DdlException("Can not recover materialized view '" + tableName + "' or table id '" - + tableId + "' in " + db.getFullName()); - } + if (table.getType() == TableType.MATERIALIZED_VIEW) { + throw new DdlException("Can not recover materialized view '" + tableName + "' or table id '" + + tableId + "' in " + db.getFullName()); + } - innerRecoverTable(db, table, tableName, newTableName, null, false); - LOG.info("recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName()); - return true; + innerRecoverTable(db, table, tableName, newTableName, null, false); + LOG.info("recover db[{}] with table[{}]: {}", dbId, table.getId(), table.getName()); + return true; + } finally { + lock.writeLock().unlock(); + } } - public synchronized void replayRecoverTable(Database db, long tableId, String newTableName) throws DdlException { + public void replayRecoverTable(Database db, long tableId, String newTableName) throws DdlException { // make sure to get db write lock Iterator> iterator = idToTable.entrySet().iterator(); while (iterator.hasNext()) { @@ -740,7 +810,7 @@ public synchronized void replayRecoverTable(Database db, long tableId, String ne } } - private synchronized boolean innerRecoverTable(Database db, Table table, String tableName, String newTableName, + private boolean innerRecoverTable(Database db, Table table, String tableName, String newTableName, Iterator> iterator, boolean isReplay) throws DdlException { table.writeLock(); @@ -789,120 +859,126 @@ private synchronized boolean innerRecoverTable(Database db, Table table, String if (table.isManagedTable()) { DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), (OlapTable) table, isReplay); } + return true; } finally { table.writeUnlock(); } - return true; } - public synchronized void recoverPartition(long dbId, OlapTable table, String partitionName, + public void recoverPartition(long dbId, OlapTable table, String partitionName, long partitionIdToRecover, String newPartitionName) throws DdlException { - if (table.getType() == TableType.MATERIALIZED_VIEW) { - throw new DdlException("Can not recover partition in materialized view: " + table.getName()); - } + lock.writeLock().lock(); + try { + if (table.getType() == TableType.MATERIALIZED_VIEW) { + throw new DdlException("Can not recover partition in materialized view: " + table.getName()); + } - long recycleTime = -1; - // make sure to get db write lock - RecyclePartitionInfo recoverPartitionInfo = null; + long recycleTime = -1; + // make sure to get db write lock + RecyclePartitionInfo recoverPartitionInfo = null; - Iterator> iterator = idToPartition.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - RecyclePartitionInfo partitionInfo = entry.getValue(); + if (partitionIdToRecover != -1) { + recoverPartitionInfo = idToPartition.get(partitionIdToRecover); + } else { + Iterator> iterator = idToPartition.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + RecyclePartitionInfo partitionInfo = entry.getValue(); - if (partitionInfo.getTableId() != table.getId()) { - continue; - } + if (partitionInfo.getTableId() != table.getId()) { + continue; + } - if (!partitionInfo.getPartition().getName().equalsIgnoreCase(partitionName)) { - continue; - } + if (!partitionInfo.getPartition().getName().equalsIgnoreCase(partitionName)) { + continue; + } - if (partitionIdToRecover == -1) { - if (recycleTime <= idToRecycleTime.get(entry.getKey())) { - recycleTime = idToRecycleTime.get(entry.getKey()); - recoverPartitionInfo = partitionInfo; + if (recycleTime <= idToRecycleTime.get(entry.getKey())) { + recycleTime = idToRecycleTime.get(entry.getKey()); + recoverPartitionInfo = partitionInfo; + } } - } else if (entry.getKey() == partitionIdToRecover) { - recoverPartitionInfo = partitionInfo; - break; } - } - if (recoverPartitionInfo == null) { - throw new DdlException("No partition named '" + partitionName + "' or partition id '" + partitionIdToRecover - + "' in table " + table.getName()); - } + if (recoverPartitionInfo == null) { + throw new DdlException( + "No partition named '" + partitionName + "' or partition id '" + partitionIdToRecover + + "' in table " + table.getName()); + } - PartitionInfo partitionInfo = table.getPartitionInfo(); - PartitionItem recoverItem = null; - if (partitionInfo.getType() == PartitionType.RANGE) { - recoverItem = new RangePartitionItem(recoverPartitionInfo.getRange()); - } else if (partitionInfo.getType() == PartitionType.LIST) { - recoverItem = recoverPartitionInfo.getListPartitionItem(); - } - // check if partition item is invalid - if (partitionInfo.getAnyIntersectItem(recoverItem, false) != null) { - throw new DdlException("Can not recover partition[" + partitionName + "]. Partition item conflict."); - } + PartitionInfo partitionInfo = table.getPartitionInfo(); + PartitionItem recoverItem = null; + if (partitionInfo.getType() == PartitionType.RANGE) { + recoverItem = new RangePartitionItem(recoverPartitionInfo.getRange()); + } else if (partitionInfo.getType() == PartitionType.LIST) { + recoverItem = recoverPartitionInfo.getListPartitionItem(); + } + // check if partition item is invalid + if (partitionInfo.getAnyIntersectItem(recoverItem, false) != null) { + throw new DdlException("Can not recover partition[" + partitionName + "]. Partition item conflict."); + } - // check if schema change - Partition recoverPartition = recoverPartitionInfo.getPartition(); - Set tableIndex = table.getIndexIdToMeta().keySet(); - Set partitionIndex = recoverPartition.getMaterializedIndices(IndexExtState.ALL).stream() - .map(i -> i.getId()).collect(Collectors.toSet()); - if (!tableIndex.equals(partitionIndex)) { - throw new DdlException("table's index not equal with partition's index. table's index=" + tableIndex - + ", partition's index=" + partitionIndex); - } + // check if schema change + Partition recoverPartition = recoverPartitionInfo.getPartition(); + Set tableIndex = table.getIndexIdToMeta().keySet(); + Set partitionIndex = recoverPartition.getMaterializedIndices(IndexExtState.ALL).stream() + .map(i -> i.getId()).collect(Collectors.toSet()); + if (!tableIndex.equals(partitionIndex)) { + throw new DdlException("table's index not equal with partition's index. table's index=" + tableIndex + + ", partition's index=" + partitionIndex); + } - // check if partition name exists - Preconditions.checkState(recoverPartition.getName().equalsIgnoreCase(partitionName)); - if (!Strings.isNullOrEmpty(newPartitionName)) { - if (table.checkPartitionNameExist(newPartitionName)) { - throw new DdlException("Partition name[" + newPartitionName + "] is already used"); + // check if partition name exists + Preconditions.checkState(recoverPartition.getName().equalsIgnoreCase(partitionName)); + if (!Strings.isNullOrEmpty(newPartitionName)) { + if (table.checkPartitionNameExist(newPartitionName)) { + throw new DdlException("Partition name[" + newPartitionName + "] is already used"); + } + recoverPartition.setName(newPartitionName); } - recoverPartition.setName(newPartitionName); - } - // recover partition - table.addPartition(recoverPartition); + // recover partition + table.addPartition(recoverPartition); - // recover partition info - long partitionId = recoverPartition.getId(); - partitionInfo.setItem(partitionId, false, recoverItem); - partitionInfo.setDataProperty(partitionId, recoverPartitionInfo.getDataProperty()); - partitionInfo.setReplicaAllocation(partitionId, recoverPartitionInfo.getReplicaAlloc()); - partitionInfo.setIsInMemory(partitionId, recoverPartitionInfo.isInMemory()); - partitionInfo.setIsMutable(partitionId, recoverPartitionInfo.isMutable()); + // recover partition info + long partitionId = recoverPartition.getId(); + partitionInfo.setItem(partitionId, false, recoverItem); + partitionInfo.setDataProperty(partitionId, recoverPartitionInfo.getDataProperty()); + partitionInfo.setReplicaAllocation(partitionId, recoverPartitionInfo.getReplicaAlloc()); + partitionInfo.setIsInMemory(partitionId, recoverPartitionInfo.isInMemory()); + partitionInfo.setIsMutable(partitionId, recoverPartitionInfo.isMutable()); - // remove from recycle bin - idToPartition.remove(partitionId); - idToRecycleTime.remove(partitionId); + // remove from recycle bin + idToPartition.remove(partitionId); + idToRecycleTime.remove(partitionId); - if (!Env.getCurrentEnv().invalidCacheForCloud()) { - long version = table.getNextVersion(); - table.updateVisibleVersionAndTime(version, System.currentTimeMillis()); - } + if (!Env.getCurrentEnv().invalidCacheForCloud()) { + long version = table.getNextVersion(); + table.updateVisibleVersionAndTime(version, System.currentTimeMillis()); + } - dbTblIdPartitionNameToIds.computeIfPresent( - Pair.of(recoverPartitionInfo.getDbId(), recoverPartitionInfo.getTableId()), (pair, partitionMap) -> { - partitionMap.computeIfPresent(partitionName, (name, idSet) -> { - idSet.remove(recoverPartition.getId()); - return idSet.isEmpty() ? null : idSet; + dbTblIdPartitionNameToIds.computeIfPresent( + Pair.of(recoverPartitionInfo.getDbId(), recoverPartitionInfo.getTableId()), + (pair, partitionMap) -> { + partitionMap.computeIfPresent(partitionName, (name, idSet) -> { + idSet.remove(recoverPartition.getId()); + return idSet.isEmpty() ? null : idSet; + }); + return partitionMap.isEmpty() ? null : partitionMap; }); - return partitionMap.isEmpty() ? null : partitionMap; - }); - // log - RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "", - table.getName(), "", partitionName, newPartitionName); - Env.getCurrentEnv().getEditLog().logRecoverPartition(recoverInfo); - LOG.info("recover partition[{}]", partitionId); + // log + RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(), partitionId, "", + table.getName(), "", partitionName, newPartitionName); + Env.getCurrentEnv().getEditLog().logRecoverPartition(recoverInfo); + LOG.info("recover partition[{}]", partitionId); + } finally { + lock.writeLock().unlock(); + } } // The caller should keep table write lock - public synchronized void replayRecoverPartition(OlapTable table, long partitionId, + public void replayRecoverPartition(OlapTable table, long partitionId, String newPartitionName) throws DdlException { Iterator> iterator = idToPartition.entrySet().iterator(); Env currentEnv = Env.getCurrentEnv(); @@ -959,25 +1035,31 @@ public synchronized void replayRecoverPartition(OlapTable table, long partitionI } // erase database in catalog recycle bin instantly - public synchronized void eraseDatabaseInstantly(long dbId) throws DdlException { + public void eraseDatabaseInstantly(long dbId) throws DdlException { // 1. find dbInfo and erase db - RecycleDatabaseInfo dbInfo = idToDatabase.get(dbId); - if (dbInfo != null) { - // erase db - Env.getCurrentEnv().eraseDatabase(dbId, true); - - // erase db from idToDatabase and idToRecycleTime - idToDatabase.remove(dbId); - idToRecycleTime.remove(dbId); - - dbNameToIds.computeIfPresent(dbInfo.getDb().getFullName(), (k, v) -> { - v.remove(dbId); - return v.isEmpty() ? null : v; - }); + RecycleDatabaseInfo dbInfo; + lock.writeLock().lock(); + try { + dbInfo = idToDatabase.get(dbId); + if (dbInfo != null) { + // erase db + Env.getCurrentEnv().eraseDatabase(dbId, true); + + // erase db from idToDatabase and idToRecycleTime + idToDatabase.remove(dbId); + idToRecycleTime.remove(dbId); + + dbNameToIds.computeIfPresent(dbInfo.getDb().getFullName(), (k, v) -> { + v.remove(dbId); + return v.isEmpty() ? null : v; + }); - // log for erase db - String dbName = dbInfo.getDb().getName(); - LOG.info("erase db[{}]: {}", dbId, dbName); + // log for erase db + String dbName = dbInfo.getDb().getName(); + LOG.info("erase db[{}]: {}", dbId, dbName); + } + } finally { + lock.writeLock().unlock(); } // 2. remove all tables with the same dbId @@ -1015,30 +1097,36 @@ public synchronized void eraseDatabaseInstantly(long dbId) throws DdlException { } // erase table in catalog recycle bin instantly - public synchronized void eraseTableInstantly(long tableId) throws DdlException { - // 1. find tableInfo and erase table - RecycleTableInfo tableInfo = idToTable.get(tableId); - if (tableInfo != null) { - // erase table - long dbId = tableInfo.getDbId(); - Table table = tableInfo.getTable(); - if (table.getType() == TableType.OLAP || table.getType() == TableType.MATERIALIZED_VIEW) { - Env.getCurrentEnv().onEraseOlapTable(dbId, (OlapTable) table, false); - } + public void eraseTableInstantly(long tableId) throws DdlException { + RecycleTableInfo tableInfo; + lock.writeLock().lock(); + try { + // 1. find tableInfo and erase table + tableInfo = idToTable.get(tableId); + if (tableInfo != null) { + // erase table + long dbId = tableInfo.getDbId(); + Table table = tableInfo.getTable(); + if (table.getType() == TableType.OLAP || table.getType() == TableType.MATERIALIZED_VIEW) { + Env.getCurrentEnv().onEraseOlapTable(dbId, (OlapTable) table, false); + } - // erase table from idToTable and idToRecycleTime - idToTable.remove(tableId); - idToRecycleTime.remove(tableId); + // erase table from idToTable and idToRecycleTime + idToTable.remove(tableId); + idToRecycleTime.remove(tableId); - dbIdTableNameToIds.computeIfPresent(Pair.of(dbId, table.getName()), (k, v) -> { - v.remove(tableId); - return v.isEmpty() ? null : v; - }); + dbIdTableNameToIds.computeIfPresent(Pair.of(dbId, table.getName()), (k, v) -> { + v.remove(tableId); + return v.isEmpty() ? null : v; + }); - // log for erase table - String tableName = table.getName(); - Env.getCurrentEnv().getEditLog().logEraseTable(tableId); - LOG.info("erase db[{}]'s table[{}]: {}", dbId, tableId, tableName); + // log for erase table + String tableName = table.getName(); + Env.getCurrentEnv().getEditLog().logEraseTable(tableId); + LOG.info("erase db[{}]'s table[{}]: {}", dbId, tableId, tableName); + } + } finally { + lock.writeLock().unlock(); } // 2. erase all partitions with the same tableId @@ -1062,35 +1150,41 @@ public synchronized void eraseTableInstantly(long tableId) throws DdlException { } // erase partition in catalog recycle bin instantly - public synchronized void erasePartitionInstantly(long partitionId) throws DdlException { + public void erasePartitionInstantly(long partitionId) throws DdlException { // 1. find partitionInfo to erase RecyclePartitionInfo partitionInfo = idToPartition.get(partitionId); if (partitionInfo == null) { throw new DdlException("No partition id '" + partitionId + "'"); } - // 2. erase partition - Partition partition = partitionInfo.getPartition(); - Env.getCurrentEnv().onErasePartition(partition); + lock.writeLock().lock(); + try { - // 3. erase partition in idToPartition and idToRecycleTime - idToPartition.remove(partitionId); - idToRecycleTime.remove(partitionId); + // 2. erase partition + Partition partition = partitionInfo.getPartition(); + Env.getCurrentEnv().onErasePartition(partition); - dbTblIdPartitionNameToIds.computeIfPresent( - Pair.of(partitionInfo.getDbId(), partitionInfo.getTableId()), (pair, partitionMap) -> { - partitionMap.computeIfPresent(partition.getName(), (name, idSet) -> { - idSet.remove(partitionId); - return idSet.isEmpty() ? null : idSet; + // 3. erase partition in idToPartition and idToRecycleTime + idToPartition.remove(partitionId); + idToRecycleTime.remove(partitionId); + + dbTblIdPartitionNameToIds.computeIfPresent( + Pair.of(partitionInfo.getDbId(), partitionInfo.getTableId()), (pair, partitionMap) -> { + partitionMap.computeIfPresent(partition.getName(), (name, idSet) -> { + idSet.remove(partitionId); + return idSet.isEmpty() ? null : idSet; + }); + return partitionMap.isEmpty() ? null : partitionMap; }); - return partitionMap.isEmpty() ? null : partitionMap; - }); - // 4. log for erase partition - long tableId = partitionInfo.getTableId(); - String partitionName = partition.getName(); - Env.getCurrentEnv().getEditLog().logErasePartition(partitionId); - LOG.info("erase table[{}]'s partition[{}]: {}", tableId, partitionId, partitionName); + // 4. log for erase partition + long tableId = partitionInfo.getTableId(); + String partitionName = partition.getName(); + Env.getCurrentEnv().getEditLog().logErasePartition(partitionId); + LOG.info("erase table[{}]'s partition[{}]: {}", tableId, partitionId, partitionName); + } finally { + lock.writeLock().unlock(); + } } // no need to use synchronized. @@ -1191,7 +1285,10 @@ protected void runAfterCatalogReady() { eraseDatabase(currentTimeMs, keepNum); } - public synchronized List> getInfo() { + /** + * only used in show, so we do not lock. + */ + public List> getInfo() { Map> dbToDataSize = new HashMap<>(); List> tableInfos = Lists.newArrayList(); for (Map.Entry entry : idToTable.entrySet()) { @@ -1308,7 +1405,10 @@ public synchronized List> getInfo() { return Stream.of(dbInfos, tableInfos, partitionInfos).flatMap(Collection::stream).collect(Collectors.toList()); } - public synchronized Map> getDbToRecycleSize() { + /** + * only used in show, so we do not lock. + */ + public Map> getDbToRecycleSize() { Map> dbToRecycleSize = new HashMap<>(); for (Map.Entry entry : idToTable.entrySet()) { RecycleTableInfo tableInfo = entry.getValue(); @@ -1347,31 +1447,34 @@ public synchronized Map> getDbToRecycleSize() { return dbToRecycleSize; } - // Need to add "synchronized", because when calling /dump api to dump image, - // this class is not protected by any lock, will throw ConcurrentModificationException. @Override - public synchronized void write(DataOutput out) throws IOException { - out.writeInt(idToDatabase.size()); - for (Map.Entry entry : idToDatabase.entrySet()) { - out.writeLong(entry.getKey()); - entry.getValue().write(out); - } - out.writeInt(idToTable.size()); - for (Map.Entry entry : idToTable.entrySet()) { - out.writeLong(entry.getKey()); - entry.getValue().write(out); - } - out.writeInt(idToPartition.size()); - for (Map.Entry entry : idToPartition.entrySet()) { - out.writeLong(entry.getKey()); - entry.getValue().write(out); - } - out.writeInt(idToRecycleTime.size()); - for (Map.Entry entry : idToRecycleTime.entrySet()) { - out.writeLong(entry.getKey()); - out.writeLong(entry.getValue()); + public void write(DataOutput out) throws IOException { + lock.writeLock().lock(); + try { + out.writeInt(idToDatabase.size()); + for (Map.Entry entry : idToDatabase.entrySet()) { + out.writeLong(entry.getKey()); + entry.getValue().write(out); + } + out.writeInt(idToTable.size()); + for (Map.Entry entry : idToTable.entrySet()) { + out.writeLong(entry.getKey()); + entry.getValue().write(out); + } + out.writeInt(idToPartition.size()); + for (Map.Entry entry : idToPartition.entrySet()) { + out.writeLong(entry.getKey()); + entry.getValue().write(out); + } + out.writeInt(idToRecycleTime.size()); + for (Map.Entry entry : idToRecycleTime.entrySet()) { + out.writeLong(entry.getKey()); + out.writeLong(entry.getValue()); + } + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } finally { + lock.writeLock().unlock(); } - Text.writeString(out, GsonUtils.GSON.toJson(this)); } public void readFieldsWithGson(DataInput in) throws IOException { @@ -1608,7 +1711,7 @@ public List getAllDbIds() { } // only for unit test - public synchronized void clearAll() { + public void clearAll() { idToDatabase.clear(); idToTable.clear(); idToPartition.clear();