Skip to content

Commit 8738223

Browse files
suxiaogang223suxiaogang223
committed
Refactor and unify external meta cache framework
1 parent 0fded79 commit 8738223

File tree

77 files changed

+3265
-1460
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+3265
-1460
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.doris.datasource.ExternalTable;
2929
import org.apache.doris.datasource.hive.HMSExternalCatalog;
3030
import org.apache.doris.datasource.hive.HMSExternalTable;
31+
import org.apache.doris.datasource.hive.HiveEngineCache;
3132
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
3233
import org.apache.doris.persist.OperationType;
3334

@@ -192,7 +193,10 @@ public void replayRefreshTable(ExternalObjectLog log) {
192193
|| (newPartNames != null && !newPartNames.isEmpty()))) {
193194
// Partition-level cache invalidation, only for hive catalog
194195
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
195-
.getMetaStoreCache((HMSExternalCatalog) catalog);
196+
.getUnifiedMetaCacheMgr()
197+
.getOrCreateEngineMetaCache((HMSExternalCatalog) catalog,
198+
HiveEngineCache.ENGINE_TYPE, HiveEngineCache.class)
199+
.getMetaStoreCache();
196200
cache.refreshAffectedPartitionsCache((HMSExternalTable) table.get(), modifiedPartNames, newPartNames);
197201
if (table.get() instanceof HMSExternalTable && log.getLastUpdateTime() > 0) {
198202
((HMSExternalTable) table.get()).setUpdateTime(log.getLastUpdateTime());
@@ -235,7 +239,7 @@ public void refreshTableInternal(ExternalDatabase db, ExternalTable table, long
235239
if (updateTime > 0) {
236240
table.setUpdateTime(updateTime);
237241
}
238-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table);
242+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidate(table);
239243
LOG.info("refresh table {}, id {} from db {} in catalog {}, update time: {}",
240244
table.getName(), table.getId(), db.getFullName(), db.getCatalog().getName(), updateTime);
241245
}
@@ -270,7 +274,15 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam
270274
return;
271275
}
272276

273-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidatePartitionsCache((ExternalTable) table, partitionNames);
277+
HiveEngineCache hiveEngineCache = Env.getCurrentEnv().getExtMetaCacheMgr()
278+
.getUnifiedMetaCacheMgr()
279+
.getEngineMetaCache(catalog.getId(), HiveEngineCache.ENGINE_TYPE, HiveEngineCache.class);
280+
if (hiveEngineCache != null) {
281+
HiveMetaStoreCache metaStoreCache = hiveEngineCache.getMetaStoreCache();
282+
for (String partitionName : partitionNames) {
283+
metaStoreCache.invalidatePartitionCache((HMSExternalTable) table, partitionName);
284+
}
285+
}
274286
((HMSExternalTable) table).setUpdateTime(updateTime);
275287
}
276288

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.doris.catalog.Env;
2323
import org.apache.doris.catalog.EnvFactory;
2424
import org.apache.doris.catalog.TableIf;
25+
import org.apache.doris.catalog.Type;
2526
import org.apache.doris.cluster.ClusterNamespace;
2627
import org.apache.doris.common.AnalysisException;
2728
import org.apache.doris.common.CaseSensibility;
@@ -41,7 +42,10 @@
4142
import org.apache.doris.datasource.hive.HMSExternalCatalog;
4243
import org.apache.doris.datasource.hive.HMSExternalDatabase;
4344
import org.apache.doris.datasource.hive.HMSExternalTable;
45+
import org.apache.doris.datasource.hive.HiveEngineCache;
46+
import org.apache.doris.datasource.mvcc.MvccUtil;
4447
import org.apache.doris.mysql.privilege.PrivPredicate;
48+
import org.apache.doris.nereids.exceptions.NotSupportedException;
4549
import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
4650
import org.apache.doris.persist.OperationType;
4751
import org.apache.doris.persist.gson.GsonPostProcessable;
@@ -131,7 +135,9 @@ private CatalogIf removeCatalog(long catalogId) {
131135
if (ConnectContext.get() != null) {
132136
ConnectContext.get().removeLastDBOfCatalog(catalog.getName());
133137
}
134-
Env.getCurrentEnv().getExtMetaCacheMgr().removeCache(catalog.getId());
138+
if (catalog instanceof ExternalCatalog) {
139+
Env.getCurrentEnv().getExtMetaCacheMgr().removeCache((ExternalCatalog) catalog);
140+
}
135141
Env.getCurrentEnv().getQueryStats().clear(catalog.getId());
136142
}
137143
return catalog;
@@ -570,10 +576,13 @@ public void replayAlterCatalogProps(CatalogLog log, Map<String, String> oldPrope
570576
}
571577
if (newProps.containsKey(METADATA_REFRESH_INTERVAL_SEC)) {
572578
long catalogId = catalog.getId();
573-
Integer metadataRefreshIntervalSec = Integer.valueOf(newProps.get(METADATA_REFRESH_INTERVAL_SEC));
579+
Integer metadataRefreshIntervalSec =
580+
Integer.valueOf(newProps.get(METADATA_REFRESH_INTERVAL_SEC));
574581
Integer[] sec = {metadataRefreshIntervalSec, metadataRefreshIntervalSec};
575582
Env.getCurrentEnv().getRefreshManager().addToRefreshMap(catalogId, sec);
576583
}
584+
catalog.modifyCatalogProps(newProps);
585+
return;
577586
}
578587
catalog.modifyCatalogProps(log.getNewProps());
579588
} finally {
@@ -725,7 +734,22 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl
725734
}
726735

727736
HMSExternalTable hmsTable = (HMSExternalTable) table;
728-
Env.getCurrentEnv().getExtMetaCacheMgr().addPartitionsCache(catalog.getId(), hmsTable, partitionNames);
737+
HiveEngineCache hiveEngineCache = Env.getCurrentEnv().getExtMetaCacheMgr()
738+
.getUnifiedMetaCacheMgr()
739+
.getEngineMetaCache(catalog.getId(), HiveEngineCache.ENGINE_TYPE, HiveEngineCache.class);
740+
if (hiveEngineCache != null) {
741+
List<Type> partitionColumnTypes;
742+
try {
743+
partitionColumnTypes = hmsTable.getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(hmsTable));
744+
} catch (NotSupportedException e) {
745+
LOG.warn("Ignore not supported hms table, message: {}", e.getMessage());
746+
partitionColumnTypes = null;
747+
}
748+
if (partitionColumnTypes != null) {
749+
hiveEngineCache.getMetaStoreCache().addPartitionsCache(
750+
hmsTable.getOrBuildNameMapping(), partitionNames, partitionColumnTypes);
751+
}
752+
}
729753
hmsTable.setUpdateTime(updateTime);
730754
}
731755

@@ -756,7 +780,12 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab
756780
}
757781

758782
HMSExternalTable hmsTable = (HMSExternalTable) table;
759-
Env.getCurrentEnv().getExtMetaCacheMgr().dropPartitionsCache(catalog.getId(), hmsTable, partitionNames);
783+
HiveEngineCache hiveEngineCache = Env.getCurrentEnv().getExtMetaCacheMgr()
784+
.getUnifiedMetaCacheMgr()
785+
.getEngineMetaCache(catalog.getId(), HiveEngineCache.ENGINE_TYPE, HiveEngineCache.class);
786+
if (hiveEngineCache != null) {
787+
hiveEngineCache.getMetaStoreCache().dropPartitionsCache(hmsTable, partitionNames, true);
788+
}
760789
hmsTable.setUpdateTime(updateTime);
761790
}
762791

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 19 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import org.apache.doris.common.Version;
3636
import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
3737
import org.apache.doris.common.util.Util;
38-
import org.apache.doris.datasource.ExternalSchemaCache.SchemaCacheKey;
3938
import org.apache.doris.datasource.connectivity.CatalogConnectivityTestCoordinator;
4039
import org.apache.doris.datasource.doris.RemoteDorisExternalDatabase;
4140
import org.apache.doris.datasource.es.EsExternalDatabase;
@@ -47,6 +46,7 @@
4746
import org.apache.doris.datasource.jdbc.JdbcExternalDatabase;
4847
import org.apache.doris.datasource.lakesoul.LakeSoulExternalDatabase;
4948
import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase;
49+
import org.apache.doris.datasource.metacache.CacheSpec;
5050
import org.apache.doris.datasource.metacache.MetaCache;
5151
import org.apache.doris.datasource.operations.ExternalMetadataOps;
5252
import org.apache.doris.datasource.paimon.PaimonExternalDatabase;
@@ -79,7 +79,6 @@
7979
import org.apache.commons.lang3.NotImplementedException;
8080
import org.apache.commons.lang3.StringUtils;
8181
import org.apache.commons.lang3.exception.ExceptionUtils;
82-
import org.apache.commons.lang3.math.NumberUtils;
8382
import org.apache.hadoop.conf.Configuration;
8483
import org.apache.logging.log4j.LogManager;
8584
import org.apache.logging.log4j.Logger;
@@ -117,6 +116,7 @@ public abstract class ExternalCatalog
117116

118117
// https://help.aliyun.com/zh/emr/emr-on-ecs/user-guide/use-rootpolicy-to-access-oss-hdfs?spm=a2c4g.11186623.help-menu-search-28066.d_0
119118
public static final String OOS_ROOT_POLICY = "oss.root_policy";
119+
// Legacy schema cache ttl property, kept for compatibility.
120120
public static final String SCHEMA_CACHE_TTL_SECOND = "schema.cache.ttl-second";
121121
// -1 means cache with no ttl
122122
public static final int CACHE_NO_TTL = -1;
@@ -392,14 +392,7 @@ public void checkProperties() throws DdlException {
392392
throw new DdlException("Invalid properties: " + CatalogMgr.METADATA_REFRESH_INTERVAL_SEC);
393393
}
394394
}
395-
396-
// check schema.cache.ttl-second parameter
397-
String schemaCacheTtlSecond = catalogProperty.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null);
398-
if (java.util.Objects.nonNull(schemaCacheTtlSecond) && NumberUtils.toInt(schemaCacheTtlSecond, CACHE_NO_TTL)
399-
< CACHE_TTL_DISABLE_CACHE) {
400-
throw new DdlException(
401-
"The parameter " + SCHEMA_CACHE_TTL_SECOND + " is wrong, value is " + schemaCacheTtlSecond);
402-
}
395+
CacheSpec.checkLongProperty(properties.get(SCHEMA_CACHE_TTL_SECOND), -1L, SCHEMA_CACHE_TTL_SECOND);
403396
}
404397

405398
/**
@@ -553,7 +546,7 @@ public void onRefreshCache(boolean invalidCache) {
553546
setLastUpdateTime(System.currentTimeMillis());
554547
refreshMetaCacheOnly();
555548
if (invalidCache) {
556-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalogCache(id);
549+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidate(this);
557550
}
558551
}
559552

@@ -567,18 +560,6 @@ private void refreshMetaCacheOnly() {
567560
}
568561
}
569562

570-
public final Optional<SchemaCacheValue> getSchema(SchemaCacheKey key) {
571-
makeSureInitialized();
572-
Optional<ExternalDatabase<? extends ExternalTable>> db = getDb(key.getNameMapping().getLocalDbName());
573-
if (db.isPresent()) {
574-
Optional<? extends ExternalTable> table = db.get().getTable(key.getNameMapping().getLocalTblName());
575-
if (table.isPresent()) {
576-
return table.get().initSchemaAndUpdateTime(key);
577-
}
578-
}
579-
return Optional.empty();
580-
}
581-
582563
@Override
583564
public long getId() {
584565
return id;
@@ -1053,7 +1034,7 @@ public void unregisterDatabase(String dbName) {
10531034
if (isInitialized()) {
10541035
metaCache.invalidate(dbName, Util.genIdByName(name, dbName));
10551036
}
1056-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(getId(), dbName);
1037+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidate(this, dbName);
10571038
}
10581039

10591040
public void registerDatabase(long dbId, String dbName) {
@@ -1197,9 +1178,20 @@ public int hashCode() {
11971178
@Override
11981179
public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
11991180
CatalogIf.super.notifyPropertiesUpdated(updatedProps);
1200-
String schemaCacheTtl = updatedProps.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null);
1201-
if (java.util.Objects.nonNull(schemaCacheTtl)) {
1202-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidSchemaCache(id);
1181+
if (updatedProps.containsKey(SCHEMA_CACHE_TTL_SECOND)) {
1182+
Env.getCurrentEnv().getExtMetaCacheMgr().getUnifiedMetaCacheMgr().removeCatalogMetaCache(id);
1183+
}
1184+
}
1185+
1186+
public long getSchemaCacheTtlSecond() {
1187+
String ttlSecond = catalogProperty.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null);
1188+
if (ttlSecond == null) {
1189+
return Config.external_cache_expire_time_seconds_after_access;
1190+
}
1191+
try {
1192+
return Long.parseLong(ttlSecond);
1193+
} catch (NumberFormatException e) {
1194+
return Config.external_cache_expire_time_seconds_after_access;
12031195
}
12041196
}
12051197

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ public void resetMetaToUninitialized() {
127127
metaCache.invalidateAll();
128128
}
129129
}
130-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDbCache(getCatalog().getId(), getFullName());
130+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidate(this);
131131
}
132132

133133
public boolean isInitialized() {
@@ -568,7 +568,7 @@ public void unregisterTable(String tableName) {
568568
lowerCaseToTableName.remove(dorisTable.getName().toLowerCase());
569569
}
570570

571-
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(dorisTable);
571+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidate(dorisTable);
572572
}
573573

574574
@Override

0 commit comments

Comments
 (0)