Skip to content

Commit 36b91bb

Browse files
suxiaogang223suxiaogang223
committed
Internalize engine backing caches in UnifiedMetaCacheMgr
Also remove mistakenly committed EXTERNAL_META_CACHE_FRAMEWORK_API.md.
1 parent fd14a71 commit 36b91bb

File tree

3 files changed

+57
-212
lines changed

3 files changed

+57
-212
lines changed

‎EXTERNAL_META_CACHE_FRAMEWORK_API.md‎

Lines changed: 0 additions & 158 deletions
This file was deleted.

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java‎

Lines changed: 1 addition & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,8 @@
2121
import org.apache.doris.common.Pair;
2222
import org.apache.doris.common.ThreadPoolManager;
2323
import org.apache.doris.datasource.doris.DorisExternalMetaCacheMgr;
24-
import org.apache.doris.datasource.hive.HMSExternalCatalog;
25-
import org.apache.doris.datasource.hive.HiveEngineCache;
26-
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
27-
import org.apache.doris.datasource.hudi.source.HudiCachedPartitionProcessor;
28-
import org.apache.doris.datasource.hudi.source.HudiEngineCache;
29-
import org.apache.doris.datasource.hudi.source.HudiMetadataCacheMgr;
30-
import org.apache.doris.datasource.iceberg.IcebergEngineCache;
31-
import org.apache.doris.datasource.iceberg.IcebergMetadataCache;
32-
import org.apache.doris.datasource.maxcompute.MaxComputeEngineCache;
33-
import org.apache.doris.datasource.maxcompute.MaxComputeMetadataCacheMgr;
3424
import org.apache.doris.datasource.metacache.MetaCache;
3525
import org.apache.doris.datasource.metacache.UnifiedMetaCacheMgr;
36-
import org.apache.doris.datasource.paimon.PaimonEngineCache;
37-
import org.apache.doris.datasource.paimon.PaimonMetadataCache;
3826
import org.apache.doris.fs.FileSystemCache;
3927
import org.apache.doris.system.Backend;
4028

@@ -87,16 +75,10 @@ public class ExternalMetaCacheMgr {
8775
// This executor is used to schedule the getting split tasks
8876
private ExecutorService scheduleExecutor;
8977

90-
private final CatalogScopedCacheMgr<HiveMetaStoreCache> hiveMetaStoreCacheMgr;
91-
private final CatalogScopedCacheMgr<IcebergMetadataCache> icebergMetadataCacheMgr;
92-
private final CatalogScopedCacheMgr<PaimonMetadataCache> paimonMetadataCacheMgr;
93-
// hudi partition manager
94-
private final HudiMetadataCacheMgr hudiMetadataCacheMgr;
9578
// all catalogs could share the same fsCache.
9679
private FileSystemCache fsCache;
9780
// all external table row count cache.
9881
private ExternalRowCountCache rowCountCache;
99-
private final MaxComputeMetadataCacheMgr maxComputeMetadataCacheMgr;
10082
private final DorisExternalMetaCacheMgr dorisExternalMetaCacheMgr;
10183
private final UnifiedMetaCacheMgr unifiedMetaCacheMgr;
10284

@@ -126,30 +108,7 @@ public ExternalMetaCacheMgr(boolean isCheckpointCatalog) {
126108
fsCache = new FileSystemCache();
127109
rowCountCache = new ExternalRowCountCache(rowCountRefreshExecutor);
128110

129-
hudiMetadataCacheMgr = new HudiMetadataCacheMgr(commonRefreshExecutor);
130-
maxComputeMetadataCacheMgr = new MaxComputeMetadataCacheMgr();
131-
unifiedMetaCacheMgr = new UnifiedMetaCacheMgr();
132-
hiveMetaStoreCacheMgr = new CatalogScopedCacheMgr<>(
133-
catalog -> new HiveMetaStoreCache((HMSExternalCatalog) catalog,
134-
commonRefreshExecutor, fileListingExecutor));
135-
icebergMetadataCacheMgr = new CatalogScopedCacheMgr<>(
136-
catalog -> new IcebergMetadataCache(catalog, commonRefreshExecutor));
137-
paimonMetadataCacheMgr = new CatalogScopedCacheMgr<>(
138-
catalog -> new PaimonMetadataCache(catalog, commonRefreshExecutor));
139-
unifiedMetaCacheMgr.registerEngineFactory(PaimonEngineCache.ENGINE_TYPE,
140-
catalog -> new PaimonEngineCache(paimonMetadataCacheMgr.getCache(catalog)));
141-
unifiedMetaCacheMgr.registerEngineFactory(IcebergEngineCache.ENGINE_TYPE,
142-
catalog -> new IcebergEngineCache(icebergMetadataCacheMgr.getCache(catalog)));
143-
unifiedMetaCacheMgr.registerEngineFactory(MaxComputeEngineCache.ENGINE_TYPE,
144-
catalog -> new MaxComputeEngineCache(
145-
maxComputeMetadataCacheMgr.getMaxComputeMetadataCache(catalog)));
146-
unifiedMetaCacheMgr.registerEngineFactory(HiveEngineCache.ENGINE_TYPE,
147-
catalog -> new HiveEngineCache(hiveMetaStoreCacheMgr.getCache(catalog)));
148-
unifiedMetaCacheMgr.registerEngineFactory(HudiEngineCache.ENGINE_TYPE,
149-
catalog -> new HudiEngineCache(
150-
(HudiCachedPartitionProcessor) hudiMetadataCacheMgr.getPartitionProcessor(catalog),
151-
hudiMetadataCacheMgr.getFsViewProcessor(catalog),
152-
hudiMetadataCacheMgr.getHudiMetaClientProcessor(catalog)));
111+
unifiedMetaCacheMgr = new UnifiedMetaCacheMgr(commonRefreshExecutor, fileListingExecutor);
153112
dorisExternalMetaCacheMgr = new DorisExternalMetaCacheMgr(commonRefreshExecutor);
154113
}
155114

@@ -204,18 +163,6 @@ public void removeCache(ExternalCatalog catalog) {
204163
Objects.requireNonNull(catalog, "catalog cannot be null");
205164
long catalogId = catalog.getId();
206165
unifiedMetaCacheMgr.removeCatalogMetaCache(catalogId);
207-
if (hiveMetaStoreCacheMgr.removeCache(catalogId) != null) {
208-
LOG.info("remove hive metastore cache for catalog {}", catalogId);
209-
}
210-
if (icebergMetadataCacheMgr.removeCache(catalogId) != null) {
211-
LOG.info("remove iceberg meta cache for catalog {}", catalogId);
212-
}
213-
hudiMetadataCacheMgr.removeCache(catalogId);
214-
maxComputeMetadataCacheMgr.removeCache(catalogId);
215-
PaimonMetadataCache paimonMetadataCache = paimonMetadataCacheMgr.removeCache(catalogId);
216-
if (paimonMetadataCache != null) {
217-
paimonMetadataCache.invalidateCatalogCache(catalogId);
218-
}
219166
dorisExternalMetaCacheMgr.removeCache(catalogId);
220167
}
221168

‎fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/UnifiedMetaCacheMgr.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,26 @@
1717

1818
package org.apache.doris.datasource.metacache;
1919

20+
import org.apache.doris.datasource.CatalogScopedCacheMgr;
2021
import org.apache.doris.datasource.ExternalCatalog;
2122
import org.apache.doris.datasource.ExternalTable;
2223
import org.apache.doris.datasource.SchemaCacheValue;
24+
import org.apache.doris.datasource.hive.HMSExternalCatalog;
2325
import org.apache.doris.datasource.hive.HMSExternalTable;
2426
import org.apache.doris.datasource.hive.HiveEngineCache;
27+
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
28+
import org.apache.doris.datasource.hudi.source.HudiCachedPartitionProcessor;
2529
import org.apache.doris.datasource.hudi.source.HudiEngineCache;
30+
import org.apache.doris.datasource.hudi.source.HudiMetadataCacheMgr;
2631
import org.apache.doris.datasource.iceberg.IcebergEngineCache;
2732
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
33+
import org.apache.doris.datasource.iceberg.IcebergMetadataCache;
2834
import org.apache.doris.datasource.maxcompute.MaxComputeEngineCache;
2935
import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable;
36+
import org.apache.doris.datasource.maxcompute.MaxComputeMetadataCacheMgr;
3037
import org.apache.doris.datasource.paimon.PaimonEngineCache;
3138
import org.apache.doris.datasource.paimon.PaimonExternalTable;
39+
import org.apache.doris.datasource.paimon.PaimonMetadataCache;
3240

3341
import com.google.common.collect.ImmutableSet;
3442

@@ -38,6 +46,7 @@
3846
import java.util.Optional;
3947
import java.util.Set;
4048
import java.util.concurrent.ConcurrentHashMap;
49+
import java.util.concurrent.ExecutorService;
4150
import java.util.function.Consumer;
4251
import java.util.function.Function;
4352

@@ -48,6 +57,41 @@ public class UnifiedMetaCacheMgr {
4857
private final Map<Long, CatalogMetaCache> catalogMetaCaches = new ConcurrentHashMap<>();
4958
private final Map<String, Function<ExternalCatalog, ? extends EngineMetaCache>> engineFactories =
5059
new ConcurrentHashMap<>();
60+
private final CatalogScopedCacheMgr<HiveMetaStoreCache> hiveMetaStoreCacheMgr;
61+
private final CatalogScopedCacheMgr<IcebergMetadataCache> icebergMetadataCacheMgr;
62+
private final CatalogScopedCacheMgr<PaimonMetadataCache> paimonMetadataCacheMgr;
63+
private final HudiMetadataCacheMgr hudiMetadataCacheMgr;
64+
private final MaxComputeMetadataCacheMgr maxComputeMetadataCacheMgr;
65+
66+
public UnifiedMetaCacheMgr(ExecutorService commonRefreshExecutor, ExecutorService fileListingExecutor) {
67+
Objects.requireNonNull(commonRefreshExecutor, "commonRefreshExecutor cannot be null");
68+
Objects.requireNonNull(fileListingExecutor, "fileListingExecutor cannot be null");
69+
70+
hudiMetadataCacheMgr = new HudiMetadataCacheMgr(commonRefreshExecutor);
71+
maxComputeMetadataCacheMgr = new MaxComputeMetadataCacheMgr();
72+
hiveMetaStoreCacheMgr = new CatalogScopedCacheMgr<>(
73+
catalog -> new HiveMetaStoreCache((HMSExternalCatalog) catalog,
74+
commonRefreshExecutor, fileListingExecutor));
75+
icebergMetadataCacheMgr = new CatalogScopedCacheMgr<>(
76+
catalog -> new IcebergMetadataCache(catalog, commonRefreshExecutor));
77+
paimonMetadataCacheMgr = new CatalogScopedCacheMgr<>(
78+
catalog -> new PaimonMetadataCache(catalog, commonRefreshExecutor));
79+
80+
registerEngineFactory(PaimonEngineCache.ENGINE_TYPE,
81+
catalog -> new PaimonEngineCache(paimonMetadataCacheMgr.getCache(catalog)));
82+
registerEngineFactory(IcebergEngineCache.ENGINE_TYPE,
83+
catalog -> new IcebergEngineCache(icebergMetadataCacheMgr.getCache(catalog)));
84+
registerEngineFactory(MaxComputeEngineCache.ENGINE_TYPE,
85+
catalog -> new MaxComputeEngineCache(
86+
maxComputeMetadataCacheMgr.getMaxComputeMetadataCache(catalog)));
87+
registerEngineFactory(HiveEngineCache.ENGINE_TYPE,
88+
catalog -> new HiveEngineCache(hiveMetaStoreCacheMgr.getCache(catalog)));
89+
registerEngineFactory(HudiEngineCache.ENGINE_TYPE,
90+
catalog -> new HudiEngineCache(
91+
(HudiCachedPartitionProcessor) hudiMetadataCacheMgr.getPartitionProcessor(catalog),
92+
hudiMetadataCacheMgr.getFsViewProcessor(catalog),
93+
hudiMetadataCacheMgr.getHudiMetaClientProcessor(catalog)));
94+
}
5195

5296
public void registerEngineFactory(String engineType,
5397
Function<ExternalCatalog, ? extends EngineMetaCache> engineFactory) {
@@ -123,6 +167,7 @@ public CatalogMetaCache removeCatalogMetaCache(long catalogId) {
123167
if (catalogMetaCache != null) {
124168
catalogMetaCache.invalidateCatalog();
125169
}
170+
removeCatalogBackingCaches(catalogId);
126171
return catalogMetaCache;
127172
}
128173

@@ -215,6 +260,17 @@ private ExternalCatalog requireCatalog(ExternalTable table) {
215260
return Objects.requireNonNull(table.getCatalog(), "table catalog cannot be null");
216261
}
217262

263+
private void removeCatalogBackingCaches(long catalogId) {
264+
hiveMetaStoreCacheMgr.removeCache(catalogId);
265+
icebergMetadataCacheMgr.removeCache(catalogId);
266+
hudiMetadataCacheMgr.removeCache(catalogId);
267+
maxComputeMetadataCacheMgr.removeCache(catalogId);
268+
PaimonMetadataCache paimonMetadataCache = paimonMetadataCacheMgr.removeCache(catalogId);
269+
if (paimonMetadataCache != null) {
270+
paimonMetadataCache.invalidateCatalogCache(catalogId);
271+
}
272+
}
273+
218274
private void withCatalogMetaCache(long catalogId, Consumer<CatalogMetaCache> consumer) {
219275
CatalogMetaCache catalogMetaCache = catalogMetaCaches.get(catalogId);
220276
if (catalogMetaCache != null) {

0 commit comments

Comments
 (0)