|
28 | 28 | import org.apache.doris.datasource.ExternalTable; |
29 | 29 | import org.apache.doris.datasource.InitCatalogLog; |
30 | 30 | import org.apache.doris.datasource.SessionContext; |
| 31 | +import org.apache.doris.datasource.hudi.source.HudiEngineCache; |
31 | 32 | import org.apache.doris.datasource.iceberg.IcebergMetadataOps; |
32 | 33 | import org.apache.doris.datasource.iceberg.IcebergUtils; |
33 | | -import org.apache.doris.datasource.metacache.CacheSpec; |
| 34 | +import org.apache.doris.datasource.metacache.UnifiedCacheModuleKey; |
34 | 35 | import org.apache.doris.datasource.operations.ExternalMetadataOperations; |
35 | 36 | import org.apache.doris.datasource.property.metastore.AbstractHiveProperties; |
36 | 37 | import org.apache.doris.fs.FileSystemProvider; |
|
39 | 40 | import org.apache.doris.transaction.TransactionManagerFactory; |
40 | 41 |
|
41 | 42 | import com.google.common.annotations.VisibleForTesting; |
| 43 | +import com.google.common.collect.ImmutableList; |
42 | 44 | import com.google.common.collect.Sets; |
43 | 45 | import org.apache.iceberg.hive.HiveCatalog; |
44 | 46 | import org.apache.logging.log4j.LogManager; |
45 | 47 | import org.apache.logging.log4j.Logger; |
46 | 48 |
|
47 | 49 | import java.util.List; |
48 | 50 | import java.util.Map; |
49 | | -import java.util.Objects; |
50 | 51 | import java.util.Set; |
51 | 52 | import java.util.concurrent.ThreadPoolExecutor; |
52 | 53 |
|
@@ -78,52 +79,19 @@ public class HMSExternalCatalog extends ExternalCatalog { |
78 | 79 | public static final String HUDI_META_CLIENT_CACHE_ENABLE = "meta.cache.hudi.meta-client.enable"; |
79 | 80 | public static final String HUDI_META_CLIENT_CACHE_TTL_SECOND = "meta.cache.hudi.meta-client.ttl-second"; |
80 | 81 | public static final String HUDI_META_CLIENT_CACHE_CAPACITY = "meta.cache.hudi.meta-client.capacity"; |
81 | | - private static final String[] UNIFIED_CACHE_BOOLEAN_KEYS = { |
82 | | - HIVE_PARTITION_VALUES_CACHE_ENABLE, |
83 | | - HIVE_PARTITION_CACHE_ENABLE, |
84 | | - HIVE_FILE_CACHE_ENABLE, |
85 | | - HUDI_PARTITION_CACHE_ENABLE, |
86 | | - HUDI_FS_VIEW_CACHE_ENABLE, |
87 | | - HUDI_META_CLIENT_CACHE_ENABLE |
88 | | - }; |
89 | | - private static final String[] UNIFIED_CACHE_TTL_KEYS = { |
90 | | - HIVE_PARTITION_VALUES_CACHE_TTL_SECOND, |
91 | | - HIVE_PARTITION_CACHE_TTL_SECOND, |
92 | | - HIVE_FILE_CACHE_TTL_SECOND, |
93 | | - HUDI_PARTITION_CACHE_TTL_SECOND, |
94 | | - HUDI_FS_VIEW_CACHE_TTL_SECOND, |
95 | | - HUDI_META_CLIENT_CACHE_TTL_SECOND |
96 | | - }; |
97 | | - private static final String[] UNIFIED_CACHE_CAPACITY_KEYS = { |
98 | | - HIVE_PARTITION_VALUES_CACHE_CAPACITY, |
99 | | - HIVE_PARTITION_CACHE_CAPACITY, |
100 | | - HIVE_FILE_CACHE_CAPACITY, |
101 | | - HUDI_PARTITION_CACHE_CAPACITY, |
102 | | - HUDI_FS_VIEW_CACHE_CAPACITY, |
103 | | - HUDI_META_CLIENT_CACHE_CAPACITY |
104 | | - }; |
105 | | - private static final String[] HIVE_UNIFIED_CACHE_KEYS = { |
106 | | - HIVE_PARTITION_VALUES_CACHE_ENABLE, |
107 | | - HIVE_PARTITION_VALUES_CACHE_TTL_SECOND, |
108 | | - HIVE_PARTITION_VALUES_CACHE_CAPACITY, |
109 | | - HIVE_PARTITION_CACHE_ENABLE, |
110 | | - HIVE_PARTITION_CACHE_TTL_SECOND, |
111 | | - HIVE_PARTITION_CACHE_CAPACITY, |
112 | | - HIVE_FILE_CACHE_ENABLE, |
113 | | - HIVE_FILE_CACHE_TTL_SECOND, |
114 | | - HIVE_FILE_CACHE_CAPACITY |
115 | | - }; |
116 | | - private static final String[] HUDI_UNIFIED_CACHE_KEYS = { |
117 | | - HUDI_PARTITION_CACHE_ENABLE, |
118 | | - HUDI_PARTITION_CACHE_TTL_SECOND, |
119 | | - HUDI_PARTITION_CACHE_CAPACITY, |
120 | | - HUDI_FS_VIEW_CACHE_ENABLE, |
121 | | - HUDI_FS_VIEW_CACHE_TTL_SECOND, |
122 | | - HUDI_FS_VIEW_CACHE_CAPACITY, |
123 | | - HUDI_META_CLIENT_CACHE_ENABLE, |
124 | | - HUDI_META_CLIENT_CACHE_TTL_SECOND, |
125 | | - HUDI_META_CLIENT_CACHE_CAPACITY |
126 | | - }; |
| 82 | + private static final List<UnifiedCacheModuleKey> HIVE_CACHE_MODULE_KEYS = ImmutableList.of( |
| 83 | + UnifiedCacheModuleKey.of(HiveEngineCache.ENGINE_TYPE, "partition-values"), |
| 84 | + UnifiedCacheModuleKey.of(HiveEngineCache.ENGINE_TYPE, "partition"), |
| 85 | + UnifiedCacheModuleKey.of(HiveEngineCache.ENGINE_TYPE, "file")); |
| 86 | + private static final List<UnifiedCacheModuleKey> HUDI_CACHE_MODULE_KEYS = ImmutableList.of( |
| 87 | + UnifiedCacheModuleKey.of(HudiEngineCache.ENGINE_TYPE, "partition"), |
| 88 | + UnifiedCacheModuleKey.of(HudiEngineCache.ENGINE_TYPE, "fs-view"), |
| 89 | + UnifiedCacheModuleKey.of(HudiEngineCache.ENGINE_TYPE, "meta-client")); |
| 90 | + private static final List<UnifiedCacheModuleKey> ALL_CACHE_MODULE_KEYS = |
| 91 | + ImmutableList.<UnifiedCacheModuleKey>builder() |
| 92 | + .addAll(HIVE_CACHE_MODULE_KEYS) |
| 93 | + .addAll(HUDI_CACHE_MODULE_KEYS) |
| 94 | + .build(); |
127 | 95 | public static final String HIVE_STAGING_DIR = "hive.staging_dir"; |
128 | 96 | public static final String DEFAULT_STAGING_BASE_DIR = "/tmp/.doris_staging"; |
129 | 97 | // broker name for file split and query scan. |
@@ -177,7 +145,7 @@ public HMSExternalCatalog(long catalogId, String name, String resource, Map<Stri |
177 | 145 | @Override |
178 | 146 | public void checkProperties() throws DdlException { |
179 | 147 | super.checkProperties(); |
180 | | - checkUnifiedCacheProperties(); |
| 148 | + UnifiedCacheModuleKey.checkProperties(catalogProperty.getProperties(), ALL_CACHE_MODULE_KEYS); |
181 | 149 | catalogProperty.checkMetaStoreAndStorageProperties(AbstractHiveProperties.class); |
182 | 150 | } |
183 | 151 |
|
@@ -279,38 +247,17 @@ public void registerDatabase(long dbId, String dbName) { |
279 | 247 | @Override |
280 | 248 | public void notifyPropertiesUpdated(Map<String, String> updatedProps) { |
281 | 249 | super.notifyPropertiesUpdated(updatedProps); |
282 | | - if (hasAnyUpdatedProperty(updatedProps, HIVE_UNIFIED_CACHE_KEYS)) { |
| 250 | + if (UnifiedCacheModuleKey.hasAnyUpdatedProperty(updatedProps, HIVE_CACHE_MODULE_KEYS)) { |
283 | 251 | Env.getCurrentEnv().getExtMetaCacheMgr() |
284 | 252 | .getUnifiedMetaCacheMgr() |
285 | 253 | .getOrCreateEngineMetaCache(this, HiveEngineCache.ENGINE_TYPE, HiveEngineCache.class) |
286 | 254 | .getMetaStoreCache().init(); |
287 | 255 | } |
288 | | - if (hasAnyUpdatedProperty(updatedProps, HUDI_UNIFIED_CACHE_KEYS)) { |
| 256 | + if (UnifiedCacheModuleKey.hasAnyUpdatedProperty(updatedProps, HUDI_CACHE_MODULE_KEYS)) { |
289 | 257 | Env.getCurrentEnv().getExtMetaCacheMgr().removeCache(this); |
290 | 258 | } |
291 | 259 | } |
292 | 260 |
|
293 | | - private void checkUnifiedCacheProperties() throws DdlException { |
294 | | - for (String key : UNIFIED_CACHE_BOOLEAN_KEYS) { |
295 | | - CacheSpec.checkBooleanProperty(catalogProperty.getOrDefault(key, null), key); |
296 | | - } |
297 | | - for (String key : UNIFIED_CACHE_TTL_KEYS) { |
298 | | - CacheSpec.checkLongProperty(catalogProperty.getOrDefault(key, null), -1L, key); |
299 | | - } |
300 | | - for (String key : UNIFIED_CACHE_CAPACITY_KEYS) { |
301 | | - CacheSpec.checkLongProperty(catalogProperty.getOrDefault(key, null), 0L, key); |
302 | | - } |
303 | | - } |
304 | | - |
305 | | - private static boolean hasAnyUpdatedProperty(Map<String, String> updatedProps, String[] propertyKeys) { |
306 | | - for (String propertyKey : propertyKeys) { |
307 | | - if (Objects.nonNull(updatedProps.get(propertyKey))) { |
308 | | - return true; |
309 | | - } |
310 | | - } |
311 | | - return false; |
312 | | - } |
313 | | - |
314 | 261 | @Override |
315 | 262 | public void setDefaultPropsIfMissing(boolean isReplay) { |
316 | 263 | super.setDefaultPropsIfMissing(isReplay); |
|
0 commit comments