Skip to content

Commit c9e5835

Browse files
suxiaogang223suxiaogang223
committed
Keep legacy schema and hive ttl properties compatibility
1 parent 8277a06 commit c9e5835

File tree

7 files changed

+82
-73
lines changed

7 files changed

+82
-73
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -563,10 +563,7 @@ public void replayAlterCatalogProps(CatalogLog log, Map<String, String> oldPrope
563563
CatalogIf catalog = idToCatalog.get(log.getCatalogId());
564564
if (catalog instanceof ExternalCatalog) {
565565
Map<String, String> newProps = log.getNewProps();
566-
Map<String, String> effectiveNewProps = isReplay
567-
? ((ExternalCatalog) catalog).filterDeprecatedPropertiesForReplay(newProps)
568-
: newProps;
569-
((ExternalCatalog) catalog).tryModifyCatalogProps(effectiveNewProps);
566+
((ExternalCatalog) catalog).tryModifyCatalogProps(newProps);
570567
if (!isReplay) {
571568
try {
572569
((ExternalCatalog) catalog).checkProperties();
@@ -577,14 +574,14 @@ public void replayAlterCatalogProps(CatalogLog log, Map<String, String> oldPrope
577574
throw ddlException;
578575
}
579576
}
580-
if (effectiveNewProps.containsKey(METADATA_REFRESH_INTERVAL_SEC)) {
577+
if (newProps.containsKey(METADATA_REFRESH_INTERVAL_SEC)) {
581578
long catalogId = catalog.getId();
582579
Integer metadataRefreshIntervalSec =
583-
Integer.valueOf(effectiveNewProps.get(METADATA_REFRESH_INTERVAL_SEC));
580+
Integer.valueOf(newProps.get(METADATA_REFRESH_INTERVAL_SEC));
584581
Integer[] sec = {metadataRefreshIntervalSec, metadataRefreshIntervalSec};
585582
Env.getCurrentEnv().getRefreshManager().addToRefreshMap(catalogId, sec);
586583
}
587-
catalog.modifyCatalogProps(effectiveNewProps);
584+
catalog.modifyCatalogProps(newProps);
588585
return;
589586
}
590587
catalog.modifyCatalogProps(log.getNewProps());

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 15 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.apache.doris.datasource.jdbc.JdbcExternalDatabase;
4747
import org.apache.doris.datasource.lakesoul.LakeSoulExternalDatabase;
4848
import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase;
49+
import org.apache.doris.datasource.metacache.CacheSpec;
4950
import org.apache.doris.datasource.metacache.MetaCache;
5051
import org.apache.doris.datasource.operations.ExternalMetadataOps;
5152
import org.apache.doris.datasource.paimon.PaimonExternalDatabase;
@@ -115,7 +116,8 @@ public abstract class ExternalCatalog
115116

116117
// https://help.aliyun.com/zh/emr/emr-on-ecs/user-guide/use-rootpolicy-to-access-oss-hdfs?spm=a2c4g.11186623.help-menu-search-28066.d_0
117118
public static final String OOS_ROOT_POLICY = "oss.root_policy";
118-
public static final String DEPRECATED_SCHEMA_CACHE_TTL_SECOND = "schema.cache.ttl-second";
119+
// Legacy schema cache ttl property, kept for compatibility.
120+
public static final String SCHEMA_CACHE_TTL_SECOND = "schema.cache.ttl-second";
119121
// -1 means cache with no ttl
120122
public static final int CACHE_NO_TTL = -1;
121123
// 0 means cache is disabled; >0 means cache with ttl;
@@ -248,9 +250,6 @@ public void setDefaultPropsIfMissing(boolean isReplay) {
248250
if (catalogProperty.getOrDefault(CatalogProperty.ENABLE_MAPPING_TIMESTAMP_TZ, "").isEmpty()) {
249251
catalogProperty.setEnableMappingTimestampTz(false);
250252
}
251-
if (isReplay) {
252-
stripDeprecatedCatalogPropertiesForReplay(catalogProperty.getProperties());
253-
}
254253
}
255254

256255
public boolean getEnableMappingVarbinary() {
@@ -393,7 +392,7 @@ public void checkProperties() throws DdlException {
393392
throw new DdlException("Invalid properties: " + CatalogMgr.METADATA_REFRESH_INTERVAL_SEC);
394393
}
395394
}
396-
ensureNoDeprecatedCatalogProperties(properties);
395+
CacheSpec.checkLongProperty(properties.get(SCHEMA_CACHE_TTL_SECOND), -1L, SCHEMA_CACHE_TTL_SECOND);
397396
}
398397

399398
/**
@@ -1179,45 +1178,21 @@ public int hashCode() {
11791178
@Override
11801179
public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
11811180
CatalogIf.super.notifyPropertiesUpdated(updatedProps);
1182-
}
1183-
1184-
/**
1185-
* Deprecated catalog-level properties that are rejected in DDL and ignored during replay.
1186-
*/
1187-
protected Set<String> getDeprecatedCatalogPropertyKeys() {
1188-
return Sets.newHashSet(DEPRECATED_SCHEMA_CACHE_TTL_SECOND);
1189-
}
1190-
1191-
public Map<String, String> filterDeprecatedPropertiesForReplay(Map<String, String> props) {
1192-
Map<String, String> filteredProps = Maps.newHashMap(props);
1193-
stripDeprecatedCatalogPropertiesForReplay(filteredProps);
1194-
return filteredProps;
1195-
}
1196-
1197-
private void ensureNoDeprecatedCatalogProperties(Map<String, String> properties) throws DdlException {
1198-
List<String> foundDeprecatedProperties = listDeprecatedCatalogProperties(properties);
1199-
if (!foundDeprecatedProperties.isEmpty()) {
1200-
throw new DdlException("Deprecated catalog cache properties are no longer supported: "
1201-
+ String.join(", ", foundDeprecatedProperties)
1202-
+ ". Please use meta.cache.<engine>.<module>.{enable,ttl-second,capacity};"
1203-
+ " defaults are from FE conf.");
1181+
if (updatedProps.containsKey(SCHEMA_CACHE_TTL_SECOND)) {
1182+
Env.getCurrentEnv().getExtMetaCacheMgr().getUnifiedMetaCacheMgr().removeCatalogMetaCache(id);
12041183
}
12051184
}
12061185

1207-
private void stripDeprecatedCatalogPropertiesForReplay(Map<String, String> properties) {
1208-
List<String> removedDeprecatedProperties = listDeprecatedCatalogProperties(properties);
1209-
removedDeprecatedProperties.forEach(properties::remove);
1210-
if (!removedDeprecatedProperties.isEmpty()) {
1211-
LOG.warn("Ignore deprecated catalog cache properties during replay, catalog={}, id={}, keys={}",
1212-
name, id, removedDeprecatedProperties);
1186+
public long getSchemaCacheTtlSecond() {
1187+
String ttlSecond = catalogProperty.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null);
1188+
if (ttlSecond == null) {
1189+
return Config.external_cache_expire_time_seconds_after_access;
1190+
}
1191+
try {
1192+
return Long.parseLong(ttlSecond);
1193+
} catch (NumberFormatException e) {
1194+
return Config.external_cache_expire_time_seconds_after_access;
12131195
}
1214-
}
1215-
1216-
private List<String> listDeprecatedCatalogProperties(Map<String, String> properties) {
1217-
return getDeprecatedCatalogPropertyKeys().stream()
1218-
.filter(properties::containsKey)
1219-
.sorted()
1220-
.collect(Collectors.toList());
12211196
}
12221197

12231198
public CatalogProperty getCatalogProperty() {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.apache.doris.datasource.hudi.source.HudiEngineCache;
3232
import org.apache.doris.datasource.iceberg.IcebergMetadataOps;
3333
import org.apache.doris.datasource.iceberg.IcebergUtils;
34+
import org.apache.doris.datasource.metacache.CacheSpec;
3435
import org.apache.doris.datasource.metacache.UnifiedCacheModuleKey;
3536
import org.apache.doris.datasource.operations.ExternalMetadataOperations;
3637
import org.apache.doris.datasource.property.metastore.AbstractHiveProperties;
@@ -41,14 +42,12 @@
4142

4243
import com.google.common.annotations.VisibleForTesting;
4344
import com.google.common.collect.ImmutableList;
44-
import com.google.common.collect.Sets;
4545
import org.apache.iceberg.hive.HiveCatalog;
4646
import org.apache.logging.log4j.LogManager;
4747
import org.apache.logging.log4j.Logger;
4848

4949
import java.util.List;
5050
import java.util.Map;
51-
import java.util.Set;
5251
import java.util.concurrent.ThreadPoolExecutor;
5352

5453
/**
@@ -57,6 +56,7 @@
5756
public class HMSExternalCatalog extends ExternalCatalog {
5857
private static final Logger LOG = LogManager.getLogger(HMSExternalCatalog.class);
5958

59+
// Legacy cache properties (kept for compatibility).
6060
public static final String FILE_META_CACHE_TTL_SECOND = "file.meta.cache.ttl-second";
6161
public static final String PARTITION_CACHE_TTL_SECOND = "partition.cache.ttl-second";
6262

@@ -146,17 +146,13 @@ public HMSExternalCatalog(long catalogId, String name, String resource, Map<Stri
146146
public void checkProperties() throws DdlException {
147147
super.checkProperties();
148148
UnifiedCacheModuleKey.checkProperties(catalogProperty.getProperties(), ALL_CACHE_MODULE_KEYS);
149+
CacheSpec.checkLongProperty(catalogProperty.getProperties().get(FILE_META_CACHE_TTL_SECOND),
150+
-1L, FILE_META_CACHE_TTL_SECOND);
151+
CacheSpec.checkLongProperty(catalogProperty.getProperties().get(PARTITION_CACHE_TTL_SECOND),
152+
-1L, PARTITION_CACHE_TTL_SECOND);
149153
catalogProperty.checkMetaStoreAndStorageProperties(AbstractHiveProperties.class);
150154
}
151155

152-
@Override
153-
protected Set<String> getDeprecatedCatalogPropertyKeys() {
154-
Set<String> deprecatedKeys = Sets.newHashSet(super.getDeprecatedCatalogPropertyKeys());
155-
deprecatedKeys.add(FILE_META_CACHE_TTL_SECOND);
156-
deprecatedKeys.add(PARTITION_CACHE_TTL_SECOND);
157-
return deprecatedKeys;
158-
}
159-
160156
@Override
161157
protected synchronized void initPreExecutionAuthenticator() {
162158
if (executionAuthenticator == null) {
@@ -247,7 +243,9 @@ public void registerDatabase(long dbId, String dbName) {
247243
@Override
248244
public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
249245
super.notifyPropertiesUpdated(updatedProps);
250-
if (UnifiedCacheModuleKey.hasAnyUpdatedProperty(updatedProps, HIVE_CACHE_MODULE_KEYS)) {
246+
if (UnifiedCacheModuleKey.hasAnyUpdatedProperty(updatedProps, HIVE_CACHE_MODULE_KEYS)
247+
|| updatedProps.containsKey(FILE_META_CACHE_TTL_SECOND)
248+
|| updatedProps.containsKey(PARTITION_CACHE_TTL_SECOND)) {
251249
Env.getCurrentEnv().getExtMetaCacheMgr()
252250
.getUnifiedMetaCacheMgr()
253251
.getOrCreateEngineMetaCache(this, HiveEngineCache.ENGINE_TYPE, HiveEngineCache.class)

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java‎

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -197,15 +197,52 @@ public FileCacheValue load(FileCacheKey key) {
197197

198198
private void refreshCacheSpecs() {
199199
Map<String, String> properties = catalog.getProperties();
200-
partitionValuesCacheSpec = CacheSpec.fromUnifiedProperties(properties, HiveEngineCache.ENGINE_TYPE,
201-
"partition-values", true,
202-
Config.external_cache_expire_time_seconds_after_access, Config.max_hive_partition_table_cache_num);
203-
partitionCacheSpec = CacheSpec.fromUnifiedProperties(properties, HiveEngineCache.ENGINE_TYPE,
204-
"partition", true,
205-
Config.external_cache_expire_time_seconds_after_access, Config.max_hive_partition_cache_num);
206-
fileCacheSpec = CacheSpec.fromUnifiedProperties(properties, HiveEngineCache.ENGINE_TYPE,
207-
"file", true,
208-
Config.external_cache_expire_time_seconds_after_access, Config.max_external_file_cache_num);
200+
long partitionValuesTtlSecond = resolveTtlSecondWithLegacyFallback(properties,
201+
HMSExternalCatalog.HIVE_PARTITION_VALUES_CACHE_TTL_SECOND,
202+
HMSExternalCatalog.PARTITION_CACHE_TTL_SECOND,
203+
Config.external_cache_expire_time_seconds_after_access);
204+
long partitionTtlSecond = resolveTtlSecondWithLegacyFallback(properties,
205+
HMSExternalCatalog.HIVE_PARTITION_CACHE_TTL_SECOND,
206+
HMSExternalCatalog.PARTITION_CACHE_TTL_SECOND,
207+
Config.external_cache_expire_time_seconds_after_access);
208+
long fileTtlSecond = resolveTtlSecondWithLegacyFallback(properties,
209+
HMSExternalCatalog.HIVE_FILE_CACHE_TTL_SECOND,
210+
HMSExternalCatalog.FILE_META_CACHE_TTL_SECOND,
211+
Config.external_cache_expire_time_seconds_after_access);
212+
213+
partitionValuesCacheSpec = CacheSpec.fromProperties(properties,
214+
HMSExternalCatalog.HIVE_PARTITION_VALUES_CACHE_ENABLE, true,
215+
HMSExternalCatalog.HIVE_PARTITION_VALUES_CACHE_TTL_SECOND, partitionValuesTtlSecond,
216+
HMSExternalCatalog.HIVE_PARTITION_VALUES_CACHE_CAPACITY, Config.max_hive_partition_table_cache_num);
217+
partitionCacheSpec = CacheSpec.fromProperties(properties,
218+
HMSExternalCatalog.HIVE_PARTITION_CACHE_ENABLE, true,
219+
HMSExternalCatalog.HIVE_PARTITION_CACHE_TTL_SECOND, partitionTtlSecond,
220+
HMSExternalCatalog.HIVE_PARTITION_CACHE_CAPACITY, Config.max_hive_partition_cache_num);
221+
fileCacheSpec = CacheSpec.fromProperties(properties,
222+
HMSExternalCatalog.HIVE_FILE_CACHE_ENABLE, true,
223+
HMSExternalCatalog.HIVE_FILE_CACHE_TTL_SECOND, fileTtlSecond,
224+
HMSExternalCatalog.HIVE_FILE_CACHE_CAPACITY, Config.max_external_file_cache_num);
225+
}
226+
227+
private static long resolveTtlSecondWithLegacyFallback(Map<String, String> properties,
228+
String unifiedTtlKey, String legacyTtlKey, long defaultTtlSecond) {
229+
String unifiedTtl = properties.get(unifiedTtlKey);
230+
if (!Strings.isNullOrEmpty(unifiedTtl)) {
231+
return parseLongOrDefault(unifiedTtl, defaultTtlSecond);
232+
}
233+
String legacyTtl = properties.get(legacyTtlKey);
234+
if (!Strings.isNullOrEmpty(legacyTtl)) {
235+
return parseLongOrDefault(legacyTtl, defaultTtlSecond);
236+
}
237+
return defaultTtlSecond;
238+
}
239+
240+
private static long parseLongOrDefault(String value, long defaultValue) {
241+
try {
242+
return Long.parseLong(value);
243+
} catch (NumberFormatException e) {
244+
return defaultValue;
245+
}
209246
}
210247

211248
private void initMetrics() {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCache.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,15 @@
4141
*/
4242
public class CatalogMetaCache {
4343
private final long catalogId;
44+
private final long schemaCacheTtlSecond;
4445
private final Map<String, EngineMetaCache> engineCaches = new ConcurrentHashMap<>();
4546
private final Object schemaCacheLock = new Object();
4647
private volatile LoadingCache<SchemaMetaCacheKey, Optional<SchemaCacheValue>> schemaCache;
4748

48-
public CatalogMetaCache(long catalogId) {
49-
this.catalogId = catalogId;
49+
public CatalogMetaCache(ExternalCatalog catalog) {
50+
ExternalCatalog nonNullCatalog = Objects.requireNonNull(catalog, "catalog cannot be null");
51+
this.catalogId = nonNullCatalog.getId();
52+
this.schemaCacheTtlSecond = nonNullCatalog.getSchemaCacheTtlSecond();
5053
}
5154

5255
public long getCatalogId() {
@@ -144,7 +147,7 @@ private LoadingCache<SchemaMetaCacheKey, Optional<SchemaCacheValue>> getOrCreate
144147
return localSchemaCache;
145148
}
146149
CacheFactory cacheFactory = new CacheFactory(
147-
CacheSpec.toExpireAfterAccess(Config.external_cache_expire_time_seconds_after_access),
150+
CacheSpec.toExpireAfterAccess(schemaCacheTtlSecond),
148151
OptionalLong.of(Config.external_cache_refresh_time_minutes * 60),
149152
Config.max_external_schema_cache_num,
150153
false,

‎fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/UnifiedMetaCacheMgr.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ private static String resolveEngineType(ExternalTable table) {
251251
}
252252

253253
private CatalogMetaCache getOrCreateCatalogMetaCache(ExternalCatalog catalog) {
254-
return catalogMetaCaches.computeIfAbsent(catalog.getId(), CatalogMetaCache::new);
254+
return catalogMetaCaches.computeIfAbsent(catalog.getId(), ignored -> new CatalogMetaCache(catalog));
255255
}
256256

257257
private CatalogMetaCache getCatalogMetaCache(ExternalTable table) {

‎regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,7 @@ suite("test_hive_meta_cache", "p0,external,hive,external_docker,external_docker_
327327
def sql_sct01_3col = sql "show create table test_hive_meta_cache_db.sales"
328328
println "${sql_sct01_3col}"
329329
assertTrue(sql_sct01_3col[0][1].contains("CREATE TABLE `sales`(\n `id` int,\n `amount` double)\nPARTITIONED BY (\n `year` int)"));
330-
330+
331331
// add a new column in hive
332332
hive_docker "alter table test_hive_meta_cache_db.sales add columns(k1 string)"
333333
// desc table, 4 columns
@@ -359,4 +359,3 @@ suite("test_hive_meta_cache", "p0,external,hive,external_docker,external_docker_
359359
}
360360
}
361361
}
362-

0 commit comments

Comments
 (0)