Skip to content

Commit f84cfcf

Browse files
authored
[opt](config) support dynamically change hms timeout (#59051)
### What problem does this PR solve? Now we use FE config `hive_metastore_client_timeout_second` to force reset `hive.metastore.client.socket.timeout`, but there are some problem: 1. config `hive_metastore_client_timeout_second` is global, can not set for each catalog; 2. config `hive_metastore_client_timeout_second` is Mutable, but it will not take effect because of `ThriftHMSCachedClient`; How to improve: keep config `hive_metastore_client_timeout_second` as the default timeout, while user do not override `hive.metastore.client.socket.timeout`, and make the config immutable.
1 parent e95d266 commit f84cfcf

File tree

4 files changed

+22
-8
lines changed

4 files changed

+22
-8
lines changed

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2246,7 +2246,7 @@ public class Config extends ConfigBase {
22462246
* The default connection timeout for hive metastore.
22472247
* hive.metastore.client.socket.timeout
22482248
*/
2249-
@ConfField(mutable = true, masterOnly = false)
2249+
@ConfField(mutable = false, masterOnly = false)
22502250
public static long hive_metastore_client_timeout_second = 10;
22512251

22522252
/**

fe/fe-core/src/main/java/org/apache/doris/datasource/hive/ThriftHMSCachedClient.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@
3737
import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
3838
import org.apache.hadoop.hive.common.ValidWriteIdList;
3939
import org.apache.hadoop.hive.conf.HiveConf;
40-
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
4140
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
4241
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
4342
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
@@ -98,10 +97,6 @@ public class ThriftHMSCachedClient implements HMSCachedClient {
9897

9998
public ThriftHMSCachedClient(HiveConf hiveConf, int poolSize, ExecutionAuthenticator executionAuthenticator) {
10099
Preconditions.checkArgument(poolSize > 0, poolSize);
101-
if (hiveConf != null) {
102-
HiveConf.setVar(hiveConf, ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
103-
String.valueOf(Config.hive_metastore_client_timeout_second));
104-
}
105100
this.hiveConf = hiveConf;
106101
this.poolSize = poolSize;
107102
this.isClosed = false;

fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import lombok.Getter;
3131
import org.apache.commons.lang3.StringUtils;
3232
import org.apache.hadoop.hive.conf.HiveConf;
33+
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
3334

3435
import java.util.HashMap;
3536
import java.util.Map;
@@ -198,8 +199,11 @@ private void checkAndInit() {
198199
if (StringUtils.isNotBlank(hmsUserName)) {
199200
hiveConf.set(AuthenticationConfig.HADOOP_USER_NAME, hmsUserName);
200201
}
201-
HiveConf.setVar(hiveConf, HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
202-
String.valueOf(Config.hive_metastore_client_timeout_second));
202+
if (!userOverriddenHiveConfig.containsKey(ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT.toString())) {
203+
// use Config.hive_metastore_client_timeout_second as default timeout
204+
HiveConf.setVar(hiveConf, HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
205+
String.valueOf(Config.hive_metastore_client_timeout_second));
206+
}
203207
initHadoopAuthenticator();
204208
}
205209

fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSPropertiesTest.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,4 +134,19 @@ public void testHmsKerberosParams() throws UserException {
134134
HiveHMSProperties hmsProperties = getHMSProperties(params);
135135
Assertions.assertNotNull(hmsProperties);
136136
}
137+
138+
@Test
139+
public void testHmsTimeoutParams() throws UserException {
140+
Map<String, String> params = createBaseParams();
141+
// case1: normal case, use Config.hive_metastore_client_timeout_second as default
142+
HiveHMSProperties hmsProperties = getHMSProperties(params);
143+
HiveConf hiveConf = hmsProperties.getHiveConf();
144+
Assertions.assertEquals(String.valueOf(Config.hive_metastore_client_timeout_second),
145+
hiveConf.get("hive.metastore.client.socket.timeout", null));
146+
147+
params.put("hive.metastore.client.socket.timeout", "123");
148+
hmsProperties = getHMSProperties(params);
149+
hiveConf = hmsProperties.getHiveConf();
150+
Assertions.assertEquals("123", hiveConf.get("hive.metastore.client.socket.timeout", null));
151+
}
137152
}

0 commit comments

Comments
 (0)