branch-3.1: [chore](param-refactor)Optimize catalog parameter handling and add Hive hadoop.username mapping apache#54551 (apache#54753)

github-actions[bot] · CalvinKirs · web-flow · commit ef2c200b6f2c · 2025-08-14T17:39:26.000+08:00
Cherry-picked from apache#54551 Co-authored-by: Calvin Kirs <guoqiang@selectdb.com>
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
@@ -30,8 +30,7 @@ public abstract class AuthenticationConfig {
     public static String HADOOP_USER_NAME = "hadoop.username";
     public static String HADOOP_KERBEROS_PRINCIPAL = "hadoop.kerberos.principal";
     public static String HADOOP_KERBEROS_KEYTAB = "hadoop.kerberos.keytab";
-    public static String HIVE_KERBEROS_PRINCIPAL = "hive.metastore.kerberos.principal";
-    public static String HIVE_KERBEROS_KEYTAB = "hive.metastore.kerberos.keytab.file";
+    public static String HADOOP_SECURITY_AUTH_TO_LOCAL = "hadoop.security.auth_to_local";
     public static String DORIS_KRB5_DEBUG = "doris.krb5.debug";
     private static final String DEFAULT_HADOOP_USERNAME = "hadoop";
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
@@ -73,6 +73,7 @@ public class AWSGlueMetaStoreBaseProperties {
     public static AWSGlueMetaStoreBaseProperties of(Map<String, String> properties) {
         AWSGlueMetaStoreBaseProperties propertiesObj = new AWSGlueMetaStoreBaseProperties();
         ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj, properties);
+        propertiesObj.checkAndInit();
         return propertiesObj;
     }
 
@@ -102,7 +103,7 @@ private ParamRules buildRules() {
                 .require(glueEndpoint, "glue.endpoint or aws.endpoint or aws.glue.endpoint is required");
     }
 
-    public void checkAndInit() {
+    private void checkAndInit() {
         buildRules().validate();
 
         Matcher matcher = ENDPOINT_PATTERN.matcher(glueEndpoint.toLowerCase());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java
@@ -42,7 +42,6 @@ public AWSGlueMetaStoreProperties(Map<String, String> origProps) {
     public void initNormalizeAndCheckProps() {
         super.initNormalizeAndCheckProps();
         baseProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
-        baseProperties.checkAndInit();
     }
 
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
@@ -43,6 +43,7 @@ public class AliyunDLFBaseProperties {
     protected String dlfSessionToken = "";
 
     @ConnectorProperty(names = {"dlf.region"},
+            required = false,
             description = "The region of the Aliyun DLF.")
     protected String dlfRegion = "";
 
@@ -84,7 +85,7 @@ private ParamRules buildRules() {
                 .require(dlfSecretKey, "dlf.secret_key is required");
     }
 
-    public void checkAndInit() {
+    private void checkAndInit() {
         buildRules().validate();
         if (StringUtils.isBlank(dlfEndpoint) && StringUtils.isNotBlank(dlfRegion)) {
             if (BooleanUtils.toBoolean(dlfAccessPublic)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java
@@ -68,9 +68,15 @@ public class HMSBaseProperties {
 
     @ConnectorProperty(names = {"hadoop.security.authentication"},
             required = false,
-            description = "The authentication type of HDFS. The default value is 'none'.")
+            description = "The authentication type of HDFS. The default value is 'simple'.")
     private String hdfsAuthenticationType = "";
 
+    @ConnectorProperty(names = {"hive.metastore.username", "hadoop.username"},
+            required = false,
+            description = "The user name for the Hive Metastore service. "
+                    + "If not set, it will use the 'hadoop'.")
+    private String hmsUserName;
+
     @ConnectorProperty(names = {"hadoop.kerberos.principal"},
             required = false,
             description = "The principal of the kerberos authentication.")
@@ -98,6 +104,7 @@ public HMSBaseProperties(Map<String, String> origProps) {
     public static HMSBaseProperties of(Map<String, String> properties) {
         HMSBaseProperties propertiesObj = new HMSBaseProperties(properties);
         ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj, properties);
+        propertiesObj.checkAndInit();
         return propertiesObj;
     }
 
@@ -137,6 +144,10 @@ private void initHadoopAuthenticator() {
         if (StringUtils.isNotBlank(hiveMetastoreServicePrincipal)) {
             hiveConf.set("hive.metastore.kerberos.principal", hiveMetastoreServicePrincipal);
         }
+        if (StringUtils.isNotBlank(origProps.get(AuthenticationConfig.HADOOP_SECURITY_AUTH_TO_LOCAL))) {
+            hiveConf.set(AuthenticationConfig.HADOOP_SECURITY_AUTH_TO_LOCAL,
+                    origProps.get(AuthenticationConfig.HADOOP_SECURITY_AUTH_TO_LOCAL));
+        }
         if (this.hiveMetastoreAuthenticationType.equalsIgnoreCase("kerberos")) {
             hiveConf.set("hadoop.security.authentication", "kerberos");
             KerberosAuthenticationConfig authenticationConfig = new KerberosAuthenticationConfig(
@@ -169,12 +180,15 @@ private HiveConf loadHiveConfFromFile(String resourceConfig) {
         return CatalogConfigFileUtils.loadHiveConfFromHiveConfDir(resourceConfig);
     }
 
-    public void initAndCheckParams() {
+    private void checkAndInit() {
         buildRules().validate();
         this.hiveConf = loadHiveConfFromFile(hiveConfResourcesConfig);
         initUserHiveConfig(origProps);
         userOverriddenHiveConfig.forEach(hiveConf::set);
         hiveConf.set("hive.metastore.uris", hiveMetastoreUri);
+        if (StringUtils.isNotBlank(hmsUserName)) {
+            hiveConf.set(AuthenticationConfig.HADOOP_USER_NAME, hmsUserName);
+        }
         HiveConf.setVar(hiveConf, HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
                 String.valueOf(Config.hive_metastore_client_timeout_second));
         initHadoopAuthenticator();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
@@ -86,7 +86,6 @@ protected HMSGlueMetaStoreProperties(Type type, Map<String, String> origProps) {
     public void initNormalizeAndCheckProps() {
         super.initNormalizeAndCheckProps();
         baseProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
-        baseProperties.checkAndInit();
         initHiveConf();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java
@@ -55,7 +55,6 @@ public void initNormalizeAndCheckProps() {
         super.initNormalizeAndCheckProps();
         initRefreshParams();
         hmsBaseProperties = HMSBaseProperties.of(origProps);
-        hmsBaseProperties.initAndCheckParams();
         this.hiveConf = hmsBaseProperties.getHiveConf();
         this.executionAuthenticator = new HadoopExecutionAuthenticator(hmsBaseProperties.getHmsAuthenticator());
     }
@@ -66,6 +65,4 @@ private void initRefreshParams() {
         this.hmsEventsBatchSizePerRpc = hmsEventisBatchSizePerRpcInput;
     }
 
-
-
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
@@ -54,20 +54,16 @@ public String getIcebergCatalogType() {
     public void initNormalizeAndCheckProps() {
         super.initNormalizeAndCheckProps();
         glueProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
-        glueProperties.checkAndInit();
         s3Properties = S3Properties.of(origProps);
-        s3Properties.initNormalizeAndCheckProps();
     }
 
     @Override
     public Catalog initializeCatalog(String catalogName, List<StorageProperties> storageProperties) {
         Map<String, String> props = prepareBaseCatalogProps();
         appendS3Props(props);
         appendGlueProps(props);
-
         props.put("client.region", glueProperties.glueRegion);
 
-
         if (StringUtils.isNotBlank(warehouse)) {
             props.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
         } else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
@@ -59,7 +59,6 @@ public String getIcebergCatalogType() {
     public void initNormalizeAndCheckProps() {
         super.initNormalizeAndCheckProps();
         hmsBaseProperties = HMSBaseProperties.of(origProps);
-        hmsBaseProperties.initAndCheckParams();
         this.executionAuthenticator = new HadoopExecutionAuthenticator(hmsBaseProperties.getHmsAuthenticator());
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
@@ -48,7 +48,6 @@ public String getIcebergCatalogType() {
     public void initNormalizeAndCheckProps() {
         super.initNormalizeAndCheckProps();
         s3Properties = S3Properties.of(origProps);
-        s3Properties.initNormalizeAndCheckProps();
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java
@@ -39,7 +39,7 @@ protected PaimonFileSystemMetaStoreProperties(Map<String, String> props) {
     @Override
     public Catalog initializeCatalog(String catalogName, List<StorageProperties> storagePropertiesList) {
         buildCatalogOptions(storagePropertiesList);
-        Configuration conf = new Configuration(false);
+        Configuration conf = new Configuration();
         storagePropertiesList.forEach(storageProperties -> {
             for (Map.Entry<String, String> entry : storageProperties.getHadoopStorageConfig()) {
                 catalogOptions.set(entry.getKey(), entry.getValue());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
@@ -66,7 +66,6 @@ protected PaimonHMSMetaStoreProperties(Map<String, String> props) {
     public void initNormalizeAndCheckProps() {
         super.initNormalizeAndCheckProps();
         hmsBaseProperties = HMSBaseProperties.of(origProps);
-        hmsBaseProperties.initAndCheckParams();
         this.executionAuthenticator = new HadoopExecutionAuthenticator(hmsBaseProperties.getHmsAuthenticator());
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -129,6 +129,7 @@ public class S3Properties extends AbstractS3CompatibleProperties {
     public static S3Properties of(Map<String, String> properties) {
         S3Properties propertiesObj = new S3Properties(properties);
         ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj, properties);
+        propertiesObj.initNormalizeAndCheckProps();
         return propertiesObj;
     }
 
diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy
@@ -63,6 +63,10 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa
                     "fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
                     "hadoop.security.authentication" = "kerberos",
                     "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM",
+                    "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//
+                                   RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
+                                   RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*//
+                                   DEFAULT",
                     "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab"
                 );
             """
diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy
@@ -47,6 +47,10 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d
                 "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM",
                 "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab",
                 "hive.metastore.sasl.enabled " = "true",
+                "hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//
+                                   RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
+                                   RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*//
+                                   DEFAULT",
                 "hive.metastore.kerberos.principal" = "hive/hadoop-master@LABS.TERADATA.COM"
             );
         """

Original file line number	Diff line number	Diff line change
`@@ -73,6 +73,7 @@ public class AWSGlueMetaStoreBaseProperties {`
`73`	`73`	`public static AWSGlueMetaStoreBaseProperties of(Map<String, String> properties) {`
`74`	`74`	`AWSGlueMetaStoreBaseProperties propertiesObj = new AWSGlueMetaStoreBaseProperties();`
`75`	`75`	`ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj, properties);`
	`76`	`+ propertiesObj.checkAndInit();`
`76`	`77`	`return propertiesObj;`
`77`	`78`	`}`
`78`	`79`
`@@ -102,7 +103,7 @@ private ParamRules buildRules() {`
`102`	`103`	`.require(glueEndpoint, "glue.endpoint or aws.endpoint or aws.glue.endpoint is required");`
`103`	`104`	`}`
`104`	`105`
`105`		`- public void checkAndInit() {`
	`106`	`+ private void checkAndInit() {`
`106`	`107`	`buildRules().validate();`
`107`	`108`
`108`	`109`	`Matcher matcher = ENDPOINT_PATTERN.matcher(glueEndpoint.toLowerCase());`
Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,6 @@ public AWSGlueMetaStoreProperties(Map<String, String> origProps) {`
`42`	`42`	`public void initNormalizeAndCheckProps() {`
`43`	`43`	`super.initNormalizeAndCheckProps();`
`44`	`44`	`baseProperties = AWSGlueMetaStoreBaseProperties.of(origProps);`
`45`		`- baseProperties.checkAndInit();`
`46`	`45`	`}`
`47`	`46`
`48`	`47`
Original file line number	Diff line number	Diff line change
`@@ -86,7 +86,6 @@ protected HMSGlueMetaStoreProperties(Type type, Map<String, String> origProps) {`
`86`	`86`	`public void initNormalizeAndCheckProps() {`
`87`	`87`	`super.initNormalizeAndCheckProps();`
`88`	`88`	`baseProperties = AWSGlueMetaStoreBaseProperties.of(origProps);`
`89`		`- baseProperties.checkAndInit();`
`90`	`89`	`initHiveConf();`
`91`	`90`	`}`
`92`	`91`
Original file line number	Diff line number	Diff line change
`@@ -55,7 +55,6 @@ public void initNormalizeAndCheckProps() {`
`55`	`55`	`super.initNormalizeAndCheckProps();`
`56`	`56`	`initRefreshParams();`
`57`	`57`	`hmsBaseProperties = HMSBaseProperties.of(origProps);`
`58`		`- hmsBaseProperties.initAndCheckParams();`
`59`	`58`	`this.hiveConf = hmsBaseProperties.getHiveConf();`
`60`	`59`	`this.executionAuthenticator = new HadoopExecutionAuthenticator(hmsBaseProperties.getHmsAuthenticator());`
`61`	`60`	`}`
`@@ -66,6 +65,4 @@ private void initRefreshParams() {`
`66`	`65`	`this.hmsEventsBatchSizePerRpc = hmsEventisBatchSizePerRpcInput;`
`67`	`66`	`}`
`68`	`67`
`69`		`-`
`70`		`-`
`71`	`68`	`}`
Original file line number	Diff line number	Diff line change
`@@ -59,7 +59,6 @@ public String getIcebergCatalogType() {`
`59`	`59`	`public void initNormalizeAndCheckProps() {`
`60`	`60`	`super.initNormalizeAndCheckProps();`
`61`	`61`	`hmsBaseProperties = HMSBaseProperties.of(origProps);`
`62`		`- hmsBaseProperties.initAndCheckParams();`
`63`	`62`	`this.executionAuthenticator = new HadoopExecutionAuthenticator(hmsBaseProperties.getHmsAuthenticator());`
`64`	`63`	`}`
`65`	`64`
Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,6 @@ public String getIcebergCatalogType() {`
`48`	`48`	`public void initNormalizeAndCheckProps() {`
`49`	`49`	`super.initNormalizeAndCheckProps();`
`50`	`50`	`s3Properties = S3Properties.of(origProps);`
`51`		`- s3Properties.initNormalizeAndCheckProps();`
`52`	`51`	`}`
`53`	`52`
`54`	`53`	`@Override`