HIVE-29073: Overlay modified session level metaconf to new hmsclient connection

Author: Aggarwal-Raghav

ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java

@@ -272,10 +272,25 @@ public Map<String, String> getHiveVariable() {
     }
     if (register) {
       ss.getOverriddenConfigurations().put(key, value);
+
+      if (metaConfOverlayList.contains(key)) {
+        try (Hive hive = Hive.get(ss.getConf())) {
+          // Resources will be closed automatically
+          hive.close(true);
+        } catch (Exception e) {
+          LOG.error("Error closing metastore connection", e);
+          throw new RuntimeException(e);
+        }
+      }
+
     }
     return result;
   }
 
+  // This is just temporary idea list. Will make this configurable later.
+  private static final Set<String> metaConfOverlayList =
+      Set.of("fs.s3a.endpoint", "fs.s3a.security.credential.provider.path");
+
   private SortedMap<String,String> propertiesToSortedMap(Properties p){
     SortedMap<String,String> sortedPropMap = new TreeMap<String,String>();
     for (Map.Entry<Object, Object> entry : p.entrySet() ){

standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java

@@ -761,6 +761,7 @@ private void open() throws MetaException {
           LOG.error(errMsg, e);
         }
         if (isConnected) {
+          overlayMetaConfig();
           break;
         }
       }
@@ -790,6 +791,21 @@ private void open() throws MetaException {
     snapshotActiveConf();
   }
 
+  private void overlayMetaConfig() {
+    String overlayConfig = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.HIVE_METACONF_OVERLAY);
+    for (String entry : overlayConfig.split(",")) {
+      String confVal = conf.get(entry.trim());
+      if (!org.apache.commons.lang3.StringUtils.isBlank(confVal)) {
+        try {
+          setMetaConf(entry.trim(), confVal);
+        } catch (TException e) {
+          LOG.error(
+              "Failed to set metastore config for {} with value {}", entry.trim(), confVal, e);
+        }
+      }
+    }
+  }
+
   // wraps the underlyingTransport in the appropriate transport based on mode of authentication
   private TTransport createAuthBinaryTransport(URI store, TTransport underlyingTransport)
       throws MetaException {

standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java

@@ -1824,6 +1824,12 @@ public enum ConfVars {
     HIVE_TXN_STATS_ENABLED("hive.txn.stats.enabled", "hive.txn.stats.enabled", true,
         "Whether Hive supports transactional stats (accurate stats for transactional tables)"),
 
+    HIVE_METACONF_OVERLAY(
+        "hive.metastore.conf.overlay",
+        "hive.metastore.conf.overlay",
+        "fs.s3a.endpoint, fs.s3a.security.credential.provider.path",
+        "The list of configurations that needs to be overlayed to HiveMetasStore on client connection"),
+
     // External RDBMS support
     USE_CUSTOM_RDBMS("metastore.use.custom.database.product",
         "hive.metastore.use.custom.database.product", false,