HIVE-29396: Drop the property hive.metastore.try.direct.sql.ddl

common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

@@ -97,7 +97,6 @@ public class HiveConf extends Configuration {
   private static byte[] confVarByteArray = null;
 
   private static final Map<String, ConfVars> vars = new HashMap<String, ConfVars>();
-  private static final Map<String, ConfVars> metaConfs = new HashMap<String, ConfVars>();
   private final List<String> restrictList = new ArrayList<String>();
   private final Set<String> hiddenSet = new HashSet<String>();
   private final Set<String> lockedSet = new HashSet<>();
@@ -252,118 +251,11 @@ private static URL checkConfigFile(File f) {
     }
   }
 
-
-
-
   @InterfaceAudience.Private
   public static final String PREFIX_LLAP = "llap.";
   @InterfaceAudience.Private
   public static final String PREFIX_HIVE_LLAP = "hive.llap.";
 
-  /**
-   * Metastore related options that the db is initialized against. When a conf
-   * var in this is list is changed, the metastore instance for the CLI will
-   * be recreated so that the change will take effect.
-   */
-  public static final HiveConf.ConfVars[] metaVars = {
-      HiveConf.ConfVars.METASTORE_WAREHOUSE,
-      HiveConf.ConfVars.REPL_DIR,
-      HiveConf.ConfVars.METASTORE_URIS,
-      HiveConf.ConfVars.METASTORE_SELECTION,
-      HiveConf.ConfVars.METASTORE_SERVER_PORT,
-      HiveConf.ConfVars.METASTORE_THRIFT_CONNECTION_RETRIES,
-      HiveConf.ConfVars.METASTORE_THRIFT_FAILURE_RETRIES,
-      HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY,
-      HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
-      HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME,
-      HiveConf.ConfVars.METASTORE_PWD,
-      HiveConf.ConfVars.METASTORE_CONNECT_URL_HOOK,
-      HiveConf.ConfVars.METASTORE_CONNECT_URL_KEY,
-      HiveConf.ConfVars.METASTORE_SERVER_MIN_THREADS,
-      HiveConf.ConfVars.METASTORE_SERVER_MAX_THREADS,
-      HiveConf.ConfVars.METASTORE_INT_ORIGINAL,
-      HiveConf.ConfVars.METASTORE_INT_ARCHIVED,
-      HiveConf.ConfVars.METASTORE_INT_EXTRACTED,
-      HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE,
-      HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL,
-      HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL,
-      HiveConf.ConfVars.METASTORE_TOKEN_SIGNATURE,
-      HiveConf.ConfVars.METASTORE_CACHE_PINOBJTYPES,
-      HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE,
-      HiveConf.ConfVars.METASTORE_VALIDATE_TABLES,
-      HiveConf.ConfVars.METASTORE_DATANUCLEUS_INIT_COL_INFO,
-      HiveConf.ConfVars.METASTORE_VALIDATE_COLUMNS,
-      HiveConf.ConfVars.METASTORE_VALIDATE_CONSTRAINTS,
-      HiveConf.ConfVars.METASTORE_STORE_MANAGER_TYPE,
-      HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL,
-      HiveConf.ConfVars.METASTORE_TRANSACTION_ISOLATION,
-      HiveConf.ConfVars.METASTORE_CACHE_LEVEL2,
-      HiveConf.ConfVars.METASTORE_CACHE_LEVEL2_TYPE,
-      HiveConf.ConfVars.METASTORE_IDENTIFIER_FACTORY,
-      HiveConf.ConfVars.METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK,
-      HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS,
-      HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX,
-      HiveConf.ConfVars.METASTORE_EVENT_LISTENERS,
-      HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS,
-      HiveConf.ConfVars.METASTORE_EVENT_CLEAN_FREQ,
-      HiveConf.ConfVars.METASTORE_EVENT_EXPIRY_DURATION,
-      HiveConf.ConfVars.METASTORE_EVENT_MESSAGE_FACTORY,
-      HiveConf.ConfVars.METASTORE_FILTER_HOOK,
-      HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
-      HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS,
-      HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS,
-      HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_OBJECTS_MAX,
-      HiveConf.ConfVars.METASTORE_INIT_HOOKS,
-      HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS,
-      HiveConf.ConfVars.HMS_HANDLER_ATTEMPTS,
-      HiveConf.ConfVars.HMS_HANDLER_INTERVAL,
-      HiveConf.ConfVars.HMS_HANDLER_FORCE_RELOAD_CONF,
-      HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN,
-      HiveConf.ConfVars.METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS,
-      HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES,
-      HiveConf.ConfVars.USERS_IN_ADMIN_ROLE,
-      HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
-      HiveConf.ConfVars.HIVE_TXN_MANAGER,
-      HiveConf.ConfVars.HIVE_TXN_TIMEOUT,
-      HiveConf.ConfVars.HIVE_TXN_OPERATIONAL_PROPERTIES,
-      HiveConf.ConfVars.HIVE_TXN_HEARTBEAT_THREADPOOL_SIZE,
-      HiveConf.ConfVars.HIVE_TXN_MAX_OPEN_BATCH,
-      HiveConf.ConfVars.HIVE_TXN_RETRYABLE_SQLEX_REGEX,
-      HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_TUNER,
-      HiveConf.ConfVars.HIVE_METASTORE_STATS_NDV_DENSITY_FUNCTION,
-      HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_ENABLED,
-      HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_SIZE,
-      HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS,
-      HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_FPP,
-      HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_VARIANCE,
-      HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_TTL,
-      HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT,
-      HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_READER_WAIT,
-      HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_FULL,
-      HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_CLEAN_UNTIL,
-      HiveConf.ConfVars.METASTORE_FASTPATH,
-      HiveConf.ConfVars.METASTORE_HBASE_FILE_METADATA_THREADS,
-      HiveConf.ConfVars.METASTORE_WM_DEFAULT_POOL_SIZE
-      };
-
-  /**
-   * User configurable Metastore vars
-   */
-  static final HiveConf.ConfVars[] metaConfVars = {
-      HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL,
-      HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL_DDL,
-      HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
-      HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN,
-      HiveConf.ConfVars.METASTORE_CAPABILITY_CHECK,
-      HiveConf.ConfVars.METASTORE_DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES
-  };
-
-  static {
-    for (ConfVars confVar : metaConfVars) {
-      metaConfs.put(confVar.varname, confVar);
-    }
-  }
-
   public static final String HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME = "hive.llap.daemon.service.principal";
   public static final String HIVE_SERVER2_AUTHENTICATION_LDAP_USERMEMBERSHIPKEY_NAME =
       "hive.server2.authentication.ldap.userMembershipKey";
@@ -6466,10 +6358,6 @@ public static ConfVars getConfVars(String name) {
     return vars.get(name);
   }
 
-  public static ConfVars getMetaConf(String name) {
-    return metaConfs.get(name);
-  }
-
   public String getVar(ConfVars var) {
     return getVar(this, var);
   }
@@ -6665,12 +6553,6 @@ private void initialize(Class<?> cls) {
     // Overlay the values of any system properties and manual overrides
     applySystemProperties();
 
-    if ((this.get("hive.metastore.ds.retry.attempts") != null) ||
-      this.get("hive.metastore.ds.retry.interval") != null) {
-        LOG.warn("DEPRECATED: hive.metastore.ds.retry.* no longer has any effect.  " +
-        "Use hive.hmshandler.retry.* instead");
-    }
-
     // if the running class was loaded directly (through eclipse) rather than through a
     // jar then this would be needed
     if (hiveJar == null) {

ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerHS2.java

@@ -27,6 +27,7 @@
 
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.DisallowTransformHook;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException;
@@ -126,8 +127,8 @@ private void verifyParamSettability(List<String> settableParams, HiveConf proces
     assertConfModificationException(processedConf, "dummy.param");
     // does not make sense to have any of the metastore config variables to be
     // modifiable
-    for (ConfVars metaVar : HiveConf.metaVars) {
-      assertConfModificationException(processedConf, metaVar.varname);
+    for (MetastoreConf.ConfVars metaVar : MetastoreConf.metaVars) {
+      assertConfModificationException(processedConf, metaVar.getHiveName());
     }
   }
 

ql/src/test/queries/clientpositive/alter_table_column_stats.q

@@ -266,8 +266,8 @@ use default;
 drop database statsdb1;
 drop database statsdb2;
 
--- Test for external tables with metastore.try.direct.sql.ddl as false
-set metaconf:metastore.try.direct.sql.ddl=false;
+-- Test for external tables with metastore.try.direct.sql as false
+set metaconf:metastore.try.direct.sql=false;
 
 drop database if exists statsdb1;
 create database statsdb1;

standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java

@@ -250,7 +250,6 @@ public String toString() {
       ConfVars.METASTORE_PARTITIONS_PARAMETERS_EXCLUDE_PATTERN,
       // Add metaConfVars here as well
       ConfVars.TRY_DIRECT_SQL,
-      ConfVars.TRY_DIRECT_SQL_DDL,
       ConfVars.CLIENT_SOCKET_TIMEOUT,
       ConfVars.PARTITION_NAME_WHITELIST_PATTERN,
       ConfVars.PARTITION_ORDER_EXPR,
@@ -264,7 +263,6 @@ public String toString() {
    */
   public static final MetastoreConf.ConfVars[] metaConfVars = {
       ConfVars.TRY_DIRECT_SQL,
-      ConfVars.TRY_DIRECT_SQL_DDL,
       ConfVars.CLIENT_SOCKET_TIMEOUT,
       ConfVars.PARTITION_NAME_WHITELIST_PATTERN,
       ConfVars.PARTITION_ORDER_EXPR,
@@ -1663,12 +1661,6 @@ public enum ConfVars {
             "work for all queries on your datastore. If all SQL queries fail (for example, your\n" +
             "metastore is backed by MongoDB), you might want to disable this to save the\n" +
             "try-and-fall-back cost."),
-    TRY_DIRECT_SQL_DDL("metastore.try.direct.sql.ddl", "hive.metastore.try.direct.sql.ddl", true,
-        "Same as hive.metastore.try.direct.sql, for read statements within a transaction that\n" +
-            "modifies metastore data. Due to non-standard behavior in Postgres, if a direct SQL\n" +
-            "select query has incorrect syntax or something similar inside a transaction, the\n" +
-            "entire transaction will fail and fall-back to DataNucleus will not be possible. You\n" +
-            "should disable the usage of direct SQL inside transactions if that happens in your case."),
     TXN_MAX_OPEN_BATCH("metastore.txn.max.open.batch", "hive.txn.max.open.batch", 1000,
         "Maximum number of transactions that can be fetched in one call to open_txns().\n" +
             "This controls how many transactions streaming agents such as Flume or Storm open\n" +

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java

@@ -4174,11 +4174,7 @@ public GetHelper(String catalogName, String dbName, String tblName,
       this.doTrace = LOG.isDebugEnabled();
       this.isInTxn = isActiveTransaction();
 
-      // SQL usage inside a larger transaction (e.g. droptable) may not be desirable because
-      // some databases (e.g. Postgres) abort the entire transaction when any query fails, so
-      // the fallback from failed SQL to JDO is not possible.
-      boolean isConfigEnabled = MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL)
-          && (MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL_DDL) || !isInTxn);
+      boolean isConfigEnabled = MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL);
       if (isConfigEnabled && directSql == null) {
         directSql = new MetaStoreDirectSql(pm, getConf(), "");
       }

standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestGetPartitionsUsingProjectionAndFilterSpecs.java

@@ -113,7 +113,6 @@ public void setup() throws TException {
     tbl = client.getTable(dbName, tblName);
     // set directSQL to true explicitly
     client.setMetaConf(ConfVars.TRY_DIRECT_SQL.getVarname(), "true");
-    client.setMetaConf(ConfVars.TRY_DIRECT_SQL_DDL.getVarname(), "true");
   }
 
   @After

standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java

@@ -60,7 +60,6 @@ public static void startMetaStoreServer() throws Exception {
     Configuration metastoreConf = MetastoreConf.newMetastoreConf();
     MetastoreConf.setClass(metastoreConf, ConfVars.EXPRESSION_PROXY_CLASS,
       MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class);
-    MetastoreConf.setBoolVar(metastoreConf, ConfVars.TRY_DIRECT_SQL_DDL, false);
     MetaStoreTestUtils.setConfForStandloneMode(metastoreConf);
     int msPort = MetaStoreServerUtils.startMetaStore(metastoreConf);
     conf = MetastoreConf.newMetastoreConf();
@@ -97,14 +96,6 @@ public void testGetMetaConfDefaultEmptyString() throws TException {
     assertEquals(expected, actual);
   }
 
-  @Test
-  public void testGetMetaConfOverridden() throws TException {
-    ConfVars metaConfVar = ConfVars.TRY_DIRECT_SQL_DDL;
-    String expected = "false";
-    String actual = hmsc.getMetaConf(metaConfVar.toString());
-    assertEquals(expected, actual);
-  }
-
   @Test
   public void testGetMetaConfUnknownPreperty() throws TException {
     String unknownPropertyName = "hive.meta.foo.bar";

standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java

@@ -1198,7 +1198,6 @@ public void testAlterTableAlreadyExists() throws Exception {
   @Test
   public void testAlterTableExpectedPropertyMatch() throws Exception {
     Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL));
-    Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL_DDL));
     Table originalTable = testTables[0];
 
     EnvironmentContext context = new EnvironmentContext();
@@ -1213,7 +1212,6 @@ public void testAlterTableExpectedPropertyMatch() throws Exception {
   @Test(expected = MetaException.class)
   public void testAlterTableExpectedPropertyDifferent() throws Exception {
     Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL));
-    Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL_DDL));
     Table originalTable = testTables[0];
 
     EnvironmentContext context = new EnvironmentContext();
@@ -1234,7 +1232,6 @@ public void testAlterTableExpectedPropertyDifferent() throws Exception {
   @Test
   public void testAlterTableExpectedPropertyConcurrent() throws Exception {
     Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL));
-    Assume.assumeTrue(MetastoreConf.getBoolVar(metaStore.getConf(), ConfVars.TRY_DIRECT_SQL_DDL));
     Table originalTable = testTables[0];
 
     originalTable.getParameters().put("snapshot", "0");